diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..efcfd48 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,7 @@ +[net] +git-fetch-with-cli = true + +[alias] +# Every development task, including the OpenAPI documents under `openapi/`. +# One entry point, so `cargo xtask` on its own lists what there is to do. +xtask = "run --quiet -p xtask --" diff --git a/.github/buildomat/jobs/build-linux.sh b/.github/buildomat/jobs/build-linux.sh index 4347a75..aa78108 100755 --- a/.github/buildomat/jobs/build-linux.sh +++ b/.github/buildomat/jobs/build-linux.sh @@ -2,16 +2,35 @@ #: #: name = "build-linux" #: variety = "basic" -#: target = "ubuntu-22.04" +#: target = "ubuntu-24.04" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/release/*", #: ] +#: access_repos = [ +#: "oxidecomputer/ipe", +#: "oxidecomputer/anodizer" +#: ] #: #: [[publish]] #: series = "linux" #: name = "vw" #: from_output = "/work/release/vw" +#: +#: [[publish]] +#: series = "linux" +#: name = "vw-agent" +#: from_output = "/work/release/vw-agent" +#: +#: [[publish]] +#: series = "linux" +#: name = "vw-svc" +#: from_output = "/work/release/vw-svc" +#: +#: [[publish]] +#: series = "linux" +#: name = "vw-analyzer" +#: from_output = "/work/release/vw-analyzer" set -o errexit set -o pipefail @@ -22,6 +41,9 @@ sudo apt-get install build-essential pkg-config libssl-dev libfontconfig-dev -y cargo --version rustc --version +banner "api" +cargo xtask openapi generate + banner "check" cargo fmt -- --check cargo clippy --all-targets -- --deny warnings @@ -30,3 +52,6 @@ banner "build" cargo build --release mkdir -p /work/release/ cp target/release/vw /work/release/ +cp target/release/vw-agent /work/release/ +cp target/release/vw-svc /work/release/ +cp target/release/vw-analyzer /work/release/ diff --git a/.github/buildomat/jobs/build.sh b/.github/buildomat/jobs/build.sh index 74e6862..e6d9115 100755 --- a/.github/buildomat/jobs/build.sh +++ b/.github/buildomat/jobs/build.sh @@ -2,16 +2,35 @@ #: #: name = "build" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/release/*", #: ] +#: access_repos = [ +#: "oxidecomputer/ipe", +#: "oxidecomputer/anodizer" +#: ] #: #: [[publish]] #: series = "illumos" #: name = "vw" #: from_output = "/work/release/vw" +#: +#: [[publish]] +#: series = "illumos" +#: name = "vw-agent" +#: from_output = "/work/release/vw-agent" +#: +#: [[publish]] +#: series = "illumos" +#: name = "vw-svc" +#: from_output = "/work/release/vw-svc" +#: +#: [[publish]] +#: series = "illumos" +#: name = "vw-analyzer" +#: from_output = "/work/release/vw-analyzer" set -o errexit set -o pipefail @@ -23,6 +42,9 @@ rustc --version export PKG_CONFIG_PATH=/opt/ooce/lib/amd64/pkgconfig +banner "api" +cargo xtask openapi generate + banner "check" cargo fmt -- --check cargo clippy --all-targets -- --deny warnings @@ -31,3 +53,6 @@ banner "build" cargo build --release mkdir -p /work/release/ cp target/release/vw /work/release/ +cp target/release/vw-agent /work/release/ +cp target/release/vw-svc /work/release/ +cp target/release/vw-analyzer /work/release/ diff --git a/.gitignore b/.gitignore index ea8c4bf..bf8ded2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ /target +*.jou +*.log +.srcs +.claude +*.redb diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index c0bf84b..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,94 +0,0 @@ -# VHDL Workspace - -`vw` is a tool to manage VHDL workspaces. `vw` is built using the rust Clap -crate. It is similar in spirit to Rust's `cargo`. The focus includes dependency -management and testbench execution using NVC simulator. - -Consider the example file `example/vw.toml`. This file describes a workspace -with a single dependency, the quartz repository. A repo dependency has a `repo` -property that's a path to a git repository accessible over https, a `branch` -property or a `commit` property that specifies either a branch or commit within -that repository to use and a `src` property that describes where to find the -VHDL code. - -The `src` property supports multiple formats: -- **Directory path**: A path to a directory containing VHDL files (e.g., `"hdl/ip/vhd"`). Use the `recursive` flag to include subdirectories. -- **Single file**: A path to a specific VHDL file (e.g., `"hdl/ip/vhd/uart_pkg.vhd"`). -- **Glob pattern**: A glob pattern to match specific files (e.g., `"hdl/ip/vhd/**/*.vhd"` or `"hdl/**/pkg_*.vhd"`). The `recursive` flag is ignored for glob patterns as they handle their own path traversal. - -## Dependency Management - -Executing `vw update` will do a few things: - -- For each dependency create a -`$HOME/.vw/deps/-` directory with the -contents for the target repository at the defined path. Only VHDL files with -either a `vhd` or `vhdl` extension are included. - -- For each dependency an entry in a `vw.lock` file is created. This is a JSON -file that tracks what versions of dependencies are being used in the workspace -and can map this workspace's dependencies to specific downloaded artifacts in -`$HOME/.vw/deps` may contain multiple versions of a given dependency. - -- Creates a vhdl_ls.toml configuration file for the vhdl_ls language server -that includes dependencies as libraries. The file to include in vhdl_ls for the -library is one that ends in package, e.g. `some_name_pkg.vhd`. If such a file -does not exist, it is not included as a library. Dependencies may have multiple -directories, and each directory should be searched for a package file ending in -`_pkg.vhd` for library inclusion. - -## Testbench Execution - -`vw test` provides intelligent testbench execution using NVC simulator with advanced dependency analysis. - -### Usage - -```bash -# Run a specific testbench -vw test my_testbench_tb - -# Run with specific VHDL standard -vw test my_testbench_tb --std 2008 - -# List all available testbenches -vw test --list -``` - -### Features - -**Smart Dependency Analysis**: Only includes the minimal set of VHDL files actually needed by each testbench: -- Analyzes `use work.package_name` statements -- Detects direct entity instantiations like `entity work.entity_name` -- Follows component declarations and instantiations -- Recursively resolves dependency chains - -**Intelligent File Filtering**: -- Includes only referenced files from defaultlib -- Excludes other testbenches while allowing common bench utilities -- Proper topological sorting ensures correct compilation order - -**NVC Integration**: -- Analyzes non-defaultlib libraries first using `nvc --std= --work= -M 256m -a ` -- Runs testbench simulation with `nvc --std= -M 256m -L . -a --check-synthesis -e -r --dump-arrays --format=fst --wave=.fst` -- Converts library names with hyphens to underscores for NVC compatibility - -**Testbench Discovery**: -- Automatically finds testbenches in `bench/` directory -- Supports both `.vhd` and `.vhdl` extensions -- Uses regex to identify entity declarations - -### File Organization - -Place testbenches in a `bench/` directory: -``` -project/ -├── src/ -│ ├── my_entity.vhd -│ └── my_package.vhd -├── bench/ -│ ├── my_testbench_tb.vhd # Individual testbenches -│ ├── other_testbench_tb.vhd -│ └── test_utils.vhd # Common utilities (included when referenced) -├── vw.toml -└── vhdl_ls.toml -``` diff --git a/Cargo.lock b/Cargo.lock index 16a7be5..2e70581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -26,6 +32,23 @@ dependencies = [ "libc", ] +[[package]] +name = "anodizer" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/anodizer?branch=ry%2Fvw-restructure#43fbe290c08b2af04b708bd189b4f0fcddb9b400" +dependencies = [ + "camino", + "clap", + "colored", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "tokio", + "vhdl_lang", + "vw-core", +] + [[package]] name = "anstream" version = "1.0.0" @@ -82,12 +105,231 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arboard" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +dependencies = [ + "clipboard-win", + "image 0.25.10", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation", + "parking_lot", + "percent-encoding", + "windows-sys 0.59.0", + "x11rb", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atomicwrites" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" +dependencies = [ + "rustix 0.38.44", + "tempfile", + "windows-sys 0.52.0", +] + +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-creds" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3b85155d265df828f84e53886ed9e427aed979dd8a39f5b8b2162c77e142d7" +dependencies = [ + "home", + "log", + "quick-xml 0.38.4", + "rust-ini", + "serde", + "thiserror 2.0.18", + "time", + "url", +] + +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-region" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "838b36c8dc927b6db1b6c6b8f5d05865f2213550b9e83bf92fa99ed6525472c0" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -100,6 +342,48 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bstr" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -118,6 +402,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -129,6 +419,24 @@ name = "camino" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] [[package]] name = "cc" @@ -148,17 +456,51 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", ] [[package]] @@ -192,7 +534,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -201,6 +543,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -224,1443 +584,5053 @@ dependencies = [ ] [[package]] -name = "core-foundation" -version = "0.9.4" +name = "combine" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ - "core-foundation-sys", - "libc", + "bytes", + "memchr", ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "compact_str" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] [[package]] -name = "core-graphics" -version = "0.23.2" +name = "compression-codecs" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-graphics-types", - "foreign-types", - "libc", + "compression-core", + "flate2", + "memchr", ] [[package]] -name = "core-graphics-types" -version = "0.1.3" +name = "compression-core" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "libc", -] +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] -name = "core-text" -version = "20.1.0" +name = "console" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ - "core-foundation", - "core-graphics", - "foreign-types", + "encode_unicode", "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] -name = "crc32fast" -version = "1.5.0" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "crossbeam-deque" -version = "0.8.6" +name = "const-oid" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "const-random-macro", ] [[package]] -name = "crossbeam-epoch" -version = "0.9.18" +name = "const-random-macro" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "crossbeam-utils", + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", ] [[package]] -name = "crossbeam-utils" -version = "0.8.21" +name = "constant_time_eq" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] -name = "dirs" -version = "5.0.1" +name = "core-foundation" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "dirs-sys 0.4.1", + "core-foundation-sys", + "libc", ] [[package]] -name = "dirs" -version = "6.0.0" +name = "core-foundation" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "dirs-sys 0.5.0", + "core-foundation-sys", + "libc", ] [[package]] -name = "dirs-sys" -version = "0.4.1" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.6", - "windows-sys 0.48.0", -] +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "dirs-sys" -version = "0.5.0" +name = "core-graphics" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types", + "foreign-types", "libc", - "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", ] [[package]] -name = "displaydoc" -version = "0.2.5" +name = "core-graphics-types" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" dependencies = [ - "proc-macro2", - "quote", - "syn", + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", ] [[package]] -name = "dlib" -version = "0.5.3" +name = "core-text" +version = "20.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" dependencies = [ - "libloading", + "core-foundation 0.9.4", + "core-graphics", + "foreign-types", + "libc", ] [[package]] -name = "dunce" -version = "1.0.5" +name = "cpp_demangle" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] [[package]] -name = "dwrote" -version = "0.11.5" +name = "cpufeatures" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b35532432acc8b19ceed096e35dfa088d3ea037fe4f3c085f1f97f33b4d02" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ - "lazy_static", "libc", - "winapi", - "wio", ] [[package]] -name = "either" -version = "1.15.0" +name = "cpufeatures" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] [[package]] -name = "enum-map" -version = "2.7.3" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "enum-map-derive", + "cfg-if", ] [[package]] -name = "enum-map-derive" -version = "0.17.0" +name = "crossbeam-channel" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "crossbeam-utils", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "crossbeam-deque" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] [[package]] -name = "errno" -version = "0.3.14" +name = "crossbeam-epoch" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "libc", - "windows-sys 0.61.2", + "crossbeam-utils", ] [[package]] -name = "fastrand" -version = "2.3.0" +name = "crossbeam-utils" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] -name = "fdeflate" -version = "0.3.7" +name = "crossterm" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "simd-adler32", + "bitflags 2.11.0", + "crossterm_winapi", + "futures-core", + "mio 1.2.2", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "crossterm_winapi" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] [[package]] -name = "fixedbitset" -version = "0.5.7" +name = "crunchy" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] -name = "flate2" -version = "1.1.9" +name = "crypto-bigint" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "crc32fast", - "miniz_oxide", + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", ] [[package]] -name = "float-ord" -version = "0.3.2" +name = "crypto-common" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] [[package]] -name = "fnv" -version = "1.0.7" +name = "crypto-common" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] [[package]] -name = "foldhash" -version = "0.1.5" +name = "curve25519-dalek" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version 0.4.1", + "subtle", +] [[package]] -name = "font-kit" -version = "0.14.3" +name = "curve25519-dalek-derive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ - "bitflags 2.11.0", - "byteorder", - "core-foundation", - "core-graphics", - "core-text", - "dirs 6.0.0", - "dwrote", - "float-ord", - "freetype-sys", - "lazy_static", - "libc", - "log", - "pathfinder_geometry", - "pathfinder_simd", - "walkdir", - "winapi", - "yeslogic-fontconfig-sys", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "foreign-types" -version = "0.5.0" +name = "daft" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +checksum = "37da1a58f7d13865d88632f596075dab24f01df73390b011d93742db7f6229a1" dependencies = [ - "foreign-types-macros", - "foreign-types-shared", + "daft-derive", + "paste", ] [[package]] -name = "foreign-types-macros" -version = "0.2.3" +name = "daft-derive" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +checksum = "c839348835bc8a59714d79797f2f1bca237ca043e3d2d5ce4e727606396fb834" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] -name = "foreign-types-shared" -version = "0.3.1" +name = "darling" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "darling_core" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "percent-encoding", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] -name = "freetype-sys" -version = "0.20.1" +name = "darling_macro" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "cc", - "libc", - "pkg-config", + "darling_core", + "quote", + "syn 2.0.117", ] [[package]] -name = "getrandom" -version = "0.2.17" +name = "dashmap" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "libc", - "wasi", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] -name = "getrandom" -version = "0.3.4" +name = "data-encoding" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] -name = "getrandom" -version = "0.4.2" +name = "debug-ignore" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "wasip2", - "wasip3", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "gif" -version = "0.12.0" +name = "defmt-macros" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ - "color_quant", - "weezl", + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "git2" -version = "0.18.3" +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "bitflags 2.11.0", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", + "thiserror 2.0.18", ] [[package]] -name = "glob" -version = "0.3.3" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "zeroize", +] [[package]] -name = "hashbrown" -version = "0.15.5" +name = "der-parser" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ - "foldhash", + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", ] [[package]] -name = "hashbrown" -version = "0.16.1" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] [[package]] -name = "heck" -version = "0.5.0" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] [[package]] -name = "iana-time-zone" -version = "0.1.65" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "dirs" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "cc", + "dirs-sys 0.4.1", ] [[package]] -name = "icu_collections" -version = "2.1.1" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", + "dirs-sys 0.5.0", ] [[package]] -name = "icu_locale_core" -version = "2.1.1" +name = "dirs-sys" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", ] [[package]] -name = "icu_normalizer" -version = "2.1.1" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", + "libc", + "option-ext", + "redox_users 0.5.2", + "windows-sys 0.61.2", ] [[package]] -name = "icu_normalizer_data" -version = "2.1.1" +name = "dispatch2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.0", + "objc2", +] [[package]] -name = "icu_properties" -version = "2.1.2" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "icu_properties_data" -version = "2.1.2" +name = "dlib" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] [[package]] -name = "icu_provider" -version = "2.1.1" +name = "dlv-list" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", + "const-random", ] [[package]] -name = "id-arena" -version = "2.3.0" +name = "downcast-rs" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] -name = "idna" -version = "1.1.0" +name = "drift" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "a453bee38ca50a50f5b9dd2a3802e6972020bbff0f0040701b28f002a2490d27" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "anyhow", + "indexmap", + "openapiv3", + "regex", + "serde", + "serde_json", ] [[package]] -name = "idna_adapter" -version = "1.2.1" +name = "dropshot" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "803c4a57fd2a611df2ddd7069a62a565253368ee96ad4e5ac2e74e625dcf8430" dependencies = [ - "icu_normalizer", - "icu_properties", + "async-compression", + "async-stream", + "async-trait", + "base64", + "bytes", + "camino", + "chrono", + "debug-ignore", + "dropshot_endpoint", + "form_urlencoded", + "futures", + "hostname 0.4.2", + "http", + "http-body-util", + "hyper", + "hyper-util", + "indexmap", + "multer", + "openapiv3", + "paste", + "percent-encoding", + "rustls", + "rustls-pemfile", + "schemars", + "scopeguard", + "semver 1.0.28", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1 0.11.0", + "slog", + "slog-async", + "slog-bunyan", + "slog-json", + "slog-term", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "toml 1.1.4+spec-1.1.0", + "uuid", + "version_check", + "waitgroup", ] [[package]] -name = "image" -version = "0.24.9" +name = "dropshot-api-manager" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +checksum = "04789d14d7af239289cf53064f28f3b3e4fbefc0d716ebdb45a2a76d9a59d8a6" dependencies = [ - "bytemuck", - "byteorder", - "color_quant", - "jpeg-decoder", - "num-traits", - "png", + "anyhow", + "atomicwrites", + "camino", + "clap", + "debug-ignore", + "drift", + "dropshot", + "dropshot-api-manager-types", + "fs-err", + "git-stub", + "git-stub-vcs", + "hex", + "indent_write", + "newtype_derive", + "openapiv3", + "owo-colors", + "paste", + "rayon", + "semver 1.0.28", + "serde_json", + "sha2", + "similar", + "supports-color", + "textwrap", + "thiserror 2.0.18", ] [[package]] -name = "indexmap" -version = "2.13.0" +name = "dropshot-api-manager-types" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "7e89e6f3e5558d7eab06739cbc92b178605f20693715e7640eee6405fd655582" dependencies = [ - "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "anyhow", + "camino", + "paste", + "semver 1.0.28", + "serde_json", ] [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.14.0" +name = "dropshot_endpoint" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "5f176851eb52822c728ad7a1c5aea0e3c95e68d9876cfa06ae32d5018730acb4" dependencies = [ - "either", + "heck", + "proc-macro2", + "quote", + "semver 1.0.28", + "serde", + "serde_tokenstream", + "syn 2.0.117", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "jobserver" -version = "0.1.34" +name = "dwrote" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "9e1b35532432acc8b19ceed096e35dfa088d3ea037fe4f3c085f1f97f33b4d02" dependencies = [ - "getrandom 0.3.4", + "lazy_static", "libc", + "winapi", + "wio", ] [[package]] -name = "jpeg-decoder" -version = "0.3.2" +name = "dyn-clone" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] -name = "js-sys" -version = "0.3.94" +name = "ecdsa" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "once_cell", - "wasm-bindgen", + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "ed25519" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "signature", +] [[package]] -name = "leb128fmt" -version = "0.1.0" +name = "ed25519-dalek" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "sha2", + "subtle", +] [[package]] -name = "libc" -version = "0.2.183" +name = "either" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] -name = "libgit2-sys" -version = "0.16.2+1.7.2" +name = "elliptic-curve" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", ] [[package]] -name = "libloading" -version = "0.8.9" +name = "encode_unicode" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", - "windows-link", ] [[package]] -name = "libredox" -version = "0.1.14" +name = "enum-map" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" dependencies = [ - "libc", + "enum-map-derive", ] [[package]] -name = "libssh2-sys" -version = "0.3.1" +name = "enum-map-derive" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "libz-sys" -version = "1.1.25" +name = "env_filter" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", + "log", + "regex", ] [[package]] -name = "linux-raw-sys" -version = "0.12.1" +name = "env_logger" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] [[package]] -name = "litemap" -version = "0.8.1" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "lock_api" -version = "0.4.14" +name = "erased-serde" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" dependencies = [ - "scopeguard", + "serde", ] [[package]] -name = "log" -version = "0.4.29" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] -name = "memchr" -version = "2.8.0" +name = "error-code" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] -name = "miniz_oxide" -version = "0.8.9" +name = "fastrand" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" dependencies = [ - "adler2", "simd-adler32", ] [[package]] -name = "mio" -version = "1.1.1" +name = "ff" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", + "rand_core 0.6.4", + "subtle", ] [[package]] -name = "netrc" -version = "0.4.1" +name = "fiat-crypto" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9a91b326434fca226707ed8ec1fd22d4e1c96801abdf10c412afdc7d97116e0" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] -name = "num-traits" -version = "0.2.19" +name = "filedescriptor" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" dependencies = [ - "autocfg", + "libc", + "thiserror 1.0.69", + "winapi", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "filetime" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "openssl-probe" -version = "0.1.6" +name = "fixedbitset" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] -name = "openssl-sys" -version = "0.9.112" +name = "flate2" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", + "crc32fast", + "miniz_oxide", ] [[package]] -name = "option-ext" +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "pad" -version = "0.1.6" +name = "font-kit" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3" dependencies = [ - "unicode-width", + "bitflags 2.11.0", + "byteorder", + "core-foundation 0.9.4", + "core-graphics", + "core-text", + "dirs 6.0.0", + "dwrote", + "float-ord", + "freetype-sys", + "lazy_static", + "libc", + "log", + "pathfinder_geometry", + "pathfinder_simd", + "walkdir", + "winapi", + "yeslogic-fontconfig-sys", ] [[package]] -name = "parking_lot" -version = "0.12.5" +name = "foreign-types" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ - "lock_api", - "parking_lot_core", + "foreign-types-macros", + "foreign-types-shared", ] [[package]] -name = "parking_lot_core" -version = "0.9.12" +name = "foreign-types-macros" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "pathfinder_geometry" -version = "0.5.1" +name = "foreign-types-shared" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" -dependencies = [ - "log", - "pathfinder_simd", -] +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" [[package]] -name = "pathfinder_simd" -version = "0.5.5" +name = "form_urlencoded" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "rustc_version", + "percent-encoding", ] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "freetype-sys" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" +dependencies = [ + "cc", + "libc", + "pkg-config", +] [[package]] -name = "petgraph" -version = "0.8.3" +name = "fs-err" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", - "serde", + "autocfg", ] [[package]] -name = "pin-project-lite" -version = "0.2.17" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] -name = "pinned_vec" -version = "0.1.1" +name = "fsevent-sys" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268ad82d92622fb0a049ff14b01089b0f1bcd5c507fab44724394d328417348a" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] [[package]] -name = "pkg-config" +name = "futures" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] [[package]] -name = "plotters" -version = "0.3.7" +name = "futures-channel" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ - "chrono", - "font-kit", - "image", - "lazy_static", - "num-traits", - "pathfinder_geometry", - "plotters-backend", - "plotters-bitmap", - "plotters-svg", - "ttf-parser", - "wasm-bindgen", - "web-sys", + "futures-core", + "futures-sink", ] [[package]] -name = "plotters-backend" -version = "0.3.7" +name = "futures-core" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] -name = "plotters-bitmap" -version = "0.3.7" +name = "futures-executor" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ - "gif", - "image", - "plotters-backend", + "futures-core", + "futures-task", + "futures-util", ] [[package]] -name = "plotters-svg" -version = "0.3.7" +name = "futures-io" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] -name = "png" -version = "0.17.16" +name = "futures-macro" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ - "bitflags 1.3.2", - "crc32fast", - "fdeflate", - "flate2", - "miniz_oxide", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "potential_utf" -version = "0.1.4" +name = "futures-sink" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ - "zerovec", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "fuzzy-matcher" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" dependencies = [ - "proc-macro2", - "syn", + "thread_local", ] [[package]] -name = "proc-macro2" -version = "1.0.106" +name = "generic-array" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "unicode-ident", + "typenum", + "version_check", + "zeroize", ] [[package]] -name = "quote" -version = "1.0.45" +name = "gethostname" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "proc-macro2", + "rustix 1.1.4", + "windows-link 0.2.1", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "getrandom" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] [[package]] -name = "r-efi" -version = "6.0.0" +name = "getrandom" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] [[package]] -name = "rayon" -version = "1.11.0" +name = "getrandom" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "either", - "rayon-core", + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", + "wasm-bindgen", ] [[package]] -name = "rayon-core" -version = "1.13.0" +name = "gif" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "color_quant", + "weezl", ] [[package]] -name = "redox_syscall" -version = "0.5.18" +name = "git-stub" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +checksum = "4a42c6ab99e8c06cf862540c646368223b12225328d094fb0ed7755434376a1e" dependencies = [ - "bitflags 2.11.0", + "camino", + "hex", + "thiserror 2.0.18", ] [[package]] -name = "redox_users" -version = "0.4.6" +name = "git-stub-vcs" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "030ad7aaa1d066e8fb39ac2748f1a12b072eed4a4af8d2e073fc80ba74ab173c" dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", + "atomicwrites", + "camino", + "fs-err", + "git-stub", + "thiserror 2.0.18", ] [[package]] -name = "redox_users" -version = "0.5.2" +name = "git2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 2.0.18", + "bitflags 2.11.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe 0.1.6", + "openssl-sys", + "url", ] [[package]] -name = "regex" -version = "1.12.3" +name = "glob" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" dependencies = [ "aho-corasick", - "memchr", + "bstr", + "log", "regex-automata", "regex-syntax", ] [[package]] -name = "regex-automata" -version = "0.4.14" +name = "group" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "ff", + "rand_core 0.6.4", + "subtle", ] [[package]] -name = "regex-syntax" -version = "0.8.10" +name = "h2" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] [[package]] -name = "rustc_version" -version = "0.4.1" +name = "half" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "semver", + "cfg-if", + "crunchy", + "zerocopy", ] [[package]] -name = "rustix" -version = "1.1.4" +name = "hashbrown" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "bitflags 2.11.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", + "allocator-api2", + "equivalent", + "foldhash 0.1.5", ] [[package]] -name = "rustversion" -version = "1.0.22" +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] -name = "same-file" -version = "1.0.6" +name = "hashbrown" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "winapi-util", + "allocator-api2", ] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "semver" -version = "1.0.27" +name = "hermit-abi" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] -name = "serde" -version = "1.0.228" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "serde_core" -version = "1.0.228" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "serde_derive", + "digest 0.10.7", ] [[package]] -name = "serde_derive" -version = "1.0.228" +name = "home" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "windows-sys 0.61.2", ] [[package]] -name = "serde_json" -version = "1.0.149" +name = "hostname" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "libc", + "match_cfg", + "winapi", ] [[package]] -name = "serde_spanned" -version = "0.6.9" +name = "hostname" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" dependencies = [ - "serde", + "cfg-if", + "libc", + "windows-link 0.2.1", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "http" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ - "errno", - "libc", + "bytes", + "itoa", ] [[package]] -name = "simd-adler32" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" - -[[package]] -name = "smallvec" -version = "1.15.1" +name = "http-body" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] [[package]] -name = "socket2" -version = "0.6.3" +name = "http-body-util" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ - "libc", - "windows-sys 0.61.2", + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "strsim" -version = "0.11.1" +name = "httpdate" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "strum" -version = "0.27.2" +name = "hybrid-array" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ - "strum_macros", + "typenum", ] [[package]] -name = "strum_macros" -version = "0.27.2" +name = "hyper" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", ] [[package]] -name = "subst" -version = "0.3.8" +name = "hyper-rustls" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "memchr", - "unicode-width", + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", ] [[package]] -name = "syn" -version = "2.0.117" +name = "hyper-util" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-layer", + "tower-service", + "tracing", + "windows-registry", ] [[package]] -name = "synstructure" -version = "0.13.2" +name = "iana-time-zone" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ - "proc-macro2", - "quote", - "syn", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", ] [[package]] -name = "tempfile" -version = "3.27.0" +name = "iana-time-zone-haiku" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "fastrand", - "getrandom 0.4.2", - "once_cell", - "rustix", - "windows-sys 0.61.2", + "cc", ] [[package]] -name = "thiserror" -version = "1.0.69" +name = "icu_collections" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ - "thiserror-impl 1.0.69", + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "thiserror" -version = "2.0.18" +name = "icu_locale_core" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ - "thiserror-impl 2.0.18", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "thiserror-impl" -version = "1.0.69" +name = "icu_normalizer" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "proc-macro2", - "quote", - "syn", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", ] [[package]] -name = "thiserror-impl" -version = "2.0.18" +name = "icu_normalizer_data" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ - "proc-macro2", - "quote", - "syn", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] [[package]] -name = "tinystr" -version = "0.8.2" +name = "icu_properties_data" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", "zerovec", ] [[package]] -name = "tokio" -version = "1.50.0" +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "iddqd" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8450e1521a7518a32addf0b805ffcfbc8f9c558d2ad5769f068f2aeef5a81126" dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", + "allocator-api2", + "daft", + "equivalent", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "ref-cast", ] [[package]] -name = "tokio-macros" -version = "2.6.1" +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "proc-macro2", - "quote", - "syn", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "toml" -version = "0.8.23" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", + "icu_normalizer", + "icu_properties", ] [[package]] -name = "toml_datetime" -version = "0.6.11" +name = "ignore" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "7f8a7b8211e695a1d0cd91cace480d4d0bd57667ab10277cc412c5f7f4884f83" dependencies = [ - "serde", + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", ] [[package]] -name = "toml_edit" -version = "0.22.27" +name = "image" +version = "0.24.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", + "png 0.17.16", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "image" +version = "0.25.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png 0.18.1", + "tiff", +] [[package]] -name = "ttf-parser" -version = "0.20.0" +name = "indent_write" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" +checksum = "0cfe9645a18782869361d9c8732246be7b410ad4e919d3609ebabdac00ba12c3" [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "indexmap" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] [[package]] -name = "unicode-width" -version = "0.1.14" +name = "indicatif" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "instability" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "ipxact" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/ipe?branch=ry%2Finit#2f18277e5760dda1346162ff9250caebc797d06c" +dependencies = [ + "quick-xml 0.37.5", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" +dependencies = [ + "defmt", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.11.0", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +dependencies = [ + "libc", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lsp-server" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6ada348dbc2703cbe7637b2dda05cff84d3da2819c24abcb305dd613e0ba2e" +dependencies = [ + "crossbeam-channel", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "lsp-types" +version = "0.94.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66bfd44a06ae10647fe3f8214762e9369fd4248df1350924b4ef9e770a85ea1" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "netrc" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a91b326434fca226707ed8ec1fd22d4e1c96801abdf10c412afdc7d97116e0" + +[[package]] +name = "newtype_derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8cd24d9f185bb7223958d8c1ff7a961b74b1953fd05dba7cc568a63b3861ec" +dependencies = [ + "rustc_version 0.1.7", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.11.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "nucleo-matcher" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf33f538733d1a5a3494b836ba913207f14d9d4a1d3cd67030c5061bdd2cac85" +dependencies = [ + "memchr", + "unicode-segmentation", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-graphics", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openapiv3" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8d427828b22ae1fff2833a03d8486c2c881367f1c336349f307f321e7f4d05" +dependencies = [ + "indexmap", + "serde", + "serde_json", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + +[[package]] +name = "oxide" +version = "0.17.0+2026060800.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daa73fb338a8d759104a36c662130da272eff87874b6f0b33720dde93f5ef39f" +dependencies = [ + "base64", + "chrono", + "dirs 6.0.0", + "futures", + "progenitor-client", + "rand 0.9.5", + "regress", + "reqwest 0.13.4", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.18", + "toml 1.1.4+spec-1.1.0", + "toml_edit 0.25.13+spec-1.1.0", + "tracing", + "uuid", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct", + "ecdsa", + "elliptic-curve", + "primeorder", + "rand_core 0.6.4", + "sha2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pathfinder_geometry" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" +dependencies = [ + "log", + "pathfinder_simd", +] + +[[package]] +name = "pathfinder_simd" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" +dependencies = [ + "rustc_version 0.4.1", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pinned_vec" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "268ad82d92622fb0a049ff14b01089b0f1bcd5c507fab44724394d328417348a" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "chrono", + "font-kit", + "image 0.24.9", + "lazy_static", + "num-traits", + "pathfinder_geometry", + "plotters-backend", + "plotters-bitmap", + "plotters-svg", + "ttf-parser", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-bitmap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" +dependencies = [ + "gif", + "image 0.24.9", + "plotters-backend", +] + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.11.0", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "portable-pty" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a596a2b3d2752d94f51fac2d4a96737b8705dddd311a32b9af47211f08671e" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "downcast-rs", + "filedescriptor", + "lazy_static", + "libc", + "log", + "nix", + "serial2", + "shared_library", + "shell-words", + "winapi", + "winreg", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "progenitor" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8ba1d77160e6d5c95bdf0792527f76bf528791093fa83015bc2908a0ba9d076" +dependencies = [ + "progenitor-client", + "progenitor-impl", + "progenitor-macro", +] + +[[package]] +name = "progenitor-client" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8a874cf25a33cac7a01b9c1de87bcfbc8aea93f3156d09dcc3bee516a78926" +dependencies = [ + "bytes", + "futures-core", + "percent-encoding", + "reqwest 0.13.4", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "progenitor-impl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e349eed84b9a1a6a5dbe478d335e3df73d32a93c5eefe571c9b8cb298aab5d" +dependencies = [ + "heck", + "http", + "indexmap", + "openapiv3", + "proc-macro2", + "quote", + "regex", + "schemars", + "serde", + "serde_json", + "syn 2.0.117", + "thiserror 2.0.18", + "typify", + "unicode-ident", +] + +[[package]] +name = "progenitor-macro" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa969a1349979c5f64347f204e794781a86d738206d75672e6c9493f5910002" +dependencies = [ + "openapiv3", + "proc-macro2", + "progenitor-impl", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "serde_yaml", + "syn 2.0.117", +] + +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases 0.2.2", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases 0.2.2", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.11.0", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + +[[package]] +name = "redb" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e925444704b5f17d32bf42f5b6e2df050bceebc3dcd6e71cc73dafe8092e839" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "regress" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "158a764437582235e3501f683b93a0a6f8d825d04a789dbe5ed30b8799b8908a" +dependencies = [ + "hashbrown 0.16.1", + "memchr", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeedb13abdaa7e48d391de05b0569b37fa0a7a64a668dff6ffb2141ad0c2527e" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64", + "bytes", + "cfg-if", + "futures-util", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "percent-encoding", + "quick-xml 0.38.4", + "reqwest 0.12.28", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sysinfo", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-stream", + "url", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" +dependencies = [ + "semver 0.1.20", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver 1.0.28", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.117", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_tokenstream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 2.0.117", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "serial2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb6ea5562eeaed6936b8b54e086aa0f88b9e5b1bef45beb038e2519fa1185b1" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared_library" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio 1.2.2", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version 0.4.1", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +dependencies = [ + "bstr", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slog" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1" +dependencies = [ + "anyhow", + "erased-serde", + "rustversion", + "serde_core", +] + +[[package]] +name = "slog-async" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-bunyan" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaaf6e68789d3f0411f1e72bc443214ef252a1038b6e344836e50442541f190" +dependencies = [ + "hostname 0.3.1", + "slog", + "slog-json", + "time", +] + +[[package]] +name = "slog-error-chain" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/slog-error-chain?branch=main#15f69041f45774602108e47fb25e705dc23acfb2" +dependencies = [ + "slog", +] + +[[package]] +name = "slog-json" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" +dependencies = [ + "serde", + "serde_json", + "slog", + "time", +] + +[[package]] +name = "slog-term" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cb1fc680b38eed6fad4c02b3871c09d2c81db8c96aa4e9c0a34904c830f09b5" +dependencies = [ + "chrono", + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "ssh-cipher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" +dependencies = [ + "cipher", + "ssh-encoding", +] + +[[package]] +name = "ssh-encoding" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" +dependencies = [ + "base64ct", + "pem-rfc7468", + "sha2", +] + +[[package]] +name = "ssh-key" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" +dependencies = [ + "ed25519-dalek", + "p256", + "p384", + "p521", + "rand_core 0.6.4", + "rsa", + "sec1", + "sha2", + "signature", + "ssh-cipher", + "ssh-encoding", + "subtle", + "zeroize", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "subst" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" +dependencies = [ + "memchr", + "unicode-width 0.1.14", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "term" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "terminal_size", + "unicode-linebreak", + "unicode-width 0.2.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio 1.2.2", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.11.0", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower 0.5.3", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-lsp" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ba052b54a6627628d9b3c34c176e7eda8359b7da9acd497b9f20998d118508" +dependencies = [ + "async-trait", + "auto_impl", + "bytes", + "dashmap", + "futures", + "httparse", + "lsp-types", + "memchr", + "serde", + "serde_json", + "tokio", + "tokio-util", + "tower 0.4.13", + "tower-lsp-macros", + "tracing", +] + +[[package]] +name = "tower-lsp-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + +[[package]] +name = "tui-textarea" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5318dd619ed73c52a9417ad19046724effc1287fb75cdcc4eca1d6ac1acbae" +dependencies = [ + "crossterm", + "ratatui", + "unicode-width 0.2.0", +] + +[[package]] +name = "tungstenite" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.7", + "sha1 0.10.7", + "thiserror 1.0.69", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typify" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0b89f47309feaeb23c4509c15c9a04234f7deccef6f96c3bfe95319819a304" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7b026f540b148b81043c720889dbb942b08659aa8a43f624ac4f04dbfc1861" +dependencies = [ + "heck", + "log", + "proc-macro2", + "quote", + "regress", + "schemars", + "semver 1.0.28", + "serde", + "serde_json", + "syn 2.0.117", + "thiserror 2.0.18", + "unicode-ident", +] + +[[package]] +name = "typify-macro" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ed96c57f06ae0839416b986921a98f18b220da63bbb243a8570a00c8492183" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "semver 1.0.28", + "serde", + "serde_json", + "serde_tokenstream", + "syn 2.0.117", + "typify-impl", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" @@ -1668,6 +5638,18 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1678,8 +5660,15 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -1692,6 +5681,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1699,10 +5706,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "vhdl_lang" -version = "0.86.0" +name = "version_check" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f3ee86c0a8ca087e0e4eece220e516ce31a00b23de10aa8102b7f98f928977" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vhdl_lang" +version = "0.87.1" +source = "git+https://github.com/oxidecomputer/rust_hdl?branch=vw%2Fembed-api#941eae89ffcb09a8bdbfbd96eb59aa9827ac0dd2" dependencies = [ "clap", "dirs 6.0.0", @@ -1710,62 +5722,445 @@ dependencies = [ "enum-map", "fnv", "glob", - "itertools", - "pad", + "itertools 0.14.0", "parking_lot", "pinned_vec", "rayon", - "strum", + "strum 0.28.0", "subst", - "toml", + "toml 0.8.23", "vhdl_lang_macros", ] [[package]] -name = "vhdl_lang_macros" -version = "0.86.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712baaaca92e0ca66b7a924165c5a17146c91b0b42a675517ac6c364e5969132" +name = "vhdl_lang_macros" +version = "0.87.1" +source = "git+https://github.com/oxidecomputer/rust_hdl?branch=vw%2Fembed-api#941eae89ffcb09a8bdbfbd96eb59aa9827ac0dd2" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "vhdl_ls" +version = "0.87.1" +source = "git+https://github.com/oxidecomputer/rust_hdl?branch=vw%2Fembed-api#941eae89ffcb09a8bdbfbd96eb59aa9827ac0dd2" +dependencies = [ + "clap", + "env_logger", + "fnv", + "fuzzy-matcher", + "log", + "lsp-server", + "lsp-types", + "serde", + "serde_json", + "vhdl_lang", +] + +[[package]] +name = "vw" +version = "0.1.0" +dependencies = [ + "camino", + "chrono", + "clap", + "colored", + "cpp_demangle", + "crossterm", + "dirs 5.0.1", + "futures", + "gethostname", + "indicatif", + "notify", + "petgraph", + "ratatui", + "reqwest 0.13.4", + "rustc-demangle", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "tracing-subscriber", + "vw-analyzer", + "vw-api-client", + "vw-api-types-versions", + "vw-bench", + "vw-eda", + "vw-htcl", + "vw-htcl-cmd", + "vw-ip", + "vw-lib", + "vw-remote", + "vw-repl", + "vw-sync", + "vw-vivado", +] + +[[package]] +name = "vw-agent" +version = "0.1.0" +dependencies = [ + "blake3", + "camino", + "clap", + "dropshot", + "gethostname", + "rand 0.8.7", + "reqwest 0.13.4", + "rust-s3", + "serde", + "serde_json", + "slog", + "slog-async", + "slog-bunyan", + "slog-error-chain", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "vw-api-types-versions", + "vw-bench", + "vw-lib", + "vw-remote", + "vw-sync", + "vw-sync-api", +] + +[[package]] +name = "vw-analyzer" +version = "0.1.0" +dependencies = [ + "async-trait", + "camino", + "serde", + "serde_json", + "tempfile", + "tokio", + "tower-lsp", + "tracing", + "tracing-subscriber", + "vhdl_lang", + "vhdl_ls", + "vw-htcl", + "vw-lib", +] + +[[package]] +name = "vw-api" +version = "0.1.0" +dependencies = [ + "dropshot", + "dropshot-api-manager", + "dropshot-api-manager-types", + "schemars", + "serde", + "uuid", + "vw-api-types-versions", +] + +[[package]] +name = "vw-api-client" +version = "0.1.0" +dependencies = [ + "base64", + "progenitor", + "progenitor-client", + "rand 0.8.7", + "reqwest 0.13.4", + "serde", + "thiserror 1.0.69", + "uuid", + "vw-api-types-versions", +] + +[[package]] +name = "vw-api-types" +version = "0.1.0" +dependencies = [ + "schemars", + "serde", + "vw-api-types-versions", +] + +[[package]] +name = "vw-api-types-versions" +version = "0.1.0" +dependencies = [ + "clap", + "iddqd", + "oxide", + "schemars", + "serde", + "uuid", +] + +[[package]] +name = "vw-bench" +version = "0.1.0" +dependencies = [ + "camino", + "serde", + "thiserror 1.0.69", + "tokio", + "vw-lib", +] + +[[package]] +name = "vw-core" +version = "0.1.0" +dependencies = [ + "dirs 5.0.1", + "petgraph", + "regex", + "serde", + "serde_json", + "tokio", + "toml 0.8.23", + "vhdl_lang", +] + +[[package]] +name = "vw-eda" +version = "0.1.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "vw-htcl" +version = "0.1.0" +dependencies = [ + "camino", + "serde", + "tempfile", + "thiserror 1.0.69", + "vw-quote", + "winnow 0.6.26", +] + +[[package]] +name = "vw-htcl-cmd" +version = "0.1.0" +dependencies = [ + "serde", + "tempfile", + "thiserror 1.0.69", + "toml 0.8.23", + "vw-htcl", + "winnow 0.6.26", +] + +[[package]] +name = "vw-ip" +version = "0.1.0" +dependencies = [ + "ipxact", + "quick-xml 0.37.5", + "regex", + "serde", + "tempfile", + "thiserror 1.0.69", + "toml 0.8.23", + "vw-htcl", + "vw-quote", +] + +[[package]] +name = "vw-lib" +version = "0.1.0" +dependencies = [ + "anodizer", + "camino", + "dirs 5.0.1", + "git2", + "glob", + "netrc", + "petgraph", + "plotters", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "serde", + "serde_json", + "syn 2.0.117", + "tempfile", + "thiserror 1.0.69", + "tokio", + "toml 0.8.23", + "url", + "vhdl_lang", + "vw-core", +] + +[[package]] +name = "vw-openapi-manager" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "clap", + "dropshot-api-manager", + "dropshot-api-manager-types", + "vw-api", + "vw-sync-api", +] + +[[package]] +name = "vw-quote" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "vw-htcl", +] + +[[package]] +name = "vw-remote" +version = "0.1.0" dependencies = [ - "quote", - "syn", + "async-trait", + "camino", + "futures", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "toml 0.8.23", + "tracing", + "vw-bench", + "vw-eda", + "vw-lib", + "vw-vivado", ] [[package]] -name = "vw" +name = "vw-repl" +version = "0.1.0" +dependencies = [ + "arboard", + "base64", + "camino", + "crossterm", + "dirs 5.0.1", + "futures", + "libc", + "nucleo-matcher", + "ratatui", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "toml 0.8.23", + "tracing", + "tui-textarea", + "vw-eda", + "vw-htcl", + "vw-lib", + "vw-vivado", + "winnow 0.6.26", +] + +[[package]] +name = "vw-svc" version = "0.1.0" dependencies = [ + "blake3", + "bytes", "camino", "clap", - "colored", + "daft", + "dropshot", + "futures", + "http-body", + "http-body-util", + "iddqd", + "oxide", + "redb", + "reqwest 0.13.4", + "rust-s3", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "slog", + "slog-async", + "slog-bunyan", + "slog-error-chain", + "ssh-key", + "tempfile", + "thiserror 1.0.69", "tokio", + "tokio-stream", + "tokio-tungstenite", + "uuid", + "vw-api", + "vw-api-client", + "vw-api-types-versions", "vw-lib", ] [[package]] -name = "vw-lib" +name = "vw-sync" version = "0.1.0" dependencies = [ + "blake3", "camino", - "dirs 5.0.1", - "git2", - "glob", - "netrc", - "petgraph", - "plotters", - "prettyplease", - "proc-macro2", - "quote", - "regex", + "ignore", + "tempfile", + "thiserror 1.0.69", + "vw-api-types-versions", +] + +[[package]] +name = "vw-sync-api" +version = "0.1.0" +dependencies = [ + "dropshot", + "dropshot-api-manager-types", + "schemars", + "serde", + "vw-api-types-versions", +] + +[[package]] +name = "vw-vivado" +version = "0.1.0" +dependencies = [ + "async-trait", + "camino", + "colored", + "libc", + "portable-pty", "serde", "serde_json", - "syn", + "similar", "tempfile", "thiserror 1.0.69", "tokio", - "toml", - "url", - "vhdl_lang", + "tracing", + "vw-eda", + "vw-htcl", + "vw-lib", +] + +[[package]] +name = "waitgroup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" +dependencies = [ + "atomic-waker", ] [[package]] @@ -1778,6 +6173,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1815,6 +6219,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.117" @@ -1834,7 +6248,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1869,6 +6283,32 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -1878,7 +6318,7 @@ dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", - "semver", + "semver 1.0.28", ] [[package]] @@ -1891,6 +6331,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" @@ -1928,6 +6387,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1936,9 +6430,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", ] [[package]] @@ -1949,7 +6454,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1960,22 +6465,67 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -1984,7 +6534,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1996,6 +6546,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -2011,7 +6570,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2045,6 +6604,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -2135,6 +6703,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.15" @@ -2144,6 +6721,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wio" version = "0.2.2" @@ -2183,7 +6778,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -2199,7 +6794,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -2233,7 +6828,7 @@ dependencies = [ "id-arena", "indexmap", "log", - "semver", + "semver 1.0.28", "serde", "serde_derive", "serde_json", @@ -2247,6 +6842,62 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "camino", + "clap", + "rcgen", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "yeslogic-fontconfig-sys" version = "6.0.0" @@ -2277,10 +6928,30 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zerofrom" version = "0.1.6" @@ -2298,10 +6969,16 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.3" @@ -2332,7 +7009,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2340,3 +7017,18 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index fcfd4f0..9285659 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,46 @@ [workspace] resolver = "2" -members = ["vw-lib", "vw-cli"] +members = [ + "vw-core", + "vw-lib", + "vw-cli", + "vw-htcl", + "vw-eda", + "vw-vivado", + "vw-analyzer", + "vw-quote", + "vw-ip", + "vw-htcl-cmd", + "vw-repl", + "vw-remote", + "vw-bench", + "vw-svc", + "vw-agent", + "vw-sync", + "vw-sync-api", + "vw-api", + "vw-api-types", + "vw-api-client", + "vw-openapi-manager", + "xtask", + "vw-api-types/versions", +] [workspace.package] version = "0.1.0" edition = "2021" license = "MPL-2.0" -repository = "https://github.com/your-username/vw" +repository = "https://github.com/oxidecomputer/vw" [workspace.dependencies] # Shared dependencies +dropshot = "0.17.1" +# Matched to dropshot's own features exactly. A second crypto provider +# enabled here would make rustls' process default ambiguous and every +# ServerConfig::builder() call panic, including dropshot's. +rustls-pemfile = "2.2" +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "logging", "std", "tls12"] } +tokio-tungstenite = "0.21" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "0.8" @@ -24,3 +55,34 @@ url = "2.5" glob = "0.3" petgraph = "0.8.3" plotters = "0.3" +winnow = "0.6" +async-trait = "0.1" +tower-lsp = "0.20" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +futures = "0.3" +portable-pty = "0.9" +ratatui = { version = "0.29", features = ["crossterm"] } +crossterm = { version = "0.28", features = ["event-stream"] } +tui-textarea = { version = "0.7", default-features = false, features = ["crossterm", "ratatui"] } +nucleo-matcher = "0.3" +indicatif = "0.17" +similar = { version = "2.6", features = ["inline"] } +ipxact = { git = "https://github.com/oxidecomputer/ipe", branch = "ry/init" } +anodizer = { git = "https://github.com/oxidecomputer/anodizer", branch = "ry/vw-restructure" } +schemars = "0.8.22" +uuid = { version = "1.24.0", features = ["serde", "js", "v4"] } +oxide = "0.17.0" +clap = { version = "4.0", features = ["derive", "env"] } +iddqd = { version = "0.4.6", features = ["daft"] } + +# Embedded VHDL LSP. Path deps while our +# `new_with_config` / `set_config` patches are in flight against +# rust_hdl/master; drop to crates.io once merged. +vhdl_lang = { git = "https://github.com/oxidecomputer/rust_hdl", branch = "vw/embed-api" } +vhdl_ls = { git = "https://github.com/oxidecomputer/rust_hdl", branch = "vw/embed-api" } +#vhdl_lang = { path = "/home/ry/src/rust_hdl/vhdl_lang" } +#vhdl_ls = { path = "/home/ry/src/rust_hdl/vhdl_ls" } + +[patch."https://github.com/oxidecomputer/vw/"] +vw-core = { path = "vw-core" } diff --git a/constraint_evaluator.fst b/constraint_evaluator.fst new file mode 100644 index 0000000..ed86b7f Binary files /dev/null and b/constraint_evaluator.fst differ diff --git a/docs/authoring-htcl-libraries.md b/docs/authoring-htcl-libraries.md new file mode 100644 index 0000000..3514484 --- /dev/null +++ b/docs/authoring-htcl-libraries.md @@ -0,0 +1,704 @@ +# Authoring htcl libraries + +This document is a reference for engineers writing **htcl modules +that wrap underlying EDA IP, commands, or workflows**. The intended +output of such a module is one or more `proc` declarations that +downstream code can call from a workflow script to configure +hardware, drive a build, or otherwise script an EDA tool. + +The document covers: + +1. What htcl is and what it is for. +2. The full surface of the language — syntax, semantics, attributes. +3. Where htcl behaves the same as Tcl and where it differs. +4. How to validate an htcl program with `vw`. + +## 1. What htcl is + +htcl — "**h**ardware Tcl" — is a small structured dialect of Tcl +for HDL workflow scripting. It is the language `vw` uses to drive +EDA backends (today, Vivado over a pipe) and to give engineers a +source-controlled, reviewable, tool-checkable surface for everything +that would otherwise live as ad-hoc Tcl: IP configuration, block +design construction, project setup, simulation harnessing, and +custom commands that wrap a vendor tool. + +htcl is **not** specific to any single source of those wrappers. An +htcl library may be: + +- **Hand-written** by an engineer, wrapping a Vivado/Quartus + built-in or an in-house Tcl helper with a typed, doc-commented + interface. +- **Generated** from a vendor IP-XACT description (e.g. `vw ip + generate` emits an htcl wrapper for a Xilinx IP). The result is + ordinary htcl — there is nothing IP-XACT-shaped about it once + generated. +- **Generated from anything else** that has no IP-XACT — a custom + IP repo, a curated set of Tcl recipes, a board-bring-up script. + +What unifies them is the structured `proc` surface: every wrapper +is a proc with documented keyword arguments, default values, and +constraints the analyzer can check. + +At analysis time, the syntax tree drives the LSP (`vw analyzer`) — +completion, hover, signature help, error reporting. At run time, +htcl is lowered to plain Tcl and shipped to the backend. + +### Module shape + +An htcl library is one or more `.htcl` files. Most libraries place +their entry point at a conventional path (`src/.htcl`) and +may `src`-import additional files. A `proc` declared in any +imported file becomes callable in the consumer's scope. Proc names +are flat unless wrapped in a `namespace eval` block, which groups +related helpers under a `::` prefix — see §2.10 below. + +## 2. The language + +### 2.1 File overview + +An htcl file is a sequence of statements separated by newlines or +semicolons. Each statement is one of: + +| Statement | Purpose | +|---|---| +| Comment `# ...` | Free-form comment; ignored. | +| Doc comment `## ...` | Attached to the next `proc` or proc-arg; surfaces in hover. | +| Command `name word word ...` | A call to any command (Tcl builtin, EDA builtin, or htcl proc). | +| `set ` | Variable assignment (same as Tcl). | +| `proc { args } { body }` | Structured proc declaration (the main authoring construct). | +| `src ` | Import another htcl module (htcl-specific; no Tcl analogue). | + +Whitespace is significant only as a word separator. Indentation is +free-form. + +### 2.2 Word forms + +Every command word can be written in one of three forms — the same +three Tcl supports: + +| Form | Example | Semantics | +|---|---|---| +| Bare | `foo` | A literal word. May contain `$var` and `[cmd]`. | +| Quoted | `"hello $world"` | Variable and command substitution still happen; whitespace is preserved. | +| Braced | `{a b c}` | Literal text; no substitution. | + +Inside `[ … ]` command substitution, newlines are treated as +whitespace and do not need backslash continuations. **This is the +canonical form for call sites that don't fit on one line** — wrap +the call in brackets, bind the result with `set`, and let each +keyword argument live on its own line without backslash noise: + +```htcl +set cpm5_pcie1 [ + create_cpm5_cpm_pcie1 + -cell cpm5 + -max_link_speed 32.0_GT/s + -modes PCIE +] +``` + +A bare word ending in `\` does continue onto the next line (the +classic Tcl form), but bracket-bound `set` is the preferred style. + +### 2.3 Comments and doc comments + +htcl has two distinct comment forms with different purposes: + +- **Regular comments — `# ...`.** Free-form. Use these to record + rationale at call sites, to label sections of a workflow script, + to leave notes — anything that's "for the reader." The analyzer + ignores them. +- **Doc comments — `## ...`.** Semantically significant + documentation that attaches to the next `proc` declaration or + the next proc-arg. Tooling — the LSP for hover and signature + help, and documentation generators — consumes them. Use them + only inside a library, on definitions; they serve no purpose at + call sites. + +```htcl +## Configure an AXIS register slice. ;# doc-comment on the proc +proc create_axis_register_slice { + ## Block-design cell name. ;# doc-comment on this arg + cell_name + ... +} + +# Internal streaming bus between DMA and classifier. ;# call-site +# 128-bit because the classifier hits 100 Gb/s. ;# rationale — +set dma_to_classifier [ ;# use #, not ## + create_axis_register_slice + -cell_name dma_to_classifier + -tdata_width 128 +] +``` + +Multiple `##` lines stack into one block of doc-comment text on +the item they precede. + +### 2.4 The `proc` declaration + +This is the central authoring construct. The args list inside the +first `{ … }` is *structured*: each arg is a single identifier +preceded by optional doc comments and optional `@attribute(...)` +annotations. The args grammar is: + +``` +args := arg_item* +arg_item := doc_comment* attribute* IDENT +attribute := '@' IDENT ( '(' value ( ',' value )* ')' )? +value := integer | string | ident +``` + +Example: + +```htcl +## Configure a Versal CIPS instance. +## +## Sets the requested CONFIG.* properties on the supplied block-design cell. +proc create_versal_cips { + ## Block-design cell handle to set the property on. + cell + + ## Boot the secondary PCIe controller as well as the primary. + @enum(0, 1) @default(0) boot_secondary_pcie_enable + + ## Inner dict for the PMC subsystem. + @default("") ps_pmc_config +} { + set_property -dict [list \ + CONFIG.BOOT_SECONDARY_PCIE_ENABLE $boot_secondary_pcie_enable \ + CONFIG.PS_PMC_CONFIG $ps_pmc_config \ + ] $cell +} +``` + +Notes: + +- **htcl is keyword-only.** Every proc you declare accepts its + arguments as `-flag value` pairs at the call site. There is no + positional call syntax — even an arg with no `@default` (an + implicitly-required arg) must be passed as `-arg value`. The + args list above is the **declaration order**, used by the + validator for documentation and stable diagnostics, not by Tcl + for dispatch. +- Args with no `@default` are **required**. Omitting them at a + call site is a compile-time error from the validator. +- The proc body is plain Tcl text. The body refers to each arg by + its bare name (`$cell`, `$boot_secondary_pcie_enable`, …) — the + same way it would for a standard Tcl proc with named parameters. + The lowerer wires up these locals from the caller's `-flag + value` pairs at runtime via a generated `::vw::kwargs` prelude. +- Because the keyword parse happens at runtime (inside the + wrapper), call sites work uniformly at **any** nesting: at the + top level of a file, inside another proc's body, inside a + `namespace eval`, inside a `[ ... ]` command substitution, or + through an `eval`/`uplevel`. The lowerer doesn't need to see + the call site to translate it. +- `proc` itself may be declared at any depth, but only **top-level** + proc declarations are visible to the call-site validator. Procs + defined inside another proc's body ship as raw text and miss + the kwargs-prelude treatment — avoid nested proc declarations + in htcl, or write them in raw Tcl form (`proc inner args { ... + }`). + +### 2.5 Argument attributes + +Attributes go on the line(s) before the arg's identifier. They are +parsed positionally and may stack. Values are written with strings +quoted, integers bare, identifiers bare. + +| Attribute | Meaning | Example | +|---|---|---| +| `@default()` | Default value used when caller omits this arg. Presence makes the arg optional. | `@default(0) boot_secondary_pcie_enable` | +| `@enum(, , ...)` | Caller's value must match one of the listed literals. Validated when the value is a literal. | `@enum(0, 1) enable` | +| `@range(, )` | Caller's integer value must satisfy `lo <= n <= hi`. | `@range(1, 16) num_lanes` | +| `@requires()` | This arg, if set, requires `-` to also be set. | `@requires(has_tuser) tuser_width` | +| `@conflicts()` | This arg cannot coexist with `-`. | `@conflicts(slave_mode) master_mode` | +| `@deprecated[(msg)]` | Warning at call sites; optional human message. | `@deprecated("use -mode instead") legacy_mode` | + +`@enum` and `@range` only check **literal** call-site values. A +value that is itself an interpolation (`$var`, `[cmd]`) is not +statically checkable and silently passes — the runtime sees +whatever the interpolation produces. + +### 2.5.0a Argument types + +An argument may carry a type annotation in a `: TYPE` suffix on +the arg name: + +``` +proc plumb_pin { + ## What to name the external port. + name: string + + ## Identity of the pin to make external. + pin: bd_pin +} unit { + … +} +``` + +The annotation uses the same type vocabulary as the return-type +slot (§2.5.1) — primitives (`string`, `int`, `bool`, `unit`), +newtypes (`bd_cell`, `bd_pin`, …), and generics (`list`, +`dict`, with arbitrary nesting). Compatible with the +existing attribute grammar (`@default(0) count: int`). + +Annotations are optional — untyped args still parse. The +analyzer shows annotated args as `-name: TYPE` in hover and +signature help; the validator uses them to shape-check newtype +`::repr` / `from` / `to` triplets. + +See [htcl-return-types.md](htcl-return-types.md) for the full +type vocabulary, newtype declaration syntax, and worked +examples. For values that can take one of several shapes +(e.g. heterogeneous EDA return values), see +[htcl-enums.md](htcl-enums.md) for tagged sum types with +auto-generated constructors, repr, and overload dispatch. + +### 2.5.1 Return types + +A proc may carry a return-type annotation in a 4th-word slot +between the args block and the body: + +``` +proc make_widget { @arg(name) ... } widget { + …body… +} +``` + +The annotation drives the REPL printer (the result is formatted +through the type's `repr` proc) and the analyzer's hover / +signature-help (`proc NAME → TYPE`). Procs without an annotation +parse and behave identically to today — adoption is gradual. + +Available shapes: + +- Primitives: `string`, `int`, `bool`, `unit`. Built into the + compiler; no declaration needed. +- Generics: `list`, `dict`, with arbitrary nesting. +- User newtypes: any identifier introduced via `type NAME = + UNDERLYING`, accompanied by `::repr` / `from` / `to`. + +`unit` is the type for side-effecting procs that don't return a +meaningful value (logging, configuring, connecting). The REPL +suppresses the empty Result entry on `unit`-typed expressions. + +See [htcl-return-types.md](htcl-return-types.md) for the full +type vocabulary, newtype declaration syntax, and worked examples. + +### 2.6 Call sites + +The canonical call-site shape is `set [ ]` — +bind the call's return value to a name, and let the brackets handle +multi-line wrapping. Each keyword argument goes on its own line, no +backslash continuations needed: + +```htcl +set cips [ + create_versal_cips + -name cips + -cpm_config cpm5 +] + +# 250 MHz / 195 MHz aren't in the preset list but the clock +# generator will synthesize them — chosen for the eth core. +set ps_pmc_config [ + create_versal_cips_ps_pmc_config + -cell cips + -clock_mode Custom + -design_mode 1 + -pcie_apertures_dual_enable 0 + -pcie_apertures_single_enable 0 + -pmc_crp_pl0_ref_ctrl_freqmhz 250 + -pmc_crp_pl1_ref_ctrl_freqmhz 195 + -ps_board_interface Custom + -ps_pcie1_peripheral_enable 0 + -ps_pcie2_peripheral_enable 1 + -ps_pcie_reset {ENABLE 1} + -ps_use_pmcpl_clk0 1 + -ps_use_pmcpl_clk1 1 + -ps_use_pmcpl_iro_clk 0 + -smon_alarms Set_Alarms_On + -smon_enable_temp_averaging 0 + -smon_temp_averaging_samples 0 +] +``` + +A one-line call works the same way — just don't break across lines: + +```htcl +set cpm5 [create_cpm5 -name cpm5] +``` + +Rules the validator enforces on every call to a known proc: + +- Each `-flag` must be one of the declared args. +- Each `-flag` is given exactly one value (the next word). +- Each `-flag` appears at most once (a duplicate is a warning). +- Every required arg (no `@default`) must be present. +- `@requires` / `@conflicts` relationships are checked across + present args. +- Literal values are checked against `@enum` / `@range`. +- `@deprecated` flags produce warnings. + +Calls to commands that aren't declared `proc`s in the loaded +documents are **not** validated — they're assumed to be EDA or +Tcl builtins and are passed through verbatim. + +### 2.7 Variables and substitution + +Variables work as in Tcl: + +```htcl +set ref_clk 100 +puts "ref clock is $ref_clk MHz" + +# Braces suppress substitution. +puts {$ref_clk is literal here} +``` + +`$name` references resolve against the nearest enclosing scope — +local `set`s first, then the enclosing proc's parameter list. +There is no static type checking on variable values. + +### 2.8 Imports — `src` + +```htcl +src @amd-htcl/cpm5 ;# named workspace dependency +src @amd-htcl/cips +src "lib/utils.htcl" ;# relative to the importing file +src "/abs/path/to/file.htcl" ;# absolute filesystem path +``` + +The `src` statement loads and inlines another htcl module. Path +forms: + +- `@/` — resolved via `vw.toml`'s workspace + dependencies (the same dependency resolver `vw` uses for VHDL + deps). +- Anything starting with `/` — filesystem-absolute. +- Anything else — relative to the directory of the importing file. + +The path word must be a literal (bare or quoted text with no +`$var` or `[cmd]` parts). The loader is idempotent on canonical +paths: a file imported twice loads once. + +By the time an htcl program reaches the backend, all `src` imports +have been flattened into a single Tcl stream. + +### 2.9 A complete library example + +Hand-written wrapper around an IP, with no IP-XACT involved: + +```htcl +## A minimal AXIS interface configurator. +## +## Wraps the underlying create_bd_cell and set_property calls so that +## a consumer can request a configured AXIS slice with a few keyword +## arguments. +proc create_axis_register_slice { + ## Block-design cell name to instantiate at. + cell_name + + ## Width of the data bus in bits. + @enum(8, 16, 32, 64, 128, 256, 512) @default(64) tdata_width + + ## Include byte-strobe sideband. + @enum(0, 1) @default(0) has_tkeep + + ## Width of the optional user sideband; required when -has_tuser is on. + @range(1, 32) @requires(has_tuser) tuser_width + + ## Set when a user sideband is desired. + @enum(0, 1) @default(0) has_tuser + + ## Newer designs should use -has_tuser instead. + @deprecated("use -has_tuser") legacy_tuser_mode +} { + create_bd_cell -type ip -vlnv xilinx.com:ip:axis_register_slice:1.1 $cell_name + set_property -dict [list \ + CONFIG.TDATA_NUM_BYTES [expr {$tdata_width / 8}] \ + CONFIG.HAS_TKEEP $has_tkeep \ + CONFIG.HAS_TUSER $has_tuser \ + CONFIG.TUSER_WIDTH $tuser_width \ + ] [get_bd_cells $cell_name] +} +``` + +A call site, with rationale captured in plain comments: + +```htcl +src @oxide-ip/axis + +# Internal streaming bus between the DMA and the packet classifier. +# 128-bit because the classifier hits 100 Gb/s line rate; tuser carries +# the classification verdict (5 bits today, room for one more flag). +set dma_to_classifier [ + create_axis_register_slice + -cell_name dma_to_classifier + -tdata_width 128 + -has_tkeep 1 + -has_tuser 1 + -tuser_width 6 +] +``` + +### 2.10 Namespaces — `namespace eval` + +When several procs share a logical prefix (`project::set_*`, +`ip::*`, `log::*`), wrapping them in a `namespace eval` block lets +each member be defined with a short bare name while still being +*called* under the qualified `::` form: + +```htcl +namespace eval project { + ## Set the target HDL language for new sources in a project. + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { + set_property -name TARGET_LANGUAGE -value $language -objects $proj + } + + ## Set the default library new sources land in. + proc set_default_library { + proj + @default(xil_defaultlib) library + } { + set_property -name DEFAULT_LIB -value $library -objects $proj + } +} + +# At a call site: +project::set_target_language -proj $proj -language VHDL +project::set_default_library -proj $proj +``` + +The analyzer treats each inner `proc` exactly as if it had been +written `proc project::set_target_language { ... } { ... }` at the +top level — same `@enum` / `@default` / `@requires` validation, +same hover, same signature help, same completion. The only +difference is source organization. + +Mechanics: + +- The `name` word can be a multi-segment Tcl namespace + (`namespace eval foo::bar { ... }`); the analyzer uses the + entire name as the prefix. +- `namespace eval` blocks nest. An inner `proc baz` inside + `namespace eval outer { namespace eval inner { ... } }` + registers as `outer::inner::baz`. +- A call from *inside* a namespace body to a sibling member must + still use the qualified name (no automatic same-namespace + resolution in v1). Write `project::helper $x`, not bare + `helper $x`. +- Lowering walks namespace bodies recursively, so inner procs get + their attributes stripped and the same `::vw::kwargs` runtime + prelude that top-level procs get. + +## 3. How htcl differs from Tcl + +htcl is a strict superset of the Tcl subset most engineers actually +write — anything you'd type in a Vivado console as a one-off +command parses as htcl. The structural differences come from htcl +adding new constructs and tightening the rules around `proc` +declarations. + +### 3.1 What htcl adds + +| Construct | htcl | Tcl | +|---|---|---| +| Doc comments | `## ...` carry to the next `proc` / proc-arg and feed hover. | Plain `#`; no first-class doc concept. | +| Structured `proc` args | Each arg is a doc-commented, attribute-tagged identifier. | Args are flat names or `{name default}` pairs. | +| Keyword call sites | `create_x -foo a -bar b` | Positional `create_x a b`. | +| Static validation | `@enum`, `@range`, `@requires`, `@conflicts`, etc. checked at parse-time. | None — errors only at runtime. | +| Module imports | `src @dep/file` or `src "rel/path.htcl"`. | `source ./foo.tcl`, with no dependency resolution. | +| Bracket-body line continuation | Newlines inside `[ … ]` are whitespace. | Newline terminates a command unless `\`-escaped. | + +### 3.2 What htcl restricts or interprets differently + +- **`proc` args are structured.** A v1 htcl `proc` cannot declare + its args as `{name default}` pairs or as `args` for varargs the + way pure Tcl can. Every arg is a single bare identifier, + optionally preceded by attributes. Defaults live in + `@default(...)`. +- **Required args come from the absence of `@default`.** Any arg + without `@default` is required. There is no `args` catch-all. +- **Call sites must use keyword form.** When the validator sees a + call to a known proc, positional words other than `-flag value` + pairs are reported as errors. Calls to *unknown* commands + (presumed EDA/Tcl builtins) pass through verbatim with no shape + check. +- **Doc comments are semantically significant.** `##` on a `proc` + or proc-arg is consumed by tooling — the LSP for hover, + documentation generators for output — so removing or relocating + one changes observable behavior. Regular `#` comments behave + exactly like Tcl comments. +- **`src` is parsed structurally.** The path word must be a + literal; `src $name` is rejected because the analyzer needs to + follow imports statically. +- **Top-level only for declarations and validated calls.** The + validator builds its signature table from top-level `proc` + declarations. A proc declared inside another proc's body still + parses, but its signature is not used to check call sites. + Likewise, the lowering pass rewrites top-level call sites to + known procs; calls *inside* a proc body are shipped verbatim. + Write your library entry points at the top level. + +### 3.3 What is unchanged + +Everything else is plain Tcl: + +- `$var`, `[cmd]`, `"..."`, `{...}`, backslash escapes. +- `set`, `expr`, `if`, `foreach`, `puts`, `list`, `dict`, … +- The proc body is just Tcl text. Anything you can do in Tcl + works inside a proc body; htcl makes no attempt to constrain it. + +If in doubt, write Tcl. htcl only diverges in service of the +structured proc surface; the body of every command is shipped +through to the backend as written. + +## 4. Checking an htcl program with `vw` + +`vw check` parses, validates, and reports errors and warnings for +one or more `.htcl` files without executing anything. It uses the +same analyzer pipeline the LSP uses, so a clean `vw check` means +the LSP will also be quiet. + +### 4.1 Basic invocation + +```bash +vw check src/cips.htcl +vw check src/lib/*.htcl +``` + +Output on a clean file: + +``` + Checking cips +``` + +Output with errors: + +``` +error: src/cips.htcl:42:23: value 3 for -boot_secondary_pcie_enable is not in @enum. Possible values are 0, 1 +error: src/cips.htcl:58:1: missing required argument -cell +src/cips.htcl: 2 error(s), 0 warning(s) +``` + +Each line carries an absolute file path, line, and column, in +`path:line:col: message` format. Spans inside `src`-imported files +are mapped back to their originating file, so an error in an +imported module reports the imported file's path — not the entry +point's. + +### 4.2 What `vw check` enforces + +The validator runs the rules described above: + +- **Parse errors.** Anything that doesn't lex/parse cleanly: + unterminated brace groups, missing values for `-flag` words, + malformed attributes. +- **Proc shape.** Duplicate proc declarations are an error (the + later one wins, matching Tcl's redefine semantics). +- **Call sites against known procs.** + - Unknown `-flag`: `undefined argument -. Possible values + are `. + - Missing value: `argument - is missing a value`. + - Missing required: `missing required argument -`. + - Duplicate flag (warning): `duplicate argument -`. + - `@enum` violation: `value for - is not in @enum. ...`. + - `@range` violation: `value for - is out of @range(...)`. + - `@range` on a non-integer literal: `argument - expects + an integer, found `. + - `@requires` unmet: `argument - requires - to also be + set`. + - `@conflicts` triggered: `argument - conflicts with -`. + - `@deprecated` (warning): `argument - is deprecated[: msg]`. +- **`src` imports.** Unknown dependency, missing file, non-literal + path, and parse errors inside imported files. + +### 4.3 What `vw check` does *not* enforce + +- Variable type, range, or existence inside a proc body — the + body is opaque to the analyzer in v1. +- EDA- or Tcl-builtin call shapes. A call to `set_property` or + `create_bd_cell` passes through unchecked. +- Values that go through `$var` or `[cmd]` substitution. `@enum` + and `@range` only see literal call-site words. +- Module-level public/private. Every top-level proc in every + loaded file is in scope. + +### 4.4 Related `vw` commands + +- `vw run ` — parses, validates, and executes through + the EDA backend. With `--check` it stops after the parse and + reports errors. Useful when you want to confirm a file is + shippable without spinning up the backend. +- `vw analyzer` — the LSP server (stdio). Editors point at it for + completion, hover, signature help, goto, and live error + reporting. The errors are exactly what `vw check` reports. +- `vw ip generate ` — generates an htcl wrapper + from an IP-XACT component. The generated file is itself a fully + valid htcl library; reading one is a fast way to see a real + wrapper's shape. + +### 4.5 Suggested authoring loop + +1. Sketch the proc signature: name the args, attach doc comments, + set `@default` for everything optional, mark `@enum` / + `@range` where the underlying domain is known and exhaustive. +2. Write the body in plain Tcl — `set_property`, `create_bd_cell`, + whatever the backend needs. +3. Run `vw check` to confirm the proc parses cleanly. +4. Add a call site in a separate `.htcl` file and run `vw check` + on that to confirm the validator agrees with the signature. +5. Open the file in an editor with `vw analyzer` configured for + `.htcl` and verify hover and completion behave as expected — + the doc comments you wrote are what consumers will read. +6. Once the surface looks right, run the call site through + `vw run` to see the lowered Tcl actually do something on the + backend. + +## Reference summary + +```text +File := Statement* +Statement := Command | Comment | DocComment | Proc | Src +Command := Word Word* +Word := Bare | Quoted | Braced +Comment := '#' .* NEWLINE +DocComment := '##' .* NEWLINE ; attaches to next proc / proc-arg +Proc := 'proc' Name '{' ArgList '}' '{' Body '}' +ArgList := ArgItem* +ArgItem := DocComment* Attribute* Ident +Attribute := '@' Ident ( '(' Value (',' Value)* ')' )? +Value := Integer | String | Ident +Src := 'src' PathWord +PathWord := '@''/' | '/' | '' +``` + +Canonical call-site shape: + +```text +set [ <-flag value>...] +``` + +Attributes recognized by the validator: + +```text +@default() ; default value, makes arg optional +@enum(, , ...) ; allowed literal values +@range(, ) ; integer range, inclusive +@requires() ; presence implies - present +@conflicts() ; presence forbids - +@deprecated[()] ; warns at call sites +``` + +Errors and warnings surface through: + +- `vw check ` — one-shot CLI. +- `vw run --check` — same checks, no execution. +- `vw analyzer` — same checks, live in the editor. diff --git a/docs/better-stack-trace-coverage.md b/docs/better-stack-trace-coverage.md new file mode 100644 index 0000000..37c85b5 --- /dev/null +++ b/docs/better-stack-trace-coverage.md @@ -0,0 +1,228 @@ +# Better stack-trace coverage via lowered proc-body instrumentation + +## Problem + +Some Vivado warnings and errors arrive with no stack trace because they +bypass every Tcl-level emission path our shim hooks. The canonical case +is `[IP_Flow 19-7090] Invalid parameter '…' provided, Ignoring`, emitted +from inside `set_property`'s C++ property validator. By the time those +bytes reach the worker's PTY reader, Tcl has already returned control to +the C++ caller and the Tcl call stack that produced them is gone — +neither the `puts` override nor the `send_msg_id` override fired. + +Today we handle this by **per-command instrumentation**: a wrap around +`::set_property` in `vw-vivado/shim/vivado-shim.tcl::install_set_property_context` +captures the Tcl call stack via `info frame` just before delegating into +the underlying C++ command, then ferries the captured frames to the +worker through PTY-side `__VW_CTX_*` markers. The worker tags any +Warning/Error chunks arriving while a context is active. + +This works for `set_property`. It doesn't work for any other Vivado +command that emits async warnings the same way — `create_bd_cell`, +`connect_bd_intf_net`, `validate_bd_design`, etc. Each one needs its own +wrap. + +## Proposal + +Replace per-command wraps with **universal coverage** by instrumenting +every lowered htcl proc body to maintain an explicit htcl-coordinate +stack in Tcl globals. When a warning arrives via PTY, the worker reads +the current top of that stack as the context. + +This is the moral equivalent of "what if we just fed every statement +through Rust one at a time" — it gives us full visibility into the +runtime call chain without rewriting Tcl's proc dispatch. The data +lives in Tcl globals because that's where execution actually happens, +but we control what goes in and when, and the data is the same htcl +coordinates we'd track if we were stepping each statement from Rust. + +### What the lowerer emits + +`vw-htcl/src/lower.rs::lower_proc_decl` currently emits: + +```tcl +proc configure_cips {args} { ::vw::kwargs $args {…} +…body statements at their source line numbers… +} +``` + +We'd extend it to bracket each body statement with push/pop calls: + +```tcl +proc configure_cips {args} { ::vw::kwargs $args {…} + ::vw::stack push "ip/cips.htcl:14 in ::configure_cips" + + ::vw::stack swap "ip/cips.htcl:17 in ::configure_cips" + + ::vw::stack swap "ip/cips.htcl:23 in ::configure_cips" + + … + ::vw::stack pop +} +``` + +`push` adds a new frame, `swap` replaces the current top in-place (so +we don't grow the stack one entry per statement), `pop` removes it at +proc exit. + +Top-level (non-proc) statements get the same treatment in +`dispatch_eval`'s shipped script. + +### Shim helpers + +```tcl +namespace eval ::vw::stack { + variable frames {} + proc push {frame} { + variable frames + lappend frames $frame + } + proc swap {frame} { + variable frames + if {[llength $frames] > 0} { + lset frames end $frame + } else { + lappend frames $frame + } + } + proc pop {} { + variable frames + set frames [lrange $frames 0 end-1] + } + proc snapshot {} { + variable frames + return $frames + } +} +``` + +The shim's existing `attach_stack_if_message` (puts override path) keeps +using `info frame` — it works fine. For the PTY-bypass path, we replace +the per-command marker wraps with a single hook that emits the snapshot +whenever Vivado is about to do something async. The cleanest version: +emit the snapshot **on every statement boundary**, so the worker always +has the latest context without needing per-command opt-in. + +Concretely, every `stack swap` call also writes the new frame to the +PTY as a marker: + +```tcl +proc swap {frame} { + variable frames + if {[llength $frames] > 0} { + lset frames end $frame + } else { + lappend frames $frame + } + ::vw::emit_pty_ctx_replace $frames +} +``` + +`emit_pty_ctx_replace` writes one `__VW_CTX_BEGIN__` / frames / +`__VW_CTX_READY__` group, replacing whatever the worker currently has +active. No `__VW_CTX_END__` is sent — the context is always "the most +recent statement we entered." It gets replaced on the next statement +and reset when an eval completes (worker clears on `EvalDone`). + +### Worker + +The worker already handles `__VW_CTX_BEGIN__` / `__VW_CTX_FRAME__:` / +`__VW_CTX_READY__` / `__VW_CTX_END__` markers via +`worker.rs::consume_ctx_marker` and tags warnings/errors via +`emit_pty_chunk`. No changes needed there beyond: + +- Treat absence of `__VW_CTX_END__` as "always active until eval ends." + The worker should clear `active_pty_context` on `EvalDone` to avoid a + context from one user submission contaminating the next. +- Drop `install_set_property_context` from the shim — universal + coverage subsumes it. + +### Rust-side resolver + +`vw-repl/src/app.rs::resolve_stack_frames` already rewrites +`:N in ::procname` to absolute `(file, line)` via the session's +proc table. No changes needed — the marker frames are already in that +shape. + +## Tradeoffs + +### Pros + +- **Universal coverage.** Any Vivado command emitting async warnings + gets a stack trace, not just `set_property`. We stop playing whack- + a-mole every time a new IP throws a different warning class. +- **Removes per-command wraps.** `install_set_property_context` and + any future siblings (`install_validate_bd_design_context`, etc.) + all go away. +- **Statement-precise.** Currently the tagged frame for an + IP_Flow warning points at the `set_property` call site. Under + this scheme it points at the actual `create_versal_cips` call in + the user's `configure_cips` body, because the most-recent `swap` + captured *that* statement before Tcl dispatched into the wrapper + that eventually called `set_property`. Closer to "what line of my + code is responsible." + +### Cons + +- **Codegen overhead.** Every lowered proc body grows by ~one + `::vw::stack swap` call per statement. For a file like + `vivado-cmd/module.htcl` that's significant but not catastrophic; + the strings are short and Tcl's bytecode compiler handles them + cheaply. For `cpm5/module.htcl` (~880 procs, each with ~5–200 + statements) the lowered text grows by a similar factor — measure + before / after on the `--load` cold-start time to know if it + matters. +- **PTY marker volume.** Each `swap` writes a marker group + (~3 lines) to the PTY. For an eval that fires 1000 statements, + that's 3000 marker lines to filter out on the worker side. + Cheap per line but adds up. Mitigate by only emitting markers when + *about to* call into a typed external — but that gets us back to + per-command opt-in. +- **Coupling to Tcl's eval order.** The htcl-coordinate stack only + stays accurate if every statement reaches its `swap` call. If a + Tcl `error`/`break`/`continue` jumps out of a body mid-statement, + the `pop` at proc exit cleans up, but a partially-walked body + could leave a stale frame as "current" until the next `swap` or + `EvalDone` clears it. In practice this only affects the window + between the error and the next event — same scope as the current + per-command wrap. +- **Visible inside `if` / `foreach` bodies?** Control-flow constructs + in htcl are braced Tcl scripts that Tcl evaluates internally — + the lowerer doesn't walk into braced sub-scripts today, so an + `if { … } { call X }` would only emit `swap` for the outer `if`, + not for the inner `call X`. Whether that resolution matters + depends on how often the user wants line-precise info for code + inside an `if`. Could be added later by lowering braced bodies as + scripts too. + +## When to do it + +Open question. The current per-command wrap covers `set_property`, +which empirically catches ~all the IP_Flow validation warnings the +user has hit so far. Universal coverage becomes worth the codegen +overhead when: + +- A second Vivado command starts emitting async warnings we want + traced (an obvious sign: someone adds an + `install__context` proc and we realize it's the third one). +- The `set_property` wrap starts missing cases (e.g. Vivado adds a + property-setter path that bypasses `::set_property`). +- We want statement-precise warning attribution rather than + "warning happened during a `set_property` call in proc X" — i.e. + the difference between `at ip/cips.htcl:69` (the `create_versal_cips` + call) vs `at vivado-cmd/cmd/set_property.htcl:80` (inside the + wrapper that eventually invoked `set_property`). + +Until one of those bites, the per-command wrap is the cheaper bet. +This file is the breadcrumb for when it doesn't. + +## References + +- `vw-vivado/shim/vivado-shim.tcl::install_set_property_context` — the + current per-command implementation +- `vw-vivado/src/worker.rs::consume_ctx_marker`, + `vw-vivado/src/worker.rs::emit_pty_chunk` — the marker-consumer side +- `vw-htcl/src/lower.rs::lower_proc_decl` — where the + push/swap/pop emission would go +- `vw-repl/src/app.rs::resolve_stack_frames` — the htcl-coordinate + resolver, unchanged by this proposal diff --git a/docs/htcl-enums.md b/docs/htcl-enums.md new file mode 100644 index 0000000..3a1c7d0 --- /dev/null +++ b/docs/htcl-enums.md @@ -0,0 +1,350 @@ +# Enums (tagged sum types) in htcl + +Enums are htcl's way to model values that can be one of several +distinct shapes. They're tagged unions: every value carries a +variant tag at runtime, the compiler auto-generates the +boilerplate (constructors, repr, accessors), and overloaded +handlers dispatch on the tag with no runtime string introspection +on the compiler side. + +This is the principled answer to "I have a value that's +*sometimes* a scalar and *sometimes* a nested dict" — the +canonical case being Vivado property values, where +`get_property NAME $obj` returns a string but +`get_property CONFIG.PS_PMC_CONFIG $obj` returns an embedded +property dict. + +See also: [htcl-return-types.md](htcl-return-types.md) for the +broader type system, [authoring-htcl-libraries.md](authoring-htcl-libraries.md) +for the surrounding arg/return-type annotation grammar. + +## Syntax + +Declaration: + +``` +enum Property = { + Scalar: string + Nested: dict +} + +type Properties = dict +``` + +The variants block is **brace-wrapped and newline-separated** — +the same shape as `proc {arg1; arg2}`. Each variant is +`IDENT (':' TYPE)?`; the payload type is optional, so +empty-payload variants are first-class: + +``` +enum Direction = { + North + South + East + West +} +``` + +Qualified variant types for use in arg annotations: + +``` +proc handle_prop {v: Property::Scalar} string { return "scalar: $v" } +proc handle_prop {v: Property::Nested} string { return "nested children: [llength $v]" } +``` + +The compiler sees that two `handle_prop` procs share a name, that +each first arg is a different variant of `Property`, and +synthesizes a public `handle_prop` dispatcher — no user-written +`proc handle_prop {v: Property} ...` boilerplate. + +## Runtime representation + +Every variant value is a Tcl list: + +- **With payload**: `[list ]` — two elements. + `Property::Scalar "foo"` → `[list Scalar foo]`. +- **Without payload**: `[list ]` — single element. + `Direction::North` → `[list North]`. + +The variant short-name (`Scalar`, not `Property::Scalar`) is +enough for dispatch because the dispatcher already knows which +enum it's switching on. + +## Auto-emitted machinery + +For an enum declaration the compiler emits a single +`namespace eval { … }` block. For +`enum Direction = { North; South: int; East; West }` it looks +roughly like: + +```tcl +namespace eval Direction { + # Constructors — one per variant. + proc North {} { return [list North] } + proc South {v} { return [list South $v] } + proc East {} { return [list East] } + proc West {} { return [list West] } + + # Accessors — explicit unwrap entry points wrappers use when + # bridging to extern:: calls. + proc tag {v} { return [lindex $v 0] } + proc payload {v} { return [lindex $v 1] } + + # repr — switches on tag, calls payload type's repr. Renders + # as `Variant()` for payload variants, bare `Variant` + # for empty ones. + proc repr {v} { … } + + # from / to — identity (exist so generics over enums type-check + # uniformly with newtypes). + proc from {v} { return $v } + proc to {v} { return $v } +} +``` + +**No user-written triplet is required for an enum** (newtypes +require `repr`/`from`/`to`; enums get them auto-generated). If +the user wants custom rendering, they can override the proc +post-hoc — same as any other htcl proc. + +## Bridging to extern (lowering) + +EDA builtins (`extern::create_bd_cell`, `extern::get_property`, +etc.) don't understand tagged tuples — they expect bare Tcl +primitives. Any time an enum value flows into an `extern::` call, +it has to be unwrapped first. + +**v1 policy: explicit unwrap, no auto-lowering.** Wrappers (the +procs that own the `extern::` boundary) explicitly extract the +payload via `::payload`: + +``` +proc vivado_cmd::set_string_prop {obj: bd_cell; name: string; val: Property} unit { + # Property::payload extracts the inner value; if val is + # Scalar("foo"), this yields "foo". The wrapper is responsible + # for knowing this is the right shape — the compiler doesn't + # auto-coerce. + extern::set_property -dict [list $name [Property::payload $val]] -objects $obj +} +``` + +Newtypes don't need unwrap — `bd_cell` IS a string at runtime, +so `extern::foo $cell` already passes the right thing. + +Compiler-side **auto-lowering** (walk the expression tree, find +every enum-typed value being passed to an extern, insert the +unwrap automatically) requires full type inference across +expressions and is out of scope for v1. The explicit +`::payload` form is principled (no magic at call sites) +and gives wrapper authors visibility into where lowering +happens. + +## Lifting from extern + +The other direction — taking an EDA function's raw Tcl return +value and tagging it into a typed enum — is per-function +business. `extern::get_property NAME $obj` returns a scalar; +`extern::get_property CONFIG.PS_PMC_CONFIG $obj` returns an +embedded dict. Whether a given property is one or the other is +**metadata the wrapper queries from the EDA tool** (e.g. +`extern::report_property -type`), not a shape-of-string +heuristic. + +**v1 policy: lifting lives in the wrapper, not the compiler.** +Each wrapper that returns an enum decides which variant to +construct. The compiler doesn't try to be smart — there's no +shape-guessing path in compiler-emitted code. + +To avoid every wrapper reinventing the wheel, the +`~/src/htcl/amd/vivado-cmd/lift.htcl` library provides a small +set of reusable helpers: + +``` +# Structural check: is the string a well-formed Tcl list with +# an even length and bare-ident keys? Used by wrappers that +# already have other evidence the value MIGHT be a paired dict +# and need a sanity check — NOT as a primary classifier. +proc lift::looks_like_paired_dict {raw: string} bool { … } + +# Vivado-specific: lift a property value to Property using +# `extern::report_property -type` metadata. The classifier IS +# the heuristic — but it's named, scoped, and called from one +# place instead of being baked into the compiler. +proc lift::vivado_property {obj: bd_cell; name: string; raw: string} Property { … } +``` + +Wrappers compose these. Custom cases write their own lifters — +the helper library is convenience, not a requirement. + +**Future direction**: F# data providers as inspiration for +`vw ip generate`. Given an IP-XACT schema, the generator could +emit not just wrappers but also the per-component tagging logic +— declarative schema in, typed lifting out. Worth investigating +once the v1 enum machinery is in user hands and we see which +lifting patterns actually recur. + +## Overload dispatch + +When two or more procs share a name AND each one's first arg is +declared as a different variant of the same enum, the compiler +treats them as **a single overloaded function** rather than a +duplicate-definition warning. + +``` +proc handle_prop {v: Property::Scalar} string { return $v } +proc handle_prop {v: Property::Nested} string { + set parts [list] + foreach {k val} $v { lappend parts "$k=[handle_prop $val]" } + return [join $parts ", "] +} +``` + +The compiler: + +1. **Verifies exhaustiveness** — every variant of `Property` + must have a handler. Missing variants are a hard error + pointing at the first overload, listing the gaps. +2. **Verifies tail-arg agreement** — every overload must + declare identical args after the dispatched first one + (same names, attributes, type annotations). +3. **Verifies return-type agreement** — every annotated + return must be identical. Mixing annotated and unannotated + is an error. +4. **Renames specializations** to `__handle_prop__Scalar` + and `__handle_prop__Nested` internally. User procs whose + names start with `__` are forbidden — that prefix is + reserved for compiler-emitted names. +5. **Synthesizes a public dispatcher**: + ```tcl + proc handle_prop {v args} { + switch -- [lindex $v 0] { + Scalar { return [__handle_prop__Scalar [lindex $v 1] {*}$args] } + Nested { return [__handle_prop__Nested [lindex $v 1] {*}$args] } + } + } + ``` + The payload is unwrapped before the specialization runs, so + the body of `proc handle_prop {v: Property::Scalar}` sees + `$v` as the bare string — matches Haskell `case` semantics. +6. **Registers a synthetic public signature** in the proc table + under the public name. Specializations register under their + mangled names so analyzer drill-down still finds them. + +### What's NOT allowed + +Two procs sharing a name where the first args aren't both +variants of one enum is a **hard error** ("ad-hoc overloading +not supported"). Examples: + +- `proc foo {x: int}` + `proc foo {x: string}` — different + primitives, no enum to dispatch on. +- `proc foo {x: Property::Scalar}` + `proc foo {x: Color::Red}` + — different enums. +- `proc foo {x: Property::Scalar}` + `proc foo {x: Property::Scalar}` + — duplicate variants. + +If you legitimately want a single function that handles +unrelated types, rename one of them or wrap the union in an +enum. + +## Recursive types + +Enums and the types they reference can be mutually recursive: + +``` +enum Property = { + Scalar: string + Nested: Properties +} +type Properties = dict +``` + +`Property` references `Properties`, which references `Property`. +Codegen handles this fine — Tcl resolves proc references at call +time, not parse time, so the order in which the namespaces are +emitted doesn't matter. The validator's type-decl-table +collection runs to completion before per-type checks fire, so +forward references work. + +## Worked example: `util::props` + +The motivating case. Vivado property values are heterogeneous — +some scalars (`NAME cips`), some embedded dicts +(`CONFIG.PS_PMC_CONFIG CLOCK_MODE Custom DESIGN_MODE 1 …`). + +Pre-enum (today): `util::props` returns `dict`. +Embedded dict values render as long single lines that wrap at +the terminal — visually confusing. + +With enums: + +``` +# types.htcl +enum Property = { + Scalar: string + Nested: Properties +} +type Properties = dict + +# lift.htcl — the heuristic lives in a named, scoped place. +proc lift::vivado_property {obj: bd_cell; name: string; raw: string} Property { + set kind [extern::report_property -type $obj $name] + if {$kind eq "bool" || $kind eq "string" || $kind eq "long"} { + return [Property::Scalar $raw] + } + # Composite: recurse through the embedded dict. + set inner [dict create] + foreach {k v} $raw { + dict set inner $k [lift::vivado_property $obj "$name.$k" $v] + } + return [Property::Nested $inner] +} + +# util.htcl — the wrapper just builds the typed result; the +# compiler handles the rendering via the auto-generated +# Property::repr and the monomorphized Properties::repr. +proc util::props {object: bd_cell} Properties { + set result [dict create] + foreach name [extern::list_property $object] { + set raw [extern::get_property $name $object] + dict set result $name [lift::vivado_property $object $name $raw] + } + return $result +} +``` + +In the REPL: + +``` +› util::props -object $cips + CLASS Scalar(bd_cell) + NAME Scalar(cips) + CONFIG.PS_PMC_CONFIG Nested( + CLOCK_MODE Scalar(Custom) + DESIGN_MODE Scalar(1) + PCIE_APERTURES_DUAL_ENABLE Scalar(0) + … + ) + … +``` + +— recursive structure rendered with no string-shape heuristics +in the compiler-emitted code. The `Property::repr` switch +dispatches on tag, `Properties::repr` (auto-monomorphized from +`dict`) iterates pairs and recurses. + +## Out of scope for v1 + +- **Compiler-side auto-lowering at extern:: call sites** — would + need cross-expression type inference. Wrappers explicitly + unwrap via `::payload`. +- **Ad-hoc overloading** (procs sharing a name where args aren't + variants of one enum) — hard error; add as a distinct feature + later if needed. +- **Multi-arg dispatch** (Julia-style) — first-arg dispatch only. +- **Generic enums** (`enum Result = Ok: T | Err: E`) — needs + type-parameter machinery; defer. +- **Pattern guards / nested patterns** — single arm per variant. +- **F#-style data-provider generation** for IP-XACT schemas — + declarative schema → typed lifting code is a multi-week + project of its own. Note as future direction. diff --git a/docs/htcl-return-types.md b/docs/htcl-return-types.md new file mode 100644 index 0000000..2bcfd48 --- /dev/null +++ b/docs/htcl-return-types.md @@ -0,0 +1,246 @@ +# Return-type annotations in htcl + +htcl procs may declare a return type in a 4th-word slot between the +args block and the body: + +``` +proc make_widget { @arg(name) ... } widget { + …body… +} +``` + +The annotation is purely additive — procs without it parse and run +identically to today. With it, the REPL printer and the analyzer +(hover, signature help) start treating the proc's result as +typed. + +## Syntax + +Three pieces: + +``` +proc NAME { ARGS } TYPE { BODY } +``` + +The args list and body keep their existing shapes (see +[authoring-htcl-libraries.md](authoring-htcl-libraries.md) for +the arg attribute grammar). The new `TYPE` slot is a single htcl +word: + +- A bare identifier: `string`, `int`, `bool`, `unit`, `bd_cell`, + `widget`, … +- A generic with no whitespace: `list`, + `dict`, `list>`. +- A brace-wrapped type when the expression contains spaces: + `{dict}` — the parser strips the outer braces + before type-parsing. + +### Grammar + +``` +Type ::= IDENT ('<' Type (',' Type)* '>')? +``` + +Nested generics work to arbitrary depth. + +For heterogeneous values that can take one of several distinct +shapes — e.g. Vivado property values that are sometimes scalars +and sometimes embedded dicts — use **enums** (tagged sum types). +See [htcl-enums.md](htcl-enums.md) for the full design. + +## Type vocabulary + +### Primitives (built into the compiler) + +| Type | Repr | +| --------- | ------------------------------------------------------- | +| `string` | identity | +| `int` | `[format %d $v]` | +| `bool` | `true` / `false` | +| `unit` | empty string; the REPL suppresses the Result entry | + +`unit` is the "I don't return a meaningful value" type — use it on +side-effecting procs (logging, connecting, configuring) so the +REPL stops trying to render their empty return as output. + +### Newtypes (user-declared) + +Any other type is a newtype, introduced by: + +``` +type NAME = UNDERLYING +``` + +Every newtype declaration **must** be accompanied by three procs +in a namespace matching the type name — the validator rejects the +program otherwise: + +| Proc | Signature | Purpose | +| -------------- | ---------------------------------- | -------------------------------------------- | +| `::repr` | `proc ::repr { v } string { … }` | Render an instance for display. | +| `::from` | `proc ::from { v } { … }` | Validate + lift an underlying value into T. | +| `::to` | `proc ::to { v } { … }` | Extract the underlying value back out of T. | + +The `from` proc is the one place to validate — e.g. reject +strings that don't match the Vivado-path shape `^/[\w/]+$` before +they get treated as a `bd_cell`. + +### Example: Vivado's typed handles + +The whole `bd_*` family lives in +`~/src/htcl/amd/vivado-cmd/types.htcl`: + +``` +type bd_cell = string + +proc bd_cell::repr {v} string { return $v } +proc bd_cell::from {v} bd_cell { + if {![regexp {^/[\w/]+$} $v]} { + error "bd_cell::from: '$v' is not a valid block-design path" + } + return $v +} +proc bd_cell::to {v} string { return $v } +``` + +All `bd_pin`, `bd_intf_pin`, etc. follow the same template. + +### Example: a domain newtype + +A user library can introduce its own types the same way: + +``` +type pcie_lane_count = int + +proc pcie_lane_count::repr {v} string { + return "x$v" ;# render as "x1", "x2", "x4", "x8", "x16" +} + +proc pcie_lane_count::from {v} pcie_lane_count { + if {$v ni {1 2 4 8 16}} { + error "pcie_lane_count must be one of {1 2 4 8 16}, got $v" + } + return $v +} + +proc pcie_lane_count::to {v} int { return $v } +``` + +Now a proc annotated `} pcie_lane_count {` will render `x4` in +the REPL instead of `4`. + +### Generics + +`list` and `dict` work over any composition of primitives +and newtypes — the compiler monomorphizes a `repr` proc per unique +instantiation, dispatching to the user's per-type `::repr` at +element boundaries. + +``` +proc list_of_cells {} list { return [list /a /b /c] } +``` + +The REPL invokes the compiler-generated +`list_bd_cell::repr` on the result, which iterates the list and +joins each element's `bd_cell::repr` rendering with newlines: + +``` +› list_of_cells + /a + /b + /c +``` + +For dicts the rendering is `KEY VAL` pairs, one per line: + +``` +proc props {} dict { … } + +› props -object $cips + CLASS bd_cell + NAME cips + … +``` + +## What the type drives + +| Subsystem | Behavior | +| ---------------------- | ---------------------------------------------------------------------------------------- | +| REPL result printer | Wraps the expression with the type's `repr` proc; `unit` suppresses the Result entry. | +| Analyzer hover | Shows `proc NAME → TYPE` in the hover popup. | +| Analyzer signature help | Appends ` → TYPE` to the signature label. | + +Unannotated procs keep the legacy heuristic formatter as a fallback, +so adopting annotations is gradual — annotate as you go. + +## Argument types + +Arguments use the same vocabulary as return types, declared with +a `: TYPE` suffix on the arg name: + +``` +proc plumb_pin { + ## What to name the external port. + name: string + + ## Identity of the pin to make external. + pin: bd_pin +} unit { + … +} +``` + +Rules: + +- Same grammar as return types — primitives, newtypes, and + generics with arbitrary nesting. +- Compatible with existing attributes (`@default(0) count: int`, + `@enum(Master, Slave) mode: string`). +- Optional. Untyped args still parse — adoption is gradual. + +What the annotation drives: + +- **Validator shape checks.** Newtype `::repr/from/to` procs + get a full shape check: `repr` must take `v: T` and return + `string`; `from` must take `v: ` and return `T`; + `to` must take `v: T` and return ``. Annotations + are *optional* on these procs — unannotated args/returns pass + as "trust the user". Annotated mismatches are a hard error. +- **Analyzer display.** Hover and signature help render the arg + as `-name: TYPE` instead of the bare `-name` form. + +Out of scope for v1 (future work): call-site validation +("you're passing a `string` where a `bd_cell` is expected"), +unions, and inference for unannotated args. + +## Authoring conventions + +- **Annotate as you write.** Same effort as documenting an arg, + same payoff as a TypeScript return-type hint. +- **Prefer specific named newtypes over `string`** for values + that have a well-defined shape (paths, IDs, port names). The + `from`/`to` triplet documents the invariant and the `from` + validator catches typos at the boundary. +- **Use `unit` for side-effecting procs.** Anything that calls + `set_property`, `connect_*`, `puts`, or `log::*` is almost + certainly `unit`. The REPL won't bother trying to display + whatever Tcl-internal value falls out. +- **Generics nest freely.** `dict>` is fine. + Don't be afraid to be specific. + +## Limitations (v1) + +- **No arg-type annotations yet.** Args still use only the + attribute grammar (`@default`, `@enum`, etc.). The + `::repr/from/to` validator only checks the procs EXIST, + not that their signatures match shape — that arrives with arg + types. +- **No inference.** Unannotated procs are simply untyped; we + don't walk the body to derive a return type from `return`. +- **No union or function types.** Start small; extend the grammar + as need shows up. + +For the implementation, see `vw-htcl/src/repr.rs` (codegen), +`vw-htcl/src/type_parse.rs` (mini-parser), and the design notes +in [the original plan](../docs/plans/return-types.md) if it +survives the cleanup. diff --git a/docs/htcl.md b/docs/htcl.md new file mode 100644 index 0000000..ef04189 --- /dev/null +++ b/docs/htcl.md @@ -0,0 +1,93 @@ +# HoloTCL + +## Background + +TCL, the Tool Command Language is heavily used to drive Electronic Design +Automation (EDA) software. The typical TCL interface to an EDA software +suite includes not just commands to automate EDA workflows, but also commands +to configure complex intellectual property (IP) packages, inspect designs, +analyze the outputs of the EDA processes that carry a design from source to +implementation such as synthesized netlists, and provide critical parameters and +constraints at multiple stages of the automation process that collectively make +complex electronic designs realizable. + +For designs built on EDA platforms that have a TCL interface, TCL code is a +major part of the engineering process and the overall codebase that needs to +be maintained. The TCL interface that engineers have to work against is quite +large. At the time of writing, Vivado 2025.1, the EDA suite for AMD FPGAs has +around 900 TCL procedures for automation. While certainly non-trivial, the +automation procedures pale in comparison to IP configuration interfaces. A +complex IP can have thousands of interrelated configuration parameters. + +The engineering process for both automation, IP configuration and design +parameterization revolves around a few core questions. + +1. Discovery: what interfaces are available? +2. Semantics: how are those interfaces intended to be used? +3. Structure: what is the shape of those interfaces? +4. Correctness: have I used those interfaces correctly? + +For automation functions, most EDA suites' TCL will give you something +for discovery, semantics and structure. There is a TCL shell that provides +rudimentary command completion and command help menus that informally describe +what the arguments for each command are and give you a vague idea of what +parameters might be acceptable, maybe even with some examples. But because of +TCL's "everything is a string" view of the world, there are fundamental limits +on the amount of structure that can be communicated through a TCL interface +description. + +While it's possible to get by with TCL for EDA process automation. Where the +wheels really fall off is IP configuration. Configuring via TCL is done through +dictionaries where the keys are strings and values are basically anything: +strings, lists of strings, nested dictionaries, lists of dictionaries, lists of +lists etc. These dictionary configuration interfaces are undocumented. While +almost all IP comes with PDF-based documentation, and some come with sections +on the IP parameterization. Experience has shown that these documents are +nowhere near complete, are often just straight up wrong, and provide no +real basis for discovery, semantics, structure or anything close to correctness +validation. + +The EDA disposition is to provide GUIs for IP configuration that emit TCL behind +the scenes. Alas, even these GUIs suffer from the same issues as the TCL itself. +There is no comprehensive documentation for their configuration, and using them +for IP configuration devolves to a guessing game rather than deliberate +engineering. Compounding these issues is that GUI-based configuration that +generates TCL is a lossy channel. An engineer cannot be expected to make a +decision in a GUI, reverse engineer how that decision manifests in generated TCL +that uses a completely undocumented interface and then annotate the generated +TCL, the only engineering artifact they can even put in source control, with the +rationale for their configuration decision, or the way in which they decided to +integrate the IP in to a broader design. Not to mention that the next time they +make changes in the GUI and emit TCL, _new_ TCL will get emitted and the engineer +must merge this with their existing corpus of annotated TCL. Put differently +the GUI to TCL channel is an intrinsically lossy one. A lossy channel that emits +code using an undiscoverable, undocumented and unstructured interface. + +## Rationale + +HoloTCL is an evolution of TCL that aims to make EDA interfaces discoverable, +not just for the simple notion of discovering what procedures and parameters are +available, but discovery of interfaces with semantics and structural definition +build into the interface definitions themselves. Providing a foundation for +tools that can ensure structural and many forms of static correctness by +analyzing an HTCL codebase rather than having to execute an EDA process only to +find simple syntactic, structural or semantic errors hours into a design run. + +This is accomplished with the following foundations. + +- The IP configuration interface moves from unstructured dictionaries to + configuration procedures. + +- HTCL introduces documentation comments and requires them for all procedures + and procedure arguments. + +- All procedure arguments are named keyword arguments in HTCL. + +- HTCL introduces a basic type system, with standard primitive types, + enumerations and type parameterized lists and dictionaries. + +- Procedure arguments are typed, and have return types. + +The idea behind all of this is that the language requires by construction +that the EDA interfaces engineers use are documented and well structured by +construction. diff --git a/docs/launch-runs.md b/docs/launch-runs.md new file mode 100644 index 0000000..25e9dba --- /dev/null +++ b/docs/launch-runs.md @@ -0,0 +1,180 @@ +# `launch_runs` — a possible future + +vw currently drives Vivado through direct commands: `synth_ip`, +`synth_design`, `place_design`, `route_design`, etc. These are the +non-project-mode idiom, and vw calls them against an on-disk +project (`create_project -dir …`). That hybrid works, but Vivado +periodically complains — `[Vivado 12-5447] synth_ip is not +supported in project mode`, `[Project 1-5563] BD generation state +is stale`, `[Vivado 12-13650] IP file has been moved from its +original location`. Each was a rabbit hole; each ended with a +targeted `set_msg_config -suppress` in `~/src/htcl/vw/module.htcl`. + +The "correct" project-mode alternative is Vivado's **runs** +infrastructure: named jobs like `synth_1`, `impl_1`, and one per-IP +OOC synth run (`_synth_1`), driven by `launch_runs +` + `wait_on_run `. Vivado tracks status/progress in +the `.xpr` and materializes DCPs into standard project subdirs. +Instead of us calling `synth_ip` / `synth_design` / +`place_design` / `route_design` directly, we'd do something like: + +```tcl +create_ip_run [get_files primary_clock.xci] ;# if not auto-created +launch_runs {primary_clock_synth_1 …} -jobs 4 +wait_on_run primary_clock_synth_1 +launch_runs synth_1 ;# top synth uses cached IP DCPs +wait_on_run synth_1 +launch_runs impl_1 -to_step write_bitstream +wait_on_run impl_1 +``` + +Notes about state of confidence: I've read Vivado's documented +runs flow but haven't personally driven a large project through +it end-to-end. Estimates below are honest guesses; some things +may prove easier or harder than described. + +## In favor + +- **Ends the fight with Vivado.** Every warning we've been + fighting comes from using non-project commands in a project. + `launch_runs` is what project mode expects. The whole class + should go away — clean baseline, no more `-suppress` list to + maintain. +- **Parallel IP synth for free.** `launch_runs -jobs N` runs + multiple IPs concurrently. Right now `synth_ip` is serial per + IP. Meaningful wall-clock savings on designs with many + top-level IPs. +- **Vivado handles IP-DCP freshness.** Runs track IP source + changes and re-run only what's needed. Our custom + fingerprint/manifest machinery for the IP part + (`synth_needs_update`, etc.) largely becomes redundant — + Vivado's own run status is authoritative. +- **`get_msg_config -count` may start working.** The per-process + CW-count bug (which drove us to build our own Rust-side + counter with a `critical_warning_count` RPC) exists because + `place_design`'s sub-processes aren't tracked. `launch_runs` + uses managed run processes, which the counter is documented + to handle correctly. +- **Compatibility with GUI Vivado.** Anyone opening the `.xpr` + in the IDE sees a "normal" project with proper runs, not a + hybrid state driven from Tcl. + +## Against + +- **Async by default.** `launch_runs` returns immediately; the + run happens in the background. `wait_on_run` blocks until + completion. Different mental model from `synth_design` + (blocking, in-process). Not fatal, but error handling + changes — a run can be in states like `Running`, `Complete!`, + `Failed`, etc., and we'd need to query. +- **Sub-processes = separate log streams.** Each run spawns a + child Vivado process. Its stdout/stderr goes into + `/.runs//runme.log`, not our PTY stream. + We'd lose the real-time output the user sees now — or we'd + need to tail those log files back into our stream. A real UX + regression to solve. +- **Losing our checkpoint conventions.** Today we produce + `target/synth/.dcp`, `target/place/.dcp`, + `target/route/.dcp` at predictable workspace-relative + paths. Runs put DCPs at `/.runs//.dcp` + — Vivado-managed. We'd either symlink or change our conventions. +- **Our per-phase caching becomes tangled.** + `synth_needs_update` / `place_needs_update` / + `route_needs_update` — fingerprinted source tracking with + sidecar `.manifest` files — was designed around us owning DCP + paths and lifecycle. With runs, Vivado owns them and has its + own freshness logic. We'd either delete our machinery (defer + to Vivado's, which we don't currently trust as authoritative + for our source-tree changes — the `synth` fingerprint covers + htcl / vw.toml files Vivado's runs don't know about) or bridge + them (complicated). +- **CW-gated checkpoint writes get harder.** Our current "skip + DCP write if the phase emitted CWs" logic wraps a synchronous + `place_design`. With `launch_runs`, the CWs happen in a + sub-process; we'd read them after the fact from the run log. + Doable but different plumbing. +- **`vw::configure_ip` gets restructured.** Currently we run + user `ip::configure` code that calls `create_ip`, + `create_bd_design`, etc. — those all still work under runs, + but the "synthesize them" step moves from `synth_ip` (which + we call directly) to `launch_runs`. Reasonable, but touches + more code and re-plumbs the CW-gate. +- **`vw test` isolation.** Today `vw test` uses `-in_memory` + and never touches runs. Would need to keep that path working, + so `vw::synth` (and friends) would have TWO paths: in-memory + (current code) and on-disk (launch_runs). Complexity tax. +- **Real effort.** Rough estimate — at least a day of focused + work, maybe two, including debugging Vivado's async behavior + when things go wrong. The wall-clock cost of "just suppress + the warning" is 30 seconds. + +## When it's worth doing + +Suppression is 30 seconds and buys the same end user +experience — clean logs, working flow. `launch_runs` is +architecturally correct and would end the entire class of +project-mode/non-project-command friction, but it's real +engineering and forces us to solve the sub-process-log-streaming +and DCP-path-convention problems. + +Prefer `launch_runs` if any of: + +- You anticipate more warnings of this shape and want to close + the door on them permanently. +- Parallel IP synth would meaningfully speed up iteration. +- You want vw's `.xpr` to be openable in the Vivado GUI as a + "normal" project. + +Otherwise, keep the current flow and add targeted +`set_msg_config -suppress` calls as new warnings surface. + +## What would move if we did this + +Concrete files/procs to expect touching, so scope is calibratable: + +- `~/src/htcl/vw/module.htcl` — `vw::configure_ip`, + `vw::synth`, `vw::place`, `vw::route` would be replaced or + substantially rewritten. New helpers like `vw::_wait_run` + and a per-run CW-log-scraper. +- `~/src/htcl/amd/vivado-cmd/cmd/` — the missing + `create_ip_run.htcl`, `launch_runs.htcl`, `wait_on_run.htcl`, + `get_runs.htcl` wrappers may need to be authored or + regenerated from Vivado man pages. +- `vw-lib/src/lib.rs` — the `synth_source_fingerprint` / + `place_source_fingerprint` / `route_source_fingerprint` + helpers and their manifest sidecars are either deleted or + their scope narrows to "source-tree freshness" (delegating + DCP freshness to Vivado's runs). Removes ~200 lines but + requires re-thinking cross-stage invalidation. +- `vw-vivado/src/handlers.rs` — matching RPCs + (`{synth,place,route}_needs_update`, + `{synth,place,route}_mark_checkpoint`) either shrink or go. +- `vw-vivado/src/worker.rs` — need a way to tail + `/.runs/*/runme.log` files during + `wait_on_run` so the user sees Vivado's progress instead of a + silent block. +- Path conventions — `target/{synth,place,route}/.dcp` are + hardcoded in vw::synth/place/route. Either symlink from those + paths to the runs-generated DCPs, or update every caller. + +That last point (path conventions) is where documentation +outside the code lives — `docs/whypoints.md`, +`docs/new-structure.md` — and would need a pass. + +## Current suppressions (for reference when evaluating) + +Every `set_msg_config -suppress` we've added because of the +project-mode/non-project-command hybrid, so the value proposition +of removing them is concrete: + +- `[Vivado 12-5447] synth_ip is not supported in project mode` — + in `vw::synth`, before `synth_ip` on top-level standalone IPs. +- `[Project 1-5563] File '.bd' generation state is stale` — + in `vw::configure_ip`, before flipping + `synth_checkpoint_mode None` on BDs. +- `[Vivado 12-13650] IP file has been moved from its original + location` — the one that prompted this doc. Not yet + suppressed; deferred pending this decision. + +If we go the `launch_runs` route, all three should become +unnecessary. diff --git a/docs/new-structure.md b/docs/new-structure.md new file mode 100644 index 0000000..ff87745 --- /dev/null +++ b/docs/new-structure.md @@ -0,0 +1,136 @@ +# New Structure + +We're going to be putting together a new structure for `vw` workspaces. A vw +workspace contains + +- A VHDL design +- A Rust-based operating system driver for the VHDL hardware design +- Test suites at multiple levels + - Mixed mode analog/digital testbenches for the hardware/wire interface + - Pure digital testbenches for the VHDL design + - Pure unit and rust integration tests for the Rust driver + - Codesign tests that integrate combinations of Rust, digital and mixed mode + digital/analog tests. + +The filesystem structure that captures this is the following + +workspace-root +|- `design.htcl`: entry point for HTCL-based configuration and automation +|- `ip/**/*.htcl`: IP configuration files used by design.htcl +|- `hdl/**/*.vhd`: VHDL design sources +|- `bench/**/*.rs`: Rust-based hardware testbenches, pure digital and mixed mode +|- `driver/**/*.rs`: Rust-based kernel driver for the hardware design + +All of this machinery exists in ~/src/redhawk. But the organization is quite +haphazard. Within ~/src/redhawk most of the design and testbenches live within +host/hdl/n1 under design and bench respectively. This is basically the simulated +world. But then when we need to enter the synthesized world, we have to go to +vivado/vpk120-evb where a set of completely unmaintainable TCL scripts smash +a vivado project onto host/hdl/n1/design that's locally symlinked into +vivado/vpk120-evb. Within vivado/ there is also the metro folder which is a +vivado project for our custom metro motherboard with a versal on it, as opposed +to the vpk120-evb which is a vivado project for an AMD evaluation board. + +The bottom line is redhawk has become a hot mess and we're going to start +incrementally pulling redhawk sources here into metroid using the organization +I described above. + +Some other things to be aware of are: + +## Anodizer + +Much of our test infra depends on anodizer (at ~/src/anodizer) +which is machinery for generating Rust structures that correspond to VHDL +records. + +## Rust Co-sim + +We have developed Rust-based cosimulation machinery (at ~/src/rust-cosim). This +allows us to define our test benches in Rust instead of VHDL. Recently, this +machinery grew support for testing VHDL entities directly in rust, without +having to build a VHDL test bench wrapper. This is the path forward, but we +still have many test benches with explicit VHDL testbench harnesses that are +driven by Rust. + +## RSF + +Eventually we want to generate RSF specifications (~/src/rsf) for registe +interfaces. Right now these are manually maintained between our VHDL register +interfaces and our Rust kernel drivers. + +## Builds from the bottom up + +One of the goals here is to build form the bottom up. + +1. Configure IP, generate wrappers and synthesize IP. +2. Elaborate/Synthesize/implement VHDL which depends on IP synth and wrappers + from (1) +4. Generate RSF specs from VHDL (this has yet to be done, maybe with anodizer + later?) +5. Build rust testbenches using anodizer +6. Build driver code using cargo with a build.rs that pulls in generated RSF + and anodizer artifacts (if/when needed) + +A lot of the machinery here has yet to be built, but this is the overall flow +we are looking for, and for it to be completely automatic. Data structures +should not be manually maintaind across the hardware/software boundary. + +## Vhdl-ls + +There is currently a strange relationship between vhdl-ls and vw, in that vw +uses vhdl-ls config to find VHDL files. I want to stop that. I'd like vhdl-ls +to get it's config from vw and not the other way around. I would even like to +explore embedding vhdl-ls as a library and have the vw LSP be the entry point +for HTCL, VHDL and Rust sources combined (also need to figure out the Rust +side of that). I've put the vhdl-ls sources at ~/src/rust_hdl. + +To further drive home the point, the structure described at the beginning of +this document means that vw knows where ALL files are according to that +structure, so we don't need to glean it from other configs. + +## Where to start + +### 1. Pull in design code and synthesize it + +I would like to start by pulling in our VHDL design code and then synthesize it. +This will involve + +1. Copying redhawk hdl design code into the hdl directory +2. Adding a `vhdl_dependency_sources` function to the vw htcl module. This will + be rust under the hood that we call via our RPC mechanism that returns all + the VHDL sources VW has pulled in as dependencies. +3. Adding a `vhdl_design_sources` function to the vw htcl module. This will be + rust under the hood that we call via our RPC mechanism that finds all the + design sources in the vw workspace and returns them as a list. +4. Extending design.htcl to synthesize our VHDL design sources together with + our VHDL dependency sources. + +### 2. Sort out the vw/vhdl-ls relationship + +Now that (1) is done and we can synthesize our VHDL design sources, we need +to get vhdl-ls working. Let's explore having our existing vw LSP use vhdl-ls +as a library to see if that is a reasonable approach. Something else to +consider is using vhdl-ls standalone, but figure out a way for vw to provide +it with a dynamic configuration source. + +### 3. Start to bring over test benches + +We've got quite a few test benches to bring over. This may be one of the +more compex tasks. I'd like them to be runnable with `vw bench` and have +the same nice interface we worked to develop for `vw test` for HTCL tests. +This will take a fair amount of iteration to get right and to fully integrate +anodizer, rust-cosim and mixed-mode simulation support that uses Xyce under +the hood. There are some heavyweight C/C++ dependencies lurking beneath here +that we want to integrate in a way that can capture all the build and runtime +complexity there in a reasonable way that does not make using the testbenches +a constant pain. + +### 4. Bring the driver over + +We can start by just bringing the rhdrv cargo workspace over from redhawk. The +harder part is going to be thinking about how we integate this with everything +else. e.g. generating RSF specs from VHDL code, propagating those up to a place +where build.rs can capture them and having all that be automatic. Automatic +means robust change and rebuild detection as well. + + diff --git a/docs/remote-holistic-builds.md b/docs/remote-holistic-builds.md new file mode 100644 index 0000000..9f35c44 --- /dev/null +++ b/docs/remote-holistic-builds.md @@ -0,0 +1,69 @@ +# Remote Holistic Builds + +VW builds require at least two different types of machines today + +1. A Linux machine that has vivado installed for building FPGA images +2. A Helios machine for building kernel modules. + +The Oxide Cloud Computer is the perfect substrate to execute these +multi-instance builds. But that means we need to create a vw service +that can manage VW build environments composed of multiple underlying +instances. + +The concept for this looks like the following + +``` + ┌───────────────────────────────────────────────┐ + │ Oxide Cloud Computer │ + │ │ + │ ┌───────────────────────────┐ │ + │ │ environment ├┐ │ + src, │ │ ┌──────────┐ ┌──────────┐ │├┐ │ +┌────────┐ commands │ ┌────────┐ │ │ vivado │ │ helios │ │││ │ +│ │─────────────┼─▶│ │ │ │ instance │ │ instance │ │││ │ +│ vw-cli │ │ │ vw-svc │ │ └──────────┘ └──────────┘ │││ │ +│ │◀────────────┼─ │ │ │ ┌──────────┐ │││ │ +└────────┘ results │ └────────┘ │ │ artifact │ │││ │ + │ │ │ instance │ │││ │ + │ │ └──────────┘ │││ │ + │ └┬──────────────────────────┘││ │ + │ └┬──────────────────────────┘│ │ + │ └───────────────────────────┘ │ + │ │ + └───────────────────────────────────────────────┘ + +``` + +We add new `vw-svc` crate. It's a binary create that produces a daemon +that exposes an API server for + +1. Managing build environments. +2. Carrying out tasks within those build environments. + +In the diagram above, each build environment contains a vivado instance, +a helios instance, and an artifact instance. The artifact instance is +an S3 server that provides a place for build outputs to go, and also +a place for intermediate artifacts the be shared between instances. + +An example workflow could look something like this. + +1. Set up a remote environment with `vw cloud init `. +2. Now just use `vw` like it's used today, `vw run` executes `design.htcl` + `vw repl` takes the user into the repl. But when a cloud environment + has been initialized, all of this takes place remotely on the vivado instance. + Same thing for `vw bench`, it will feel the same as it does today, but it will + execute on the vivado instance in the cloud. + +This obviously means that we'll need a background daemon that synchronizes sources +from the local machine running the `vw` client to the `vw-svc` daemon which will +then distribute the sources to the backend instances they need to go to. The `vw` +cli should manage that daemon directly, the user should not need to muck with it. + +We'll also need to build an artifact daemon for the vivado and helios instances +that watch `target` directories and ship artifacts to the artifact instance S3 +server. + +Something that's important about this system is that it be interactive. A call +to `vw run` cannot be no output for an hour+ while a synth/place/route run is +going. We need to stream data back in real time to the client and deliver the +same experience as the local tool has today. diff --git a/docs/snippets/hdl/top.vhd b/docs/snippets/hdl/top.vhd new file mode 100644 index 0000000..278d69f --- /dev/null +++ b/docs/snippets/hdl/top.vhd @@ -0,0 +1,8 @@ +library ieee; +use ieee.std_logic_1164.all; + +entity top is + port ( + clk: in std_logic + ); +end top; diff --git a/docs/snippets/module.htcl b/docs/snippets/module.htcl new file mode 100644 index 0000000..6314b67 --- /dev/null +++ b/docs/snippets/module.htcl @@ -0,0 +1,20 @@ +src @clk-wizard + +set clkout [ + clk_wizard::clkout + -requested_out_frequency {400,_,_,_,_,_,_} +] + +set clk_cfg [ + clk_wizard::configure + -prim_in_freq {250} + -prim_source No_buffer + -clkout $clkout + -use [ + clk_wizard::use + -locked true + -reset true + ] +] + +clk_wizard::create -name primary_clock -config $clk_cfg diff --git a/docs/snippets/vw.lock b/docs/snippets/vw.lock new file mode 100644 index 0000000..97101a7 --- /dev/null +++ b/docs/snippets/vw.lock @@ -0,0 +1,19 @@ +[dependencies.vivado-cmd] +repo = "https://github.com/oxidecomputer/vivado-cmd.htcl" +commit = "90ffd540e999170cda2a98022f130e9da78d165d" +src = [] +path = "vivado-cmd-90ffd540e999170cda2a98022f130e9da78d165d" +recursive = false +sim_only = false +submodules = false +exclude = [] + +[dependencies.clk-wizard] +repo = "https://github.com/oxidecomputer/clk-wizard.htcl" +commit = "75c2685d33903b9bf269076252285a1b7e72d014" +src = [] +path = "clk-wizard-75c2685d33903b9bf269076252285a1b7e72d014" +recursive = false +sim_only = false +submodules = false +exclude = [] diff --git a/docs/snippets/vw.toml b/docs/snippets/vw.toml new file mode 100644 index 0000000..f6af4f4 --- /dev/null +++ b/docs/snippets/vw.toml @@ -0,0 +1,15 @@ +[workspace] +name = "." +version = "0.1.0" +variants = [] +top = "top" + +[[workspace.target-parts]] +part = "xcau25p-ffvb676-2-e" +default = true + +[dependencies.clk-wizard] +repo = "https://github.com/oxidecomputer/clk-wizard.htcl" +branch = "main" + +[test-dependencies] diff --git a/docs/testbench-streamline.md b/docs/testbench-streamline.md new file mode 100644 index 0000000..5a8f47f --- /dev/null +++ b/docs/testbench-streamline.md @@ -0,0 +1,9 @@ +# Streamlining Testbenches + +Currently running a testbench requires the following procedure. + +``` +mkdir -p target/anodizer/build +mkdir -p target/anodizer/generated +anodizer gen_structs --build-dir target/anodizer/build --output target/anodizer/build +```` diff --git a/docs/whypoints.md b/docs/whypoints.md new file mode 100644 index 0000000..c763de9 --- /dev/null +++ b/docs/whypoints.md @@ -0,0 +1,74 @@ +# Why + +Why HTCL + +1. Structured, documented discoverable interface. +2. Catch common programming errors at analysis time. + +The H stands for holomorphic, meaning this type of TCL captures the whole +shape of a tool's interface. + +## Notes + +1. Documentation versus engineering reality + - There is always a detla between Xilinix product guides and the configuation + surface reality. + - DCMAC `tx_data_out_*` is an example of this. It's not mentioned anywhere in + PG 369. Other examples are abundant. + - Tightening up PDF documentation is not the answer. Decoupled PDF as a means + of documenting an engineering interface is a failure by design. + - The interface that engineers actually use must itself be documented, it's + only way to do this. + +2. Structured interfaces + - Configuring IP is complex, both in the number of parameters available and + how those paramters are structured as an overall configuration. + - Structured interfaces make it clear what configuration opions there are + and how they can be composed. + - Structured interfaces enforce structurally correct configurations by + construction, e.g. they make it impossible to compose structurually invalid + configurations. + - Building structure into interfaces empowers analyzers to catch many clases + of bugs. + - Incorrect assignment of values can be caught by the type system through + enums and composite types. + - Subtle issues like multiple assignment of the same paramter can be caught + at compile/analysis time rather than runtime (actually *running* an IP + configuration and synthesis script can take hours) + - Configuration key typos manifest as compile/analysis time errors. + - Structured interfaces enable discoverability, through tools like LSPs as + well as documentation generators. When configuration takes place through + functions with well documented arguments, the design surface of interest + is readily discoverable by the engineer. And critically, there are no more + guessing games on what the right paramter actually is. + - The analyzer can catch and warn about unused variables. Forgetting to + actually use a variable can lead to subtle bugs that can take hours if not + days to catch for complex designs. + - Configuration options as typed enumerations is extremely powerful + - If a config option just takes a string, it's + 1. Not at all clear what the valid options are + 2. Even if you think you know the valid options, it's easy to be wrong or + make a type + - Typed enumerations take away both problems entirely. + - They structually define what values are valid, making them *discoverable* + - This means analyzers can catch invalid issues, not the runtime after an + hour of running. + - It provides a natural surface for documenting the input alternatives. + +3. Catch common programming errors at analysis time + - Design build scripts can take an enormous amount of time to run. + - There are many things that can be caught by basic analysis and not having + to run them at all. + - Functions that declare a return type but: + - Return nothing + - Return the wrong type + - Don't _always_ return the right type + - Passing incorrect arguments types to procs + - Passing the wrong arguments to procs + - Calling undefined procs + - Refrencing undefined variables + - Sourcing dependencies that don't exist + - Supplying enumeration values as strings with typos + - Catching these hours into a built is a punch in the face that is completely + unnecessary and avoidable. + - HTCL catches all of these at analysis time in seconds. diff --git a/openapi/vw-admin-api/vw-admin-api-1.0.0-805e95.json b/openapi/vw-admin-api/vw-admin-api-1.0.0-805e95.json new file mode 100644 index 0000000..eb35133 --- /dev/null +++ b/openapi/vw-admin-api/vw-admin-api-1.0.0-805e95.json @@ -0,0 +1,355 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "VW admin API", + "description": "Manage every user's vw build environments. Restricted to the operators named in the service's --admin-users argument.", + "version": "1.0.0" + }, + "paths": { + "/environment/{user}/{name}": { + "delete": { + "summary": "Delete an environment with the specified name for the specified user.", + "operationId": "delete_environment", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "user", + "description": "User the environment belongs to", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environments": { + "get": { + "summary": "Return a list of all environments.", + "operationId": "get_environments", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserEnvironmentResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Environment": { + "description": "An environment is a collection of instances that work together to build, analyze and test vw designs.", + "type": "object", + "properties": { + "artifact_instance": { + "nullable": true, + "description": "The Oxide instance id for the artifact instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + }, + "helios_instance": { + "nullable": true, + "description": "The Oxide instance id for the helios instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + }, + "images": { + "nullable": true, + "description": "The images this environment's instances boot from, chosen when the environment was created.\n\nAbsent when the service has no Oxide backend configured, in which case the environment is a bare record that will never be provisioned.", + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentImages" + } + ] + }, + "name": { + "description": "The name of this environment.", + "type": "string" + }, + "vivado_instance": { + "nullable": true, + "description": "The Oxide instance id for the vivado instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + } + }, + "required": [ + "name" + ] + }, + "EnvironmentImages": { + "description": "The images each of an environment's instances boots from.", + "type": "object", + "properties": { + "artifact": { + "description": "Image the artifact instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + }, + "helios": { + "description": "Image the helios instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + }, + "vivado": { + "description": "Image the vivado instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + } + }, + "required": [ + "artifact", + "helios", + "vivado" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "ImageRef": { + "description": "An Oxide image an environment's instances are built from.\n\nPinned by id, so publishing a newer image does not silently change what an existing environment boots. The name is carried along for display.", + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] + }, + "InstanceState": { + "description": "Running state of an Instance (primarily: booted or stopped)\n\nThis typically reflects whether it's starting, running, stopping, or stopped, but also includes states related to the Instance's lifecycle\n\n
JSON schema\n\n```json { \"description\": \"Running state of an Instance (primarily: booted or stopped)\\n\\nThis typically reflects whether it's starting, running, stopping, or stopped, but also includes states related to the Instance's lifecycle\", \"oneOf\": [ { \"description\": \"The instance is being created.\", \"type\": \"string\", \"enum\": [ \"creating\" ] }, { \"description\": \"The instance is currently starting up.\", \"type\": \"string\", \"enum\": [ \"starting\" ] }, { \"description\": \"The instance is currently running.\", \"type\": \"string\", \"enum\": [ \"running\" ] }, { \"description\": \"The instance has been requested to stop and a transition to \\\"Stopped\\\" is imminent.\", \"type\": \"string\", \"enum\": [ \"stopping\" ] }, { \"description\": \"The instance is currently stopped.\", \"type\": \"string\", \"enum\": [ \"stopped\" ] }, { \"description\": \"The instance is in the process of rebooting - it will remain in the \\\"rebooting\\\" state until the VM is starting once more.\", \"type\": \"string\", \"enum\": [ \"rebooting\" ] }, { \"description\": \"The instance is in the process of migrating - it will remain in the \\\"migrating\\\" state until the migration process is complete and the destination propolis is ready to continue execution.\", \"type\": \"string\", \"enum\": [ \"migrating\" ] }, { \"description\": \"The instance is attempting to recover from a failure.\", \"type\": \"string\", \"enum\": [ \"repairing\" ] }, { \"description\": \"The instance has encountered a failure.\", \"type\": \"string\", \"enum\": [ \"failed\" ] }, { \"description\": \"The instance has been deleted.\", \"type\": \"string\", \"enum\": [ \"destroyed\" ] } ] } ```
", + "oneOf": [ + { + "description": "The instance is being created.", + "type": "string", + "enum": [ + "creating" + ] + }, + { + "description": "The instance is currently starting up.", + "type": "string", + "enum": [ + "starting" + ] + }, + { + "description": "The instance is currently running.", + "type": "string", + "enum": [ + "running" + ] + }, + { + "description": "The instance has been requested to stop and a transition to \"Stopped\" is imminent.", + "type": "string", + "enum": [ + "stopping" + ] + }, + { + "description": "The instance is currently stopped.", + "type": "string", + "enum": [ + "stopped" + ] + }, + { + "description": "The instance is in the process of rebooting - it will remain in the \"rebooting\" state until the VM is starting once more.", + "type": "string", + "enum": [ + "rebooting" + ] + }, + { + "description": "The instance is in the process of migrating - it will remain in the \"migrating\" state until the migration process is complete and the destination propolis is ready to continue execution.", + "type": "string", + "enum": [ + "migrating" + ] + }, + { + "description": "The instance is attempting to recover from a failure.", + "type": "string", + "enum": [ + "repairing" + ] + }, + { + "description": "The instance has encountered a failure.", + "type": "string", + "enum": [ + "failed" + ] + }, + { + "description": "The instance has been deleted.", + "type": "string", + "enum": [ + "destroyed" + ] + } + ] + }, + "OxideInstance": { + "description": "Information about an Oxide instance that underpins a VW instance.", + "type": "object", + "properties": { + "external_ip": { + "nullable": true, + "description": "The address to reach this instance on from outside the rack, once it has one.\n\nAbsent until the instance exists and the control plane has attached an external address to it. This is what a developer's ssh goes to.", + "type": "string", + "format": "ip" + }, + "id": { + "nullable": true, + "description": "The Oxide instance id, once the control plane has assigned one.\n\nAbsent in the window between asking for an instance and hearing back about it, which is long enough to be worth showing: an environment mid-creation reports `creating` with no id rather than looking like nothing has happened.", + "type": "string", + "format": "uuid" + }, + "internal_ip": { + "nullable": true, + "description": "The instance's address on the rack's own network.\n\nWhat `vw-svc` sends source to, rather than the external address: the internal path is a regional fabric rather than the public internet, and the difference is most of the bandwidth.", + "type": "string", + "format": "ip" + }, + "state": { + "$ref": "#/components/schemas/InstanceState" + } + }, + "required": [ + "state" + ] + }, + "UserEnvironment": { + "description": "An environment together with the owner's username.", + "type": "object", + "properties": { + "environment": { + "description": "Environment info", + "allOf": [ + { + "$ref": "#/components/schemas/Environment" + } + ] + }, + "user": { + "description": "User the environment belongs to", + "type": "string" + } + }, + "required": [ + "environment", + "user" + ] + }, + "UserEnvironmentResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/UserEnvironment" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/vw-admin-api/vw-admin-api-latest.json b/openapi/vw-admin-api/vw-admin-api-latest.json new file mode 120000 index 0000000..d1b766f --- /dev/null +++ b/openapi/vw-admin-api/vw-admin-api-latest.json @@ -0,0 +1 @@ +vw-admin-api-1.0.0-805e95.json \ No newline at end of file diff --git a/openapi/vw-sync-api/vw-sync-api-1.0.0-b72ec8.json b/openapi/vw-sync-api/vw-sync-api-1.0.0-b72ec8.json new file mode 100644 index 0000000..5c4eb8c --- /dev/null +++ b/openapi/vw-sync-api/vw-sync-api-1.0.0-b72ec8.json @@ -0,0 +1,826 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "VW agent API", + "description": "Receive source on a build instance. Reachable only from vw-svc over the rack's internal network.", + "version": "1.0.0" + }, + "paths": { + "/environment/{environment}/artifact-target": { + "get": { + "summary": "Where this instance currently believes its artifacts should go.", + "description": "Answers `404` when it has never been told. Lets the service notice an instance that needs configuring — one created before there was a store, or whose store has since been rebuilt with a new key — without pushing credentials at every instance on every restart.", + "operationId": "get_artifact_target", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3Credentials" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Tell this instance where to put the artifacts it builds.", + "description": "Sent by `vw-svc`, which got it from the instance that runs the store — this one cannot ask directly, since it has no way to know which of its neighbours holds it. Remembered on disk, so a reboot between two builds does not lose the answer.", + "operationId": "put_artifact_target", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3Credentials" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/bench/session": { + "get": { + "summary": "Run this workspace's testbenches on this instance.", + "description": "A websocket for the same reason a vivado session is one: a batch takes minutes and finishes one bench at a time, and a developer watching it should see each result land rather than a verdict at the end.", + "operationId": "bench_session", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "concurrency", + "description": "How many benches run at once. Absent lets the instance decide from its own processor count, which is the number that matters — it is the machine doing the work.", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "query", + "name": "filter", + "description": "Substring match against a testbench's entity name. Absent runs all.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "ignore", + "description": "Directory names to skip while looking for benches, comma separated.\n\nOne string rather than a repeated parameter because a query parameter has to be scalar, and because that is already how `--ignore` is written on the command line.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "standard", + "description": "The VHDL standard, as `nvc` spells it.", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + }, + "/environment/{environment}/build-output": { + "delete": { + "summary": "Remove everything a build wrote on this instance.", + "description": "The opposite of what synchronization does: `target/` is the one thing a sync will never send and never delete, which is exactly why removing it needs saying explicitly. Source is untouched, so the next build starts over without anything having to be pushed again.", + "operationId": "clean_build_output", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/credentials": { + "put": { + "summary": "Put the credentials a build needs to fetch its dependencies in place.", + "description": "Written as a `.netrc`, which is what git, cargo and the rest already know how to read, so nothing on the instance needs teaching about where its credentials come from.\n\nThese belong to whoever is synchronizing, and `vw-svc` sends them with every sync rather than once: an instance rebuilt underneath us comes back with no credentials at all, and the alternative is builds that fail to fetch until something notices.", + "operationId": "put_credentials", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Credentials" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/driver/build": { + "get": { + "summary": "Build the driver on this instance.", + "description": "A websocket because a build takes minutes and produces output the whole time, and because a developer who interrupts one should not leave cargo running on a machine nobody is watching.\n\nCargo is spawned rather than linked: the driver pins its toolchain in `rust-toolchain.toml`, which the rustup shim honours and a linked cargo would not, so linking it would quietly build a kernel module with the wrong compiler.", + "operationId": "driver_build", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "args", + "description": "Anything else to put on cargo's command line, separated by spaces.\n\nOne string because a query parameter has to be scalar. Split on whitespace at the far end, so a value containing a space cannot be expressed — no flag the driver build needs has one, and the alternative is reimplementing half a shell's quoting rules.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "release", + "description": "Build with optimizations.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + }, + "/environment/{environment}/generated": { + "post": { + "summary": "The VHDL vivado generated for this environment's IP.", + "description": "A `POST` because it finishes the job first: vivado writes an instantiation template per standalone IP, and turning those into black-box entities is a mechanical step that happens in Rust after the vivado pass. On a local run that happens on the developer's machine; on a remote one there is nobody there to do it, so it happens here, where the templates are.\n\nAnswers with paths relative to the workspace, so the far end can put each file exactly where its own tools will look for it.", + "operationId": "generated_manifest", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/generated/file": { + "get": { + "summary": "One generated file's contents.", + "operationId": "generated_file", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/object-store": { + "get": { + "summary": "The key that opens this instance's object store.", + "description": "Answered only by the instance that runs the store. The admin credential that minted this key never leaves that machine — this is the one thing it hands out, and `vw-svc` passes it to the instance that has artifacts to upload.", + "operationId": "get_object_store", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "kind", + "description": "The instance kind whose bucket is wanted. Absent means vivado, which is the one that produces images today.", + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/S3Credentials" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/sync": { + "delete": { + "summary": "Discard the source tree and everything delivered towards it.", + "description": "The instance is left as though it had never been synchronized: no source, and no record of what content it holds. Build output is untouched, as it is for a commit.\n\nThis is not part of an ordinary sync, which needs no help — a commit replaces whatever differs from the manifest. It is what a sender uses when it does not believe the instance's account of what it has, so that the sync that follows sends everything rather than asking first.\n\nAnswers with the result of committing an empty manifest, so the count of what was removed is the `deleted` field.", + "operationId": "sync_clear", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/sync/blob/{digest}": { + "put": { + "summary": "Deliver one piece of content.", + "description": "Rejected if the body does not hash to the digest in the path: the digest is how every later lookup finds this content, so storing it under a name it does not have would be worse than not storing it.", + "operationId": "sync_blob", + "parameters": [ + { + "in": "path", + "name": "digest", + "description": "The digest of the content in the body, which is verified on arrival rather than taken at face value.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/sync/commit": { + "post": { + "summary": "Make the instance's tree match the manifest.", + "description": "The manifest is the complete desired state, so this adds, replaces and removes as needed. Anything a build produced is invisible to it.", + "operationId": "sync_commit", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/sync/plan": { + "post": { + "summary": "Report what content is missing before a tree can be made to match.", + "description": "Content already held anywhere in the instance's tree is not asked for, whatever path it currently sits under, so a rename or a directory move costs nothing over the wire.", + "operationId": "sync_plan", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncPlan" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{environment}/vivado/session": { + "get": { + "summary": "Drive a vivado worker on this instance.", + "description": "A websocket rather than a request and a reply because a build is not one of those. It is a conversation that runs for minutes, produces output the whole time, and has to show that output as it happens — waiting for a synthesis run to finish before saying anything would make the remote flow useless for the thing people actually do with it.\n\nWhat crosses the socket is the protocol `vw-eda` already uses to talk to a local worker: commands in, output chunks and results back. The worker is spawned when the socket opens and torn down when it closes, so no state survives between runs — the same guarantee running vivado locally gives, and what the checkpoint machinery in the htcl library already relies on for speed.", + "operationId": "vivado_session", + "parameters": [ + { + "in": "path", + "name": "environment", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "info_with_stack", + "description": "Attach the Tcl call stack to INFO messages, not only to warnings and errors.", + "schema": { + "type": "boolean" + } + }, + { + "in": "query", + "name": "part", + "description": "`--part`, for a workspace that declares parts at the top level.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "variant", + "description": "`--variant`, for a workspace that declares variants.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "verbose", + "description": "Forward vivado's unclassified chatter rather than discarding it.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + } + }, + "components": { + "schemas": { + "CleanResult": { + "description": "What removing an instance's build output came to.", + "type": "object", + "properties": { + "bytes": { + "description": "How much space it was taking, measured before it went.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "existed": { + "description": "Whether there was any build output to remove.", + "type": "boolean" + } + }, + "required": [ + "bytes", + "existed" + ] + }, + "CommitResult": { + "description": "What applying a manifest did.", + "type": "object", + "properties": { + "created": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "deleted": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "unchanged": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "updated": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "created", + "deleted", + "unchanged", + "updated" + ] + }, + "Credentials": { + "description": "What a build needs in order to fetch its dependencies.\n\nSources are synchronized from a developer's machine, but the things a build pulls from Github are not: they are fetched by the instance itself, which therefore needs credentials for them. These are the caller's own — the same token that authorized the request that carries them — so an instance can reach exactly what its owner can reach and nothing more.\n\nNothing keeps a copy. `vw-svc` reads these off the request it is already authorizing and passes them straight through.", + "type": "object", + "properties": { + "token": { + "description": "A Github access token.", + "type": "string" + }, + "user": { + "description": "The Github login the token belongs to.", + "type": "string" + } + }, + "required": [ + "token", + "user" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "FileEntry": { + "description": "One file in a synchronized tree.", + "type": "object", + "properties": { + "digest": { + "description": "The digest of its contents.", + "type": "string" + }, + "executable": { + "description": "Whether the execute bit is set. The only mode bit that survives the trip, because it is the only one a build cares about.", + "type": "boolean" + }, + "path": { + "description": "Where the file goes, relative to the tree root, `/`-separated.", + "type": "string" + } + }, + "required": [ + "digest", + "executable", + "path" + ] + }, + "S3Credentials": { + "description": "How to reach an environment's object store, and the key that opens it.\n\nGenerated on the artifact instance and handed out from there, so the admin credential that minted it never leaves the machine that holds the store.", + "type": "object", + "properties": { + "access_key_id": { + "type": "string" + }, + "bucket": { + "description": "The bucket this environment's artifacts go in.", + "type": "string" + }, + "endpoint": { + "description": "The base URL of the S3 API, e.g. `http://172.30.0.7:3900`.\n\nComposed from [`port`](Self::port) and an address by whoever knows which address the instance is reachable on. Each side supplies what it knows: the instance cannot tell which of its addresses another machine can use, and nobody else should be guessing which port it chose.", + "type": "string" + }, + "port": { + "description": "The port the store serves S3 on, as the instance running it configured it.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "region": { + "description": "The region the store answers to. Garage's own default is `garage`, and signing fails against the wrong one.", + "type": "string" + }, + "secret_access_key": { + "type": "string" + } + }, + "required": [ + "access_key_id", + "bucket", + "endpoint", + "port", + "region", + "secret_access_key" + ] + }, + "SyncPlan": { + "description": "What the receiver still needs before a manifest can be applied.", + "type": "object", + "properties": { + "missing": { + "description": "Digests the receiver holds nowhere — neither in its content store nor anywhere in the tree it already has.\n\nContent already present under a different path is not listed: a rename or a move costs nothing, because the receiver copies it locally.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "missing" + ] + }, + "TreeManifest": { + "description": "The complete desired state of a target's source tree.\n\nComplete, not a changeset: a path absent from this is a path that should not exist, which is the only way deletions and renames can be expressed.", + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FileEntry" + } + } + }, + "required": [ + "entries" + ] + }, + "TargetKind": { + "description": "Which half of an environment a request is about.\n\nOnly the two that take source. The artifact instance holds build output and is reached as an object store, so there is nothing to synchronize to it.", + "type": "string", + "enum": [ + "vivado", + "helios" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/vw-sync-api/vw-sync-api-latest.json b/openapi/vw-sync-api/vw-sync-api-latest.json new file mode 120000 index 0000000..1861011 --- /dev/null +++ b/openapi/vw-sync-api/vw-sync-api-latest.json @@ -0,0 +1 @@ +vw-sync-api-1.0.0-b72ec8.json \ No newline at end of file diff --git a/openapi/vw-user-api/vw-user-api-1.0.0-9ba36e.json b/openapi/vw-user-api/vw-user-api-1.0.0-9ba36e.json new file mode 100644 index 0000000..4a6e999 --- /dev/null +++ b/openapi/vw-user-api/vw-user-api-1.0.0-9ba36e.json @@ -0,0 +1,1282 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "VW user API", + "description": "Manage your own vw build environments. Callers are identified by a Github access token.", + "version": "1.0.0" + }, + "paths": { + "/environment/{name}": { + "get": { + "summary": "Get an environment with the specified name.", + "operationId": "get_environment", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Environment" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Create an environment with the specified name.", + "description": "The images the environment's instances boot from are chosen here and pinned for the life of the environment. Any image named in the body must already exist.\n\nReturns the ssh keypair generated for the new environment, so a caller can save it without a second round trip. The same pair is available afterwards from `get_environment_keys`.", + "operationId": "create_environment", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnvironmentCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SshKeyPair" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete an environment with the specified name.", + "operationId": "delete_environment", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/artifacts": { + "get": { + "summary": "List the artifacts an environment's builds have produced.", + "description": "Read from the environment's own object store, which lives on its artifact instance.", + "operationId": "get_artifacts", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Artifact", + "type": "array", + "items": { + "$ref": "#/components/schemas/Artifact" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Remove every artifact an environment has stored.", + "description": "Irreversible: the object store keeps no versions, so what goes is gone. The instances themselves are untouched — a build's output is still on the machine that made it until that machine is cleaned or replaced.", + "operationId": "clear_artifacts", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactsCleared" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/artifacts/{kind}/{artifact}": { + "get": { + "summary": "Download one artifact.", + "description": "Streamed through this service rather than handed out as a link to the store. The store sits on the rack's internal network, and its instance's external address is often only reachable over a VPN — needing one to collect a build's output would make this useless from anywhere else. The body is passed through as it arrives, so an image of any size costs this service no more memory than a small one.", + "operationId": "get_artifact", + "parameters": [ + { + "in": "path", + "name": "artifact", + "description": "The artifact's file name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "kind", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/bench/session": { + "get": { + "summary": "Run an environment's testbenches on its vivado instance.", + "description": "Relayed frame for frame. What comes back is the same stream of events a local run produces, so the display on a developer's terminal is driven by exactly what would have driven it here.", + "operationId": "bench_session", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "concurrency", + "description": "How many benches run at once. Absent lets the instance decide from its own processor count, which is the number that matters — it is the machine doing the work.", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "query", + "name": "filter", + "description": "Substring match against a testbench's entity name. Absent runs all.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "ignore", + "description": "Directory names to skip while looking for benches, comma separated.\n\nOne string rather than a repeated parameter because a query parameter has to be scalar, and because that is already how `--ignore` is written on the command line.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "standard", + "description": "The VHDL standard, as `nvc` spells it.", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + }, + "/environment/{name}/driver/build": { + "get": { + "summary": "Build the driver on an environment's helios instance.", + "description": "Relayed frame for frame. The driver's target is native there and its pinned toolchain is installed there, which is the whole reason the build does not happen on a developer's machine.", + "operationId": "driver_build", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "args", + "description": "Anything else to put on cargo's command line, separated by spaces.\n\nOne string because a query parameter has to be scalar. Split on whitespace at the far end, so a value containing a space cannot be expressed — no flag the driver build needs has one, and the alternative is reimplementing half a shell's quoting rules.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "release", + "description": "Build with optimizations.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + }, + "/environment/{name}/generated": { + "post": { + "summary": "The VHDL vivado generated for this environment's IP.", + "description": "A developer's static analysis needs these to resolve the design, and they only exist where vivado ran. Relayed from the vivado instance.", + "operationId": "generated_manifest", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/generated/file": { + "get": { + "summary": "One generated file's contents.", + "operationId": "generated_file", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/keys": { + "get": { + "summary": "Fetch the ssh keypair that opens an environment's instances.", + "description": "The private key is only ever handed to the environment's owner.", + "operationId": "get_environment_keys", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SshKeyPair" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/target/{kind}/build-output": { + "delete": { + "summary": "Remove everything a build wrote on one of an environment's instances.", + "description": "`target/` is the one directory synchronization will never touch, in either direction, so it outlives every push and has to be removed on purpose. Source on the instance is left alone.", + "operationId": "clean_build_output", + "parameters": [ + { + "in": "path", + "name": "kind", + "description": "Which half of it.", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "description": "The name of the environment.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/target/{kind}/sync": { + "delete": { + "summary": "Discard an environment's source tree, so the next sync sends all of it.", + "description": "An ordinary sync does not need this: the instance is told the whole desired state and replaces whatever differs from it. This is for when what the instance says it has is itself in question — with the tree and the delivered content both gone there is nothing left to be wrong about, and the sync that follows sends every file.\n\nBuild output on the instance is not touched.", + "operationId": "sync_clear", + "parameters": [ + { + "in": "path", + "name": "kind", + "description": "Which half of it.", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "description": "The name of the environment.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/target/{kind}/sync/blob/{digest}": { + "put": { + "summary": "Deliver one piece of source content.", + "operationId": "sync_blob", + "parameters": [ + { + "in": "path", + "name": "digest", + "description": "The digest of the content in the body, verified on arrival.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "kind", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/target/{kind}/sync/commit": { + "post": { + "summary": "Make the instance's source tree match the manifest.", + "operationId": "sync_commit", + "parameters": [ + { + "in": "path", + "name": "kind", + "description": "Which half of it.", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "description": "The name of the environment.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CommitResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/target/{kind}/sync/plan": { + "post": { + "summary": "Report what source content an environment's instance still needs.", + "operationId": "sync_plan", + "parameters": [ + { + "in": "path", + "name": "kind", + "description": "Which half of it.", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetKind" + } + }, + { + "in": "path", + "name": "name", + "description": "The name of the environment.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TreeManifest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncPlan" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/environment/{name}/vivado/session": { + "get": { + "summary": "Drive a vivado worker on an environment's vivado instance.", + "description": "Relayed frame for frame to the instance, which spawns the worker when this opens and tears it down when it closes. A build is a conversation that runs for a long time and produces output throughout, so it is a websocket rather than a request and a reply — the developer sees each message as vivado emits it, exactly as they would running it locally.\n\nThe source being built is whatever the last synchronization put on the instance. Nothing is shipped over this socket.", + "operationId": "vivado_session", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of this environment to create.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "info_with_stack", + "description": "Attach the Tcl call stack to INFO messages, not only to warnings and errors.", + "schema": { + "type": "boolean" + } + }, + { + "in": "query", + "name": "part", + "description": "`--part`, for a workspace that declares parts at the top level.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "variant", + "description": "`--variant`, for a workspace that declares variants.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "verbose", + "description": "Forward vivado's unclassified chatter rather than discarding it.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "101": { + "description": "Negotiating protocol upgrade from HTTP/1.1 to WebSocket" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-websocket": {} + } + }, + "/environments": { + "get": { + "summary": "Return a list of all environments for the calling user.", + "operationId": "get_environments", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnvironmentResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Artifact": { + "description": "One artifact an environment's build produced.", + "type": "object", + "properties": { + "kind": { + "description": "Which instance built it, and so which bucket it is in.", + "allOf": [ + { + "$ref": "#/components/schemas/TargetKind" + } + ] + }, + "modified": { + "nullable": true, + "description": "When the store last accepted it, as the store reports it.", + "type": "string" + }, + "name": { + "description": "The file's name, which is its key in the bucket.", + "type": "string" + }, + "size": { + "description": "Its size in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "kind", + "name", + "size" + ] + }, + "ArtifactsCleared": { + "description": "What clearing an environment's artifacts came to.", + "type": "object", + "properties": { + "bytes": { + "description": "How much space they were taking.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "removed": { + "description": "How many objects were removed.", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "bytes", + "removed" + ] + }, + "CleanResult": { + "description": "What removing an instance's build output came to.", + "type": "object", + "properties": { + "bytes": { + "description": "How much space it was taking, measured before it went.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "existed": { + "description": "Whether there was any build output to remove.", + "type": "boolean" + } + }, + "required": [ + "bytes", + "existed" + ] + }, + "CommitResult": { + "description": "What applying a manifest did.", + "type": "object", + "properties": { + "created": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "deleted": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "unchanged": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "updated": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "created", + "deleted", + "unchanged", + "updated" + ] + }, + "Environment": { + "description": "An environment is a collection of instances that work together to build, analyze and test vw designs.", + "type": "object", + "properties": { + "artifact_instance": { + "nullable": true, + "description": "The Oxide instance id for the artifact instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + }, + "helios_instance": { + "nullable": true, + "description": "The Oxide instance id for the helios instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + }, + "images": { + "nullable": true, + "description": "The images this environment's instances boot from, chosen when the environment was created.\n\nAbsent when the service has no Oxide backend configured, in which case the environment is a bare record that will never be provisioned.", + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentImages" + } + ] + }, + "name": { + "description": "The name of this environment.", + "type": "string" + }, + "vivado_instance": { + "nullable": true, + "description": "The Oxide instance id for the vivado instance.", + "allOf": [ + { + "$ref": "#/components/schemas/OxideInstance" + } + ] + } + }, + "required": [ + "name" + ] + }, + "EnvironmentCreate": { + "description": "Body of a request to create an environment.\n\nEvery field is optional; an image left unset is resolved to the newest image the service can see whose name matches that instance kind's convention. An image named here must already exist, or the request is rejected.", + "type": "object", + "properties": { + "artifact_image": { + "nullable": true, + "description": "Name of the image the artifact instance should boot from.", + "default": null, + "type": "string" + }, + "helios_image": { + "nullable": true, + "description": "Name of the image the helios instance should boot from.", + "default": null, + "type": "string" + }, + "vivado_image": { + "nullable": true, + "description": "Name of the image the vivado instance should boot from.", + "default": null, + "type": "string" + } + } + }, + "EnvironmentImages": { + "description": "The images each of an environment's instances boots from.", + "type": "object", + "properties": { + "artifact": { + "description": "Image the artifact instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + }, + "helios": { + "description": "Image the helios instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + }, + "vivado": { + "description": "Image the vivado instance boots from.", + "allOf": [ + { + "$ref": "#/components/schemas/ImageRef" + } + ] + } + }, + "required": [ + "artifact", + "helios", + "vivado" + ] + }, + "EnvironmentResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Environment" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "FileEntry": { + "description": "One file in a synchronized tree.", + "type": "object", + "properties": { + "digest": { + "description": "The digest of its contents.", + "type": "string" + }, + "executable": { + "description": "Whether the execute bit is set. The only mode bit that survives the trip, because it is the only one a build cares about.", + "type": "boolean" + }, + "path": { + "description": "Where the file goes, relative to the tree root, `/`-separated.", + "type": "string" + } + }, + "required": [ + "digest", + "executable", + "path" + ] + }, + "ImageRef": { + "description": "An Oxide image an environment's instances are built from.\n\nPinned by id, so publishing a newer image does not silently change what an existing environment boots. The name is carried along for display.", + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] + }, + "InstanceState": { + "description": "Running state of an Instance (primarily: booted or stopped)\n\nThis typically reflects whether it's starting, running, stopping, or stopped, but also includes states related to the Instance's lifecycle\n\n
JSON schema\n\n```json { \"description\": \"Running state of an Instance (primarily: booted or stopped)\\n\\nThis typically reflects whether it's starting, running, stopping, or stopped, but also includes states related to the Instance's lifecycle\", \"oneOf\": [ { \"description\": \"The instance is being created.\", \"type\": \"string\", \"enum\": [ \"creating\" ] }, { \"description\": \"The instance is currently starting up.\", \"type\": \"string\", \"enum\": [ \"starting\" ] }, { \"description\": \"The instance is currently running.\", \"type\": \"string\", \"enum\": [ \"running\" ] }, { \"description\": \"The instance has been requested to stop and a transition to \\\"Stopped\\\" is imminent.\", \"type\": \"string\", \"enum\": [ \"stopping\" ] }, { \"description\": \"The instance is currently stopped.\", \"type\": \"string\", \"enum\": [ \"stopped\" ] }, { \"description\": \"The instance is in the process of rebooting - it will remain in the \\\"rebooting\\\" state until the VM is starting once more.\", \"type\": \"string\", \"enum\": [ \"rebooting\" ] }, { \"description\": \"The instance is in the process of migrating - it will remain in the \\\"migrating\\\" state until the migration process is complete and the destination propolis is ready to continue execution.\", \"type\": \"string\", \"enum\": [ \"migrating\" ] }, { \"description\": \"The instance is attempting to recover from a failure.\", \"type\": \"string\", \"enum\": [ \"repairing\" ] }, { \"description\": \"The instance has encountered a failure.\", \"type\": \"string\", \"enum\": [ \"failed\" ] }, { \"description\": \"The instance has been deleted.\", \"type\": \"string\", \"enum\": [ \"destroyed\" ] } ] } ```
", + "oneOf": [ + { + "description": "The instance is being created.", + "type": "string", + "enum": [ + "creating" + ] + }, + { + "description": "The instance is currently starting up.", + "type": "string", + "enum": [ + "starting" + ] + }, + { + "description": "The instance is currently running.", + "type": "string", + "enum": [ + "running" + ] + }, + { + "description": "The instance has been requested to stop and a transition to \"Stopped\" is imminent.", + "type": "string", + "enum": [ + "stopping" + ] + }, + { + "description": "The instance is currently stopped.", + "type": "string", + "enum": [ + "stopped" + ] + }, + { + "description": "The instance is in the process of rebooting - it will remain in the \"rebooting\" state until the VM is starting once more.", + "type": "string", + "enum": [ + "rebooting" + ] + }, + { + "description": "The instance is in the process of migrating - it will remain in the \"migrating\" state until the migration process is complete and the destination propolis is ready to continue execution.", + "type": "string", + "enum": [ + "migrating" + ] + }, + { + "description": "The instance is attempting to recover from a failure.", + "type": "string", + "enum": [ + "repairing" + ] + }, + { + "description": "The instance has encountered a failure.", + "type": "string", + "enum": [ + "failed" + ] + }, + { + "description": "The instance has been deleted.", + "type": "string", + "enum": [ + "destroyed" + ] + } + ] + }, + "OxideInstance": { + "description": "Information about an Oxide instance that underpins a VW instance.", + "type": "object", + "properties": { + "external_ip": { + "nullable": true, + "description": "The address to reach this instance on from outside the rack, once it has one.\n\nAbsent until the instance exists and the control plane has attached an external address to it. This is what a developer's ssh goes to.", + "type": "string", + "format": "ip" + }, + "id": { + "nullable": true, + "description": "The Oxide instance id, once the control plane has assigned one.\n\nAbsent in the window between asking for an instance and hearing back about it, which is long enough to be worth showing: an environment mid-creation reports `creating` with no id rather than looking like nothing has happened.", + "type": "string", + "format": "uuid" + }, + "internal_ip": { + "nullable": true, + "description": "The instance's address on the rack's own network.\n\nWhat `vw-svc` sends source to, rather than the external address: the internal path is a regional fabric rather than the public internet, and the difference is most of the bandwidth.", + "type": "string", + "format": "ip" + }, + "state": { + "$ref": "#/components/schemas/InstanceState" + } + }, + "required": [ + "state" + ] + }, + "SshKeyPair": { + "description": "The ssh keypair that opens an environment's instances.\n\nGenerated by the service when the environment is created and handed out only to the environment's owner.", + "type": "object", + "properties": { + "private_key": { + "description": "The private key, in OpenSSH format, ready to pass to `ssh -i`.", + "type": "string" + }, + "public_key": { + "description": "The matching public key, as it appears in an `authorized_keys` file.", + "type": "string" + } + }, + "required": [ + "private_key", + "public_key" + ] + }, + "SyncPlan": { + "description": "What the receiver still needs before a manifest can be applied.", + "type": "object", + "properties": { + "missing": { + "description": "Digests the receiver holds nowhere — neither in its content store nor anywhere in the tree it already has.\n\nContent already present under a different path is not listed: a rename or a move costs nothing, because the receiver copies it locally.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "missing" + ] + }, + "TargetKind": { + "description": "Which half of an environment a request is about.\n\nOnly the two that take source. The artifact instance holds build output and is reached as an object store, so there is nothing to synchronize to it.", + "type": "string", + "enum": [ + "vivado", + "helios" + ] + }, + "TreeManifest": { + "description": "The complete desired state of a target's source tree.\n\nComplete, not a changeset: a path absent from this is a path that should not exist, which is the only way deletions and renames can be expressed.", + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FileEntry" + } + } + }, + "required": [ + "entries" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/vw-user-api/vw-user-api-latest.json b/openapi/vw-user-api/vw-user-api-latest.json new file mode 120000 index 0000000..c5b0029 --- /dev/null +++ b/openapi/vw-user-api/vw-user-api-latest.json @@ -0,0 +1 @@ +vw-user-api-1.0.0-9ba36e.json \ No newline at end of file diff --git a/projects/htcl-project-plan.md b/projects/htcl-project-plan.md new file mode 100644 index 0000000..463d73e --- /dev/null +++ b/projects/htcl-project-plan.md @@ -0,0 +1,1736 @@ +# vw: extending for HDL workflow scripting + +## Audience and intent + +This document is a working plan for extending [`vw`](https://github.com/oxidecomputer/vw) +with first-class support for HDL workflow scripting: a structured TCL dialect +("htcl"), a workflow-aware analyzer (LSP), an interactive REPL, and a +Vivado-driving executor. The audience is Claude Code working with the +author. Treat it as a living spec — open questions are called out +explicitly; close them with the author before locking in design decisions. + +## Goal + +The underlying purpose of this work is **complexity management for HDL +designs**: making IP configuration and workflow scripting first-class +source-controlled artifacts that engineers can read, write, review, and +evolve over years. See "Strategic context" below for the full framing. +The concrete capabilities the project adds to `vw`: + +1. **Provide a best-in-class interactive experience for HDL workflow + code, in both the editor and a REPL.** This is the primary goal. + Completion, hover, diagnostics, and navigation should match what a + Rust or TypeScript developer expects from their IDE — and the same + capabilities should be available in an interactive shell that + replaces Vivado's TCL console. Both surfaces consume the same + analysis backend (`vw-htcl`), so a feature built for one is + available to the other for free. Every other language-design + decision in this document is partly in service of this goal — the + parser, the proc grammar, the module system, and the reuse of + `vw`'s dependency resolver all exist in forms designed to be + statically analyzable. +2. **Establish a unified multi-language LSP for the HDL workflow.** + `vw analyzer` is designed from day one as a multi-language language + server. htcl is the first language wired up (and the focus of v1); + VHDL is the planned second, initially via a `vhdl_ls` proxy and + eventually via direct integration with Oxide's developing VHDL + frontend. The architecture (a `LanguageBackend` abstraction and + per-file dispatch) is in place from the initial analyzer phase even + while only htcl is wired up. See "LSP design" for the full + treatment. +3. Provide an ergonomic dialect of TCL ("htcl") for HDL workflow + scripting, with first-class support for structured proc + declarations, modules, and dependencies resolved via `vw`. This + dialect is not typed in v1; the structural improvements + (per-argument doc comments, attributes like `@default` / `@enum` / + `@required`, real imports) deliver most of the value without + committing to a type system before we know what shape it should + take. +4. Execute htcl by talking to Vivado's built-in TCL interpreter over a + pipe, with a thin TCL shim on the Vivado side. +5. Stay vendor-aware: Vivado first, but the architecture should + accommodate Quartus and other backends later. + +This is an alternative to `set_property -dict {...}` bag-of-strings IP +config, ad-hoc `source [file join $::ROOT ...]` module loading, and +Vivado's generally unpleasant interpreter as a development environment. + +## Strategic context: complexity management + +The underlying problem this project addresses is that today's IP +integration workflow is intrinsically lossy with respect to source +control. Integrating IP into a design currently means reading user +guides and architecture manuals, then mapping what's learned into +GUI-based configuration in Vivado. The connection between official +documentation and GUI configuration is ambiguous and — critically — +not reproducible from an engineering-process perspective. The +artifacts that end up in source control (TCL block design exports, +generated wrappers, project files) don't capture how those +configurations were created or why specific parameterizations were +chosen. There's nowhere in the workflow to record rationale, no way to +review configuration changes the way code is reviewed, and no +mechanism to evolve a design over years and remember why it looks the +way it does. + +htcl is a complexity-management tool first, and a TCL replacement +second. Configuration becomes textual source code: reviewable, +diffable, doc-commentable, version-controlled, and analyzed by +tooling. Rationale lives next to the configuration it explains. The +artifact in source control is the authoritative record of *what was +chosen and why*, not a lossy projection of decisions made in a GUI. + +### Conceptual layering: specification, interface, instantiation + +These are three distinct things. Confusing them leads to bad design +decisions, so they're named explicitly here: + +**IP specification (IP-XACT).** Describes what an IP *is* — its full +parameter space, the parameterized port set, the parameterized memory +map, and the relationships between configuration choices and the +resulting structure. The specification is static; it covers all valid +configurations. IP-XACT is the format vendors use for this. + +**Configuration interface (htcl).** A means to invoke an IP at a +specific configuration, with human ergonomics. An htcl wrapper is a +proc whose arguments are the IP's parameters and whose body emits the +underlying `create_ip` / `set_property` calls. The htcl proc *is not* +a description of the IP — it's a means to pick a configuration and +record the rationale for that pick. Different layer entirely from +IP-XACT. + +**Instantiation (RTL + memory map).** What you get when you run a +configuration interface at chosen parameter values: a specific VHDL +entity (the wrapper Vivado generates) and a specific memory map (RSF, +in our case). These are the concrete artifacts at one configuration. + +The lifecycle is: + +1. *Specification* (IP-XACT, authored by the IP vendor): all valid + configurations and their resulting structure. +2. *Configuration interface* (htcl wrapper, generated from IP-XACT by a + sideband tool): the ergonomic surface for picking a configuration. +3. *Configuration choice* (htcl call site, hand-written by the + engineer): specific parameter values, with doc comments capturing + rationale, source-controlled and reviewed. +4. *Instantiation* (generated VHDL + RSF, produced by running htcl + through Vivado): the actual artifacts at the chosen configuration. + +htcl sits squarely at layer 2, with call sites at layer 3. It does +*not* attempt to subsume IP-XACT (layer 1) or replace generated RTL +and RSF (layer 4). The value of htcl is in giving layer 3 a +first-class, source-controlled, tool-analyzable form. + +### The Vivado-team pitch + +There is an active conversation with the Vivado team about Xilinx +publishing htcl configuration interfaces alongside the IP-XACT they +already publish. The pitch: + +> IP-XACT is your specification format and stays the source of truth. +> What's missing is a published *configuration interface* — a layer +> where engineers can record what configuration they chose and why, in +> a form that is source-controllable, reviewable, and analyzable by +> tooling. Today engineers do this in GUIs, and the resulting TCL +> dumps don't capture intent. htcl wrappers, generated from your +> IP-XACT, fill that gap. You keep the specification; we get a +> rationale-preserving configuration layer that bridges to your +> existing pipelines unchanged. + +This is a smaller, more defensible pitch than "replace IP-XACT with +htcl." htcl complements IP-XACT; it doesn't compete with it. The +showcase that earns the conversation is a set of generated htcl +wrappers that a Vivado engineer would be happy to publish — ergonomic, +documented, idiomatic. + +### IP as distributed packages + +IP configuration interfaces live in packages that vw resolves as +ordinary dependencies. A Xilinx-published `xilinx-ip` package +(generated from Xilinx's IP-XACT) contains `.htcl` wrappers for each +IP. A custom-IP repository at Oxide ships its own htcl wrappers, +generated from its own IP-XACT. A third party publishes wrappers for +their IP. Consumers add the relevant package to `vw.toml` and `src +@xilinx-ip/axis_register_slice` works the same as any other import. + +There is no IP database, no central registry, no special-case +infrastructure. Repositories of htcl distributed through vw +dependencies *are* the catalog of available configuration interfaces, +decentralized by construction. This matches how the Rust crate +ecosystem works and how Oxide's existing vw-managed VHDL dependencies +work. + +### Scope of htcl as a configuration interface + +htcl describes how to *invoke* an IP. It does not describe the IP's +ports, the IP's memory map, or the IP's parameterized structure — +those are IP-XACT's job, and the artifacts at any specific +configuration come out the other side as generated RTL and RSF. + +What an htcl wrapper proc declares: + +- Parameter names, types, defaults, constraints, and inter-parameter + dependencies — the configuration interface itself. +- Doc comments on each parameter (sourced from IP-XACT descriptions + when generated). +- Doc comments on the wrapper as a whole. + +What an htcl wrapper proc *emits* (in its body): + +- `create_ip` and `set_property` calls that hand the configuration + choice to Vivado. +- Optionally, directives that influence Vivado's wrapper generation + (see "Wrapper documentation" below). + +What an htcl wrapper proc does *not* contain: + +- Port lists. The ports of any specific instantiation come from the + generated VHDL/Verilog wrapper. The space of possible ports across + all configurations is in the IP-XACT specification. +- Memory maps. The register interface of any specific instantiation + comes from RSF generated by an IP-XACT-aware pipeline (see "RSF + generation" below). The space of possible memory maps is in the + IP-XACT specification. + +This scope is the point: htcl is small, focused on the human-ergonomic +configuration layer, and stays out of the description and +instantiation layers where IP-XACT and generated artifacts already +serve well. + +### RSF generation + +Software needs to know the register map of any IP it talks to. RSF is +Oxide's register-spec format and the natural target for this +information. + +The RSF for a specific IP instantiation is a function of two inputs: +the IP-XACT specification (which describes the parameterized memory +map) and the chosen configuration values (which pin the parameters). +The pipeline is: + +``` +IP-XACT spec + chosen parameter values --> RSF for this instance +``` + +This pipeline is not part of htcl, vw, the analyzer, or the REPL. It +is its own tool (call it `ipxact2rsf` or whatever the team names it) +that reads the IP-XACT spec, takes the chosen configuration values +(extracted from the htcl call site, or passed in directly, or queried +from Vivado post-instantiation), and emits RSF. + +What htcl contributes to this pipeline: it is the place where the +configuration values are pinned down in a source-controlled form. The +RSF generator can extract those values by reading the htcl call site, +by running the htcl through to Vivado and querying the instantiated +IP, or by both. The mechanics are for that tool to decide; htcl just +needs to ensure the configuration is recoverable. + +### Wrapper documentation + +Vivado's generated VHDL/Verilog wrappers around configured IP are +notoriously undocumented. Port semantics, parameter effects, and +intended usage patterns are absent from the wrapper file. This is a +real obstacle to using IP correctly and reviewing changes to its +configuration. + +The information flow we want: + +``` +htcl wrapper proc (doc comments on parameters, derived from IP-XACT) + --> create_ip with directives capturing those docs + --> Vivado wrapper generator (would need to honor the directives) + --> generated VHDL with carry-through documentation +``` + +The first step we control (htcl carries the docs). The last step we +want (Vivado emits documented wrappers). The middle step is the open +question: what mechanism gets the doc strings from `create_ip` +arguments into Vivado's wrapper-generation output? Possibilities +include TCL directives Vivado already supports (likely insufficient), +TCL directives Xilinx would need to add (this is what we'd pitch), or +post-processing of generated wrappers by a separate tool (works but +fragile). + +Worth pitching to Xilinx alongside htcl adoption: a documented +mechanism for `create_ip` to accept doc strings that flow into the +generated wrapper. Independently useful even for users who never adopt +htcl. + +## Why this lives in `vw` + +`vw` is already structured library-first: `vw-lib` is the core, the `vw` +CLI is a thin clap-based shim, and `vw-lib` is already consumed by +external tools (e.g., the remote build service for Vivado projects). This +is the same architectural pattern as Cargo's relationship to +rust-analyzer: one library underneath, thin CLIs on top, all tools sharing +the manifest, lockfile, and resolved dependency graph. + +Adding htcl-language support, an analyzer, and a REPL as additional +subcommands of `vw` (or as sibling binaries that share `vw-lib`) gives us: + +- **One manifest, one lockfile, one cache.** `vw.toml` and `vw.lock` + cover VHDL and htcl dependencies uniformly. Packages can ship both + languages from the same git source, versioned together (as + discussed in earlier design conversations: a package version is a + single value across all languages it contains). +- **One LSP serving both languages.** `vw analyzer` is designed from + day one as a multi-language language server. htcl is wired up + natively from the initial analyzer phase; VHDL arrives in a + subsequent phase via a `vhdl_ls` proxy, with eventual replacement + by Oxide's developing VHDL frontend. The user-facing surface (one + LSP per workspace) is stable across that transition. See "LSP + design" for the architecture. +- **Cross-language analysis.** An htcl file that wraps a user VHDL + entity and the VHDL file defining that entity live in the same + workspace, analyzed by the same tool through a shared backend + abstraction. Go-to-definition can cross language boundaries. +- **Shared dependency resolution.** No new resolver, no new fetch + logic, no new cache layout. We add an htcl file-selection layer on + top of `vw-lib`'s existing dependency model, but the resolution + mechanism is unchanged. +- **One mental model for users.** They learn `vw` once and get + everything. + +The model: `vw analyzer` is the LSP (modeled on `rust-analyzer`'s +relationship to Cargo). `vw repl` is the interactive htcl shell. `vw run` +(or similar) executes an htcl script against a Vivado worker. All of +these are thin subcommands over `vw-lib` plus a new `vw-htcl` crate that +holds the htcl-specific analysis. + +## Architecture overview + +``` + +-----------------------------+ + | vw-lib (existing) | + | - manifest / lockfile | + | - dependency resolver | + | - cache management | + | - VHDL file selection | + +--------------+--------------+ + | + +---------------------------+---------------------------+ + | | | + v v v + +----------+-----------+ +-----------+----------+ +-----------+----------+ + | vw CLI (existing) | | vw-htcl (new) | | other vw-lib users | + | vw add / update / | | - htcl parser | | (remote build svc, | + | test / etc. | | - module resolver | | future tools) | + +----------------------+ | - signature check | +----------------------+ + | - htcl -> Vivado | + | TCL emission | + +-----------+----------+ + | + +--------------------------------+--------------------------------+ + | | | + v v v + +---------+----------+ +------------+---------+ +------------+---------+ + | vw analyzer (new) | | vw repl (new) | | vw run (new) | + | LSP server, serves | | interactive htcl | | execute htcl script | + | VHDL + htcl from | | against Vivado | | against Vivado | + | one process | | worker | | worker | + +--------------------+ +----------+-----------+ +----------+-----------+ + | | + +--------------+-----------------+ + | + v wire protocol (newline- + delimited commands + + structured responses) + | + +-------------+--------------+ + | Vivado process | + | (long-lived) | + | - vivado-shim.tcl | + | (small TCL layer that | + | wraps commands and | + | emits JSON for | + | structured results) | + +----------------------------+ +``` + +Key invariants: + +- `vw-lib` is unchanged in spirit; we add new crates beside it + (`vw-htcl`, `vw-analyzer`, `vw-repl`, etc.) rather than restructuring + what exists. +- All language semantics live in Rust. The Vivado shim is a dispatcher + and serializer, nothing more. +- The wire protocol is newline-delimited commands in, structured (mixed + text/JSON) responses out. Hand-written; not RPC-framework-heavy. +- Vivado runs as a long-lived worker process. Cold-start is too expensive + for the hot path. +- The Vivado-driving binaries are self-contained; no Vivado-specific + linking. Distribution is one (or a few) Rust binaries plus the shim TCL + file. + +## Language design + +### Naming + +Working name for the dialect: `htcl`. This is the language name, used as +the file extension (`.htcl`) and as the LSP language identifier. It is not +a separate tool — htcl is a thing `vw` supports, alongside VHDL. + +Open question: final name. Avoid `tcl` in the name to reduce confusion +about it being a TCL implementation; it isn't. + +### Relationship to TCL + +htcl is **not** a TCL superset in the way TypeScript is a JS superset. +Existing Vivado TCL files are not valid htcl. The reasons: + +- We want a structured proc-argument grammar that vanilla TCL can't parse. +- We want static module imports (`src` / `use`) that aren't `source`. +- We want to reject TCL features that defeat static analysis (`upvar`, + `uplevel`, `trace`, dynamic command rewriting). + +However, htcl emits TCL when talking to Vivado, and users can drop down to raw +TCL via an escape hatch for things htcl doesn't model. Existing TCL scripts can +be `source`d through the shim if needed. + +### Proc grammar + +Procs declare structured arguments with per-argument doc comments and attributes: + +```htcl +proc axis_interface { + ## Add a TKEEP sideband signal. Indicates valid bytes in the beat. + @default(0) + has_tkeep + + ## Add a TLAST sideband signal. Indicates the last beat in a packet. + @default(1) + has_tlast + + ## Width of TDATA in bytes. + @default(8) + @enum(1, 2, 4, 8, 16, 32, 64, 128) + tdata_num_bytes + + ## Width of TUSER in bits. Only meaningful if has_tuser is set. + @default(0) + @requires(has_tuser) + tuser_width +} { + # body: emits CONFIG.* set_property calls +} +``` + +Call site uses keyword arguments: + +```htcl +set lrq_request [axis_interface -has_tkeep 1 -tdata_num_bytes 128] +``` + +Attribute set for v1 (extensible): + +- `@default(value)` — value if omitted +- `@required` — error if omitted +- `@enum(a, b, c)` — value must be one of these +- `@range(min, max)` — numeric bounds +- `@requires(other_arg)` — dependency between args +- `@conflicts(other_arg)` — mutual exclusion +- `@deprecated("message")` — soft warning + +Open question: should attributes go before or after the doc comment, or be +order-insensitive? Recommend: doc comments first, then attributes in any order, +then the argument name. Matches Rust/TypeScript conventions. + +### Module system + +Replace `source` with `src` (or `use` — see open questions): + +```htcl +src common/project # relative to current file's directory +src /opt/xilinx/lib/foo # filesystem-absolute (use sparingly, non-portable) +src @quartz/ip/bacd # named dependency, resolved via vw.toml + vw.lock +``` + +Resolution rules: + +- Leading identifier: relative to the directory of the importing file. + Subdirectory traversal allowed (`src ip/cips` is fine). +- Leading `/`: filesystem-absolute. Permitted but discouraged; lint warning + in v1, since these break across machines. +- Leading `@name/`: resolved via `vw.toml`'s `[dependencies.name]` entry. + The cached path comes from `vw-lib`'s resolution (which reads `vw.lock` + and the cache layout under `~/.vw/deps/`). The `@name` prefix means the + same thing as it does to VHDL consumers of vw — same dependency entry, + same commit, same cache directory. +- No upward traversal (`../`) in v1. Force cross-tree references to be + absolute (filesystem or named). + +Extension is implicit: `src foo/bar` resolves to `foo/bar.htcl` (or +whatever extension we settle on). Exactly one extension is recognized; +ambiguity is an error. + +Idempotence: a module is loaded at most once per interpreter run, keyed by +canonical (realpath'd) file path. Repeated `src` calls are no-ops. + +The "project root" — the base for filesystem-absolute imports' meaning of +the manifest, and the directory the analyzer walks for workspace symbols — +is the directory containing `vw.toml`. Same convention as for VHDL. + +Open question: namespace semantics. Does `src foo/bar` populate +`foo::bar::*`, or do top-level definitions land in the global namespace as +they would with `source`? Recommend: top-level definitions are scoped to +the module's namespace by default, with an explicit `export` list +controlling what's visible. Importers use bare names after `src` (the +imports of the module are pulled into the importer's namespace). This is +the bigger semantic change; if too invasive for v1, fall back to +global-namespace semantics and add scoping later. + +### Types — defer until we have usage experience + +Types are out of scope for v1, and we should be in no hurry to add them. The +proc grammar, module system, and dependency manager are the highest-value +features and don't require a type system to be useful. We need real usage +experience with htcl-without-types before we can design types that pay for +themselves. + +The risk of adding types prematurely is well-attested across other +ecosystems: type systems designed before the language has settled tend to +encode the wrong abstractions, become hard to evolve, and impose annotation +overhead that the underlying use cases don't justify. Better to live without +them, collect concrete cases where they would have caught real bugs or +documented intent better, and design to those cases later. + +If types do land eventually, they will be: + +- Optional and gradual. Unannotated code keeps working. +- Focused on HDL-specific concerns where the leverage is clear (likely + candidates: IP handle types so CONFIG.* completion works without flow + analysis, units like Hz/MHz/ns, bit widths). General-purpose typing of + TCL values is not the goal. +- Driven by accumulated evidence, not speculative design. + +The LSP's type-directed CONFIG.* completion (see LSP design section) is +achieved in v1 via flow tracking of IP handles, not a user-facing type +system. The user writes `set lrq [create_ip -name axis_register_slice ...]` +with no annotations; the analyzer infers that `$lrq` holds an +`axis_register_slice` handle. This is a narrow, internal analysis — not a +type system users interact with — and it covers the headline IP-completion +case without committing to broader type design. + +## Dependency management + +Dependency management is `vw-lib`. We do not design a new manifest, a new +lockfile, a new resolver, or a new cache. We extend the existing model in +the minimal ways htcl needs. + +### Manifest and lockfile + +`vw.toml` and `vw.lock`, as they exist today. Reference: +[vw README](https://github.com/oxidecomputer/vw/blob/main/README.md). + +Each dependency entry already specifies `repo` + `branch`/`commit`/`tag` +and a `src` selector for VHDL files. For htcl consumption, we add an +optional `htcl` selector alongside `src`: + +```toml +[dependencies.quartz] +repo = "https://github.com/oxidecomputer/quartz" +branch = "main" +src = "hdl/ip/vhd" # VHDL files (existing; consumed by VHDL flow) +htcl = "hdl/ip/htcl" # htcl files (new; consumed by vw-htcl) +recursive = true +``` + +Either or both selectors may be present. A package that ships only htcl +omits `src`; a package that ships only VHDL omits `htcl`; a package that +ships both (the common case for shared IP wrappers) has both. + +Open question: whether to keep them as separate keys (`src` / `htcl`) or +generalize to a polymorphic selector keyed by file type. The separate-keys +version is the smallest change to vw-lib and the most explicit; keep it +unless there's a reason to generalize. If we later add other languages +(SystemVerilog, Quartus TCL), we add more keys. + +### Versioning model (unchanged from vw) + +- A package version is a single value across all languages it contains. + VHDL and htcl contents of `quartz` ship together at the same commit; no + per-language versions. +- `vw.lock` pins exact commit SHAs, not tags. + +This was previously called out as a design decision; it is, but it's +already the model vw uses, so there's nothing to design. + +### Resolution and fetch (unchanged from vw) + +- `vw update` fetches and locks. Same command, same semantics, now also + resolves htcl dependencies if `htcl` selectors are present. +- Cache at `~/.vw/deps/-/`, unchanged. +- Authentication defers to git, unchanged. + +### What `vw-htcl` does on top of `vw-lib` + +A thin file-selection and module-resolution layer: + +- Given a resolved dependency from `vw-lib`, find the htcl files within + it using the `htcl` selector. Same selector semantics as the existing + `src` field (directory / single file / glob). +- Build an index from `@name` to the resolved dependency's htcl root + directory. +- The htcl module resolver consults this index when it sees `src + @name/path`. + +This is the only dependency-related code htcl needs to write. Everything +upstream of "I have a cache directory for `@quartz`" is already done. + +### Path dependencies + +`vw` currently supports git-sourced dependencies. For monorepo/sibling- +checkout development loops, htcl's plan called for `path = "../foo"` +dependencies. Confirm whether `vw-lib` currently supports this; if not, +adding it is a small extension that benefits both VHDL and htcl +consumers. Likely worth doing in `vw-lib` itself rather than as +htcl-specific behavior. + +### Coordination questions for `vw-lib` + +Things to confirm before phase 1 implementation, and possibly upstream +changes to schedule: + +1. **Does `vw-lib` expose a stable Rust API for "give me the resolved + path for dependency X"?** Yes (per the existing remote-build-service + consumer), but confirm the exact shape — a `Resolved { name, commit, + path, src_files }` struct, or similar — so the htcl resolver can + consume it cleanly. +2. **Is the `src`-selector logic factored well enough to reuse for `htcl` + selectors?** Ideally yes; the directory / single-file / glob handling + is generic and shouldn't be reimplemented. +3. **Path dependencies:** present, absent, or in progress? +4. **Workspace concept:** if `vw.toml` ever grows multi-package workspace + support (cargo-style), how does that interact with htcl? Likely fine, + but worth a thought. + +## Wire protocol + +### Transport + +Vivado is spawned once per workflow run (or per LSP session) and stays alive. +htcl talks to it over stdin/stdout pipes. No sockets, no daemon, no +multi-user complexity in v1. + +### Request format + +Newline-delimited commands. Each command is a JSON object: + +```json +{"id": 42, "op": "eval", "tcl": "set_property -dict {CONFIG.HAS_TKEEP 1} $lrq"} +{"id": 43, "op": "eval_structured", "tcl": "report_property -all $cell"} +``` + +`id` is a monotonic request ID for matching responses. + +Two ops in v1: + +- `eval`: run the TCL, return the result as a string (or error info). +- `eval_structured`: run the TCL through a wrapper that emits JSON for known- + structured commands. The shim has a dispatch table from command name to + wrapper function. + +### Response format + +```json +{"id": 42, "ok": true, "result": ""} +{"id": 43, "ok": true, "result": {"CONFIG.HAS_TKEEP": "1", ...}} +{"id": 44, "ok": false, "error": {"message": "...", "code": "...", "info": "..."}} +``` + +For `eval_structured`, `result` is the JSON shape produced by the per-command +wrapper. For `eval`, it's a string. + +### Vivado-side shim + +A small TCL file (`vivado-shim.tcl`) loaded into Vivado at worker startup. It: + +- Reads newline-delimited JSON from stdin. +- Dispatches to a handler per `op`. +- For `eval`, calls `uplevel #0 $tcl`, captures result or error, emits + response. +- For `eval_structured`, parses the command name from the TCL, looks up a + wrapper, runs the wrapper to produce JSON. +- Wrappers are hand-written per command family. Start with: `report_property`, + `report_timing` (summary), `get_cells` / `get_pins` / `get_nets` / + `get_clocks` lists, `list_property`. Grow as needed. + +Most commands don't need a structured wrapper. Default to passthrough as a +string; opt into structure for the commands where text parsing on the Rust +side would be painful. + +### Batching and fencing + +v1: one request, one response. No batching. If round-trip latency becomes a +measured bottleneck, add a `batch` op that runs a list of commands with +fence semantics (stop on first error, or run-all-collect-errors). + +## LSP design + +LSP is a first-class concern, not a feature bolted on late. This +section describes the design in detail because LSP quality is the primary +goal of the project, and because the rest of the language design either +serves the LSP or constrains it. + +### Scope: a multi-language LSP, htcl first, VHDL next + +`vw analyzer` is designed from the start as a multi-language LSP for +the entire HDL workflow, not as an htcl-only LSP that might grow VHDL +later. The bulk of the initial implementation focus is htcl — that's +where the new language design and the headline complexity-management +features live — but the LSP server architecture, the workspace model, +and the configuration shape all assume a multi-language future and +must accommodate it from day one. + +The motivating reality: + +- Oxide's HDL work spans both htcl (this project) and VHDL (extensive + existing codebase, the larger of the two by volume). +- Today, VHDL editor support comes from `vhdl_ls`, the open-source + VHDL language server, configured via `vhdl_ls.toml` that `vw` + generates. This works well and isn't going anywhere short-term. +- Long-term, Oxide is developing its own VHDL frontend and synthesizer + as part of a complete VHDL stack. The eventual goal is for `vw + analyzer` to integrate directly with that frontend, replacing + `vhdl_ls`. + +The path: `vw analyzer` serves htcl natively from day one, and +provides VHDL support initially by proxying to `vhdl_ls`, eventually +by integrating with the Oxide VHDL frontend. The user-facing surface +(one LSP serving both languages from a unified `vw.toml` workspace) +is stable across that transition; the implementation under the hood +changes. + +Two consequences for the htcl-focused work in this plan: + +1. The LSP server architecture is multi-language from phase 3, even + while only htcl is wired up. The language-backend abstraction + exists; it has exactly one implementation initially. +2. The htcl-side analysis must surface cross-language queries through + that abstraction rather than directly to a specific VHDL + implementation. This keeps the abstraction honest from day one and + means the eventual swap from `vhdl_ls` to the Oxide frontend + doesn't ripple into htcl-side code. + +### Architectural principle: one source of truth per language + +The LSP is **not** a separate analyzer with its own parser and signature +checker. It is the same code as the CLI, exposed over the LSP protocol. +For htcl, the parser, name resolver, signature checker, and +(eventually) any type analysis are written once and used by both `vw +run` / `vw check` and `vw analyzer`. For VHDL (later), the same +discipline applies via whichever backend serves VHDL at the time. + +This matters because the dominant failure mode of language tooling is +divergence between "what the compiler does" and "what the IDE shows." +The moment they're separate implementations, they drift, and users +learn not to trust the IDE. Sharing the implementation is the only +durable fix. + +Concretely: + +- All htcl semantic analysis lives in the `vw-htcl` crate, consumed by + every subcommand that needs it (`vw run`, `vw check`, `vw analyzer`, + `vw repl`). +- VHDL analysis lives behind a language-backend abstraction (see + below). Initially: `vhdl_ls` proxy. Later: direct integration with + the Oxide VHDL frontend. htcl-side code talks to this abstraction + for cross-language queries, never to a specific VHDL implementation. +- `vw analyzer` is the LSP server process. It does protocol plumbing + (LSP request/response, text-sync, capability negotiation), dispatch + by file type, and cross-language coordination. It contains minimal + language logic of its own. +- `vw check` runs the same analyses the LSP runs on save, with the + same diagnostics. CI uses `vw check`; editors use `vw analyzer`. + Same diagnostics either way. + +### Language backend abstraction + +The LSP server dispatches per-file based on extension and routes +requests to a language backend. Each backend implements a common +trait (working name `LanguageBackend`): + +```rust +trait LanguageBackend { + fn diagnostics(&self, file: FileId) -> Vec; + fn hover(&self, file: FileId, pos: Position) -> Option; + fn completion(&self, file: FileId, pos: Position) -> Vec; + fn definition(&self, file: FileId, pos: Position) -> Vec; + fn document_symbols(&self, file: FileId) -> Vec; + // ... cross-language query surface, see below ... + fn find_symbol(&self, query: SymbolQuery) -> Vec; +} +``` + +(Exact shape is for implementation to decide; the point is the +abstraction exists from day one.) + +Initial implementations: + +- `HtclBackend` — uses `vw-htcl` directly. Native, in-process. +- `VhdlBackend` (initial) — `vhdl_ls` proxy. Spawns `vhdl_ls` as a + subprocess and forwards file-scoped requests over the standard LSP + protocol. The proxy generates `vhdl_ls.toml` from `vw.toml` (which + `vw` already does for standalone editor support) and points the + subprocess at it. + +Later, the `VhdlBackend` is replaced by a direct integration with the +Oxide VHDL frontend — same trait, different implementation. Call +sites in `vw analyzer` and in `HtclBackend` don't change. + +The cross-language query surface (`find_symbol` above, plus whatever +else accumulates as cross-language features grow) is the contract +that lets htcl ask "is there a VHDL entity named X?" without caring +which backend answers. For the `vhdl_ls` proxy, `find_symbol` is +implemented by querying `vhdl_ls` with `workspace/symbol` and +translating the results. For the Oxide frontend, `find_symbol` is a +direct API call. Same shape from the htcl side. + +### Cross-language analysis (htcl ↔ VHDL) + +A different case from the IP-XACT-generated wrapper flow described in +Strategic Context: hand-written htcl that wraps user VHDL entities. A +team's own VHDL design has entities with generics, and an htcl proc +gives those entities ergonomic instantiation interfaces with doc +comments, defaults, and validation. Because `vw analyzer` sees both +languages (through the backend abstraction), it can offer +cross-language navigation between the htcl wrapper and the underlying +VHDL entity: + +- **Go-to-definition from htcl into VHDL.** An htcl proc that wraps a + VHDL entity — e.g., `instantiate_uart` taking parameters that map + to the `uart` entity's generics — can declare its target entity via + an attribute (likely `@vhdl_entity(uart)`). When the user invokes + go-to-definition on the proc call, the LSP server asks the VHDL + backend for the location of entity `uart` and returns it. The + htcl-side code that issues this query doesn't know whether + `vhdl_ls` or the Oxide frontend answered. +- **Find references across languages.** "Find references" on a VHDL + entity surfaces both VHDL instantiations (from the VHDL backend's + references query) and htcl wrapper procs that target it (from the + htcl backend's index of `@vhdl_entity` attributes). +- **Generic-to-argument mapping.** If an htcl wrapper declares which + of its proc arguments map to which VHDL generics, the htcl backend + queries the VHDL backend for the entity's generic list and checks + for missing or extra mappings. Warns on drift when the entity's + generic list changes. + +Note: this is distinct from IP-XACT-generated wrappers. Those target +Vivado IP via `create_ip` and don't have a VHDL entity in the +workspace to navigate to — the entity comes out the other side as +generated RTL. Cross-language analysis applies to user-authored +htcl-over-VHDL, not to vendor-IP wrappers. + +Open question: how aggressively to pursue cross-language features in +v1. The minimum is "go-to-definition from htcl into VHDL"; the rest +is nice to have. I'd recommend the minimum lands in v1 (it's the +headline demo for the multi-language model) and the more +sophisticated checks come after. + +### Incremental analysis + +A useful LSP must re-analyze on every keystroke. The analysis layer is +designed for this from the start, not retrofitted. + +Approach: + +- The unit of caching is the file (module). Parsing a file produces a syntax + tree; resolving its imports produces a module-level binding. Files cache + their parsed and resolved state, keyed by content hash. +- Cross-module analysis (resolving an import, looking up an external proc + signature) reads from the cache. A change to a file invalidates that file + and any file that depends on it, transitively. +- Consider `salsa` (the framework rust-analyzer uses) for memoization and + invalidation. It's heavy machinery for a small project, but it solves + exactly this problem and the alternative is hand-rolling the same thing + badly. Decide after phase 0 whether to adopt it; for the smallest possible + v1, a hand-rolled cache keyed on file mtimes is fine. +- Parsing should be tolerant of incomplete input. The user is in the middle + of typing; the parser must produce a usable AST with error nodes rather + than bailing on the first syntax error. This shapes the parser choice + (see below). + +### Parser + +The parser is the foundation of every LSP feature. Built with +[`winnow`](https://docs.rs/winnow), the parser library used pervasively +across Oxide. Familiarity, code-review consistency, and shared idioms with +the rest of the codebase outweigh case-by-case evaluation of alternatives. + +Requirements the implementation must meet within winnow: + +- **Error-tolerant.** Recover from syntax errors and continue parsing. A + half-typed proc declaration should still produce a tree where the rest + of the file is analyzable. Winnow's `cut_err` and combinator-level + recovery are the building blocks; design recovery points around + statement boundaries (newline-terminated top-level forms, proc bodies, + `src` statements). +- **Position-preserving.** Every node knows its source span. Winnow's + `Located` adapter or equivalent span-tracking is used throughout. No + AST node without a span. +- **CST-shaped, trivia-preserving.** The output is a concrete syntax tree + that retains whitespace and comments, not a stripped abstract syntax + tree. We need comments for doc-comment extraction and trivia for + accurate formatting (`vw fmt` is a likely future feature). The AST + layer used by name resolution and signature checking is derived from + the CST. +- **Reusable across editor and CLI.** Same parser code runs in `vw run`, + `vw check`, `vw analyzer`, `vw repl`. No editor-only or CLI-only + variants. + +Incremental reparse is deferred. Most htcl files will be small enough +that full reparse per edit is fast; revisit if measurement shows +otherwise. If incremental parsing becomes necessary later, the CST +boundary makes it tractable to swap in a different strategy for hot +paths without rewriting downstream analysis. + +Open question: how to structure the CST → AST lowering. Two reasonable +shapes: (a) a single AST with optional trivia attached to nodes, or (b) +a separate AST that holds references back into the CST for source +positions. Pick after the first non-trivial grammar pass. + +### Feature inventory + +Each feature has acceptance criteria specific enough to be implementable. + +#### Completion + +What completion offers depends on cursor context. The completion system +needs a notion of "what kind of position is this," determined by the +surrounding syntax tree. + +Positions and their completion sets: + +- **Top-level statement.** Suggest: keywords (`proc`, `src`, `set`, control + flow), in-scope procs, in-scope variables. +- **After `src `.** Suggest: relative module names (subdirectories and + `.htcl` files reachable from the current file), `/` to start a filesystem + path, `@name/` for declared dependencies. After `@name/`, suggest paths + within that dependency. +- **Command position (start of a statement).** Suggest in-scope procs and + Vivado builtins. For Vivado builtins, the suggestion source is a + generated table from UG835 (see "Vivado builtins" below). +- **Argument position of a known proc call.** If the cursor is after + `axis_interface ` and the next token is `-`, suggest the proc's keyword + arguments. If the cursor is after `-has_tkeep `, suggest values + appropriate to that argument's type / `@enum` set. +- **Inside a `$variable` reference.** Suggest in-scope variable names. +- **Inside an attribute (`@`).** Suggest known attribute names + (`@default`, `@required`, `@enum`, etc.) and, where appropriate, + their arguments. + +Note: parameter completion on IP instantiation sites (the +highest-value HDL use case) is the same code path as proc-argument +completion above. An IP wrapper is an htcl proc; its parameters are +proc arguments with attributes; completion works the same way as for +any other proc. There is no special-case "IP property" completion. + +Acceptance criteria: +- Completion responds in <50ms for files under 1000 lines. +- Completion items include `detail` (short type/signature info) and + `documentation` (full doc comment) fields. +- Snippets supported for procs with required arguments — completing a proc + call inserts the proc name plus placeholders for required keyword args. + +#### Hover + +Acceptance criteria: +- Hovering on a proc name shows the proc's doc comment, signature + (arguments with their attributes), and source location. +- Hovering on a proc argument at a call site shows that argument's doc + comment, default value, and any attributes (`@enum`, `@range`, etc.). +- Hovering on a `src` import shows the resolved file path and, if + available, the module's top-level doc comment. +- Hovering on a Vivado builtin shows UG835-derived documentation. +- Hovering on an IP wrapper proc (imported from a vw package) shows + its doc comment and per-parameter docs, same as any other proc. If + the package was generated from IP-XACT, that documentation flows + through unchanged. + +#### Diagnostics + +Diagnostics are produced by the same analyzer the CLI uses. The LSP just +ships them over the wire. + +Categories: + +- **Syntax errors.** Parse failures, recovered to the best position the + parser can manage. +- **Unresolved imports.** `src foo/bar` where `foo/bar.htcl` doesn't exist. +- **Unknown procs.** Call to a name that isn't defined or imported. +- **Argument errors.** Unknown keyword argument, missing required argument, + value outside `@enum` or `@range`, `@requires` / `@conflicts` violation. +- **Unused declarations.** Unused imports, unused local variables. Warning + level, suppressible. +- **Deprecation warnings.** Call sites of procs marked `@deprecated`. + +Note: there is no separate "IP property error" diagnostic category. +Unknown arguments to an IP wrapper proc are caught by the standard +"unknown keyword argument" check; out-of-range values are caught by +the standard `@enum` / `@range` check. The diagnostics machinery +doesn't distinguish IP wrappers from other procs. + +Each diagnostic has: source range, severity, message, optional related +information (e.g., "this is the proc declaration whose required argument +you're missing"), optional code action (e.g., "add missing argument"). + +Acceptance criteria: +- Diagnostics update within 200ms of an edit. +- Every diagnostic has a precise source range, not just a line number. +- Diagnostics are stable: editing an unrelated part of a file doesn't + cause diagnostics elsewhere to flicker. + +#### Go-to-definition + +- **Proc reference → proc declaration.** Across files, following imports. +- **Variable reference → assignment.** Within a scope; "definition" for a + variable is its first assignment in the current scope or a containing + scope. +- **`src` target → the imported file.** Open the imported `.htcl` file. +- **Vivado builtin → UG835 entry.** Either open a generated stub file with + the documentation, or open the UG835 URL. Implementation-defined; the + point is the user can find the docs. + +#### Find references + +- For procs, find all call sites and any explicit references (passing as a + value, etc.). +- For variables, find all reads and writes in scope. +- For modules, find all `src` statements that import them. + +Acceptance criteria: +- Find references on a proc returns results across the whole project, + searching all `.htcl` files transitively reachable from the project root. +- Results include the source range and one line of context. + +#### Rename + +Lowest priority but high value when it works. Rename a proc, variable, or +module and update all references atomically. + +Caveats: +- Renaming across the project boundary (into dependencies) is forbidden. +- Renaming requires the LSP to be confident about every reference; if any + reference is ambiguous (e.g., dynamically constructed), abort with an + error rather than rename incorrectly. + +#### Document symbols and workspace symbols + +- Document symbols: every proc, top-level variable, and module-level + declaration in the current file. Used for the editor's outline view. +- Workspace symbols: same, across the project. Used for "go to symbol in + project" pickers. + +#### Code actions + +A small set in v1, expanded over time: + +- "Add missing required argument." +- "Remove unused import." +- "Convert raw `set_property -dict` to a structured IP configuration call." + (Big one for migration off existing Vivado TCL.) +- "Extract selection to proc." + +#### Formatting + +`htcl fmt` is a separate CLI command; the LSP exposes it via the +`textDocument/formatting` request. Formatter implementation is a phase past +v1, but the architectural slot for it should exist from the start (the CST +must preserve enough information to reformat). + +### Configuration completion on IP instances + +The highest-impact LSP feature for HDL work is parameter completion at +IP instantiation sites. The user imports an IP from a vw dependency: + +```htcl +src @xilinx-ip/axis_register_slice +# ... +set lrq [axis_register_slice -has_tkeep 1 -tdata_num_bytes | + ^cursor here +``` + +The analyzer offers completion for the IP's parameters (`tuser_width`, +`tdest_width`, etc.), with hover documentation, defaults, and `@enum` +constraint values pulled from the proc's declared signature. + +Crucially: **this is the same code path as completion on any other +htcl proc.** The IP wrapper is a proc; the proc has structured +arguments with attributes (per the proc grammar); completion of those +arguments works the same as completion of arguments on a hand-written +proc. There is no special-case "IP property" subsystem in the +analyzer. + +This is the architectural payoff of treating IP as ordinary htcl +packages distributed through vw dependencies: the LSP doesn't need to +know anything about IP-XACT, IP catalogs, or Vivado-specific +introspection. It just analyzes htcl. + +Flow tracking of IP handles is still useful for downstream features — +"this variable is an instance of `axis_register_slice`, so when it's +passed to `connect_axis`, here's what we can validate" — but it's a +narrow extension of proc return-type tracking, not a separate +mechanism for IP. Defer until the cross-IP wiring story matures. + +### Vivado builtins + +htcl needs knowledge of Vivado's built-in TCL commands (`get_cells`, +`report_timing`, `current_design`, etc.) to provide completion and +hover for them. These aren't IP — they're the underlying Vivado +language surface htcl sits on top of. + +Sources: + +- UG835 (the Tcl Command Reference) parsed into a structured form. The + doc has consistent enough structure to be machine-readable, though + it's not trivial. +- `help ` output from a live Vivado, scraped at + builtin-data-generation time. +- Hand-written annotations layered on top for things UG835 gets wrong + or doesn't explain. + +The result ships with vw (in a `vw-vivado-data` crate, or similar) as +a generated data file consumed by the analyzer. Regenerated per Vivado +release; the version targeted in `vw.toml` selects which data file is +used. + +### LSP server implementation + +- Crate: `vw-analyzer`, a binary crate. Invoked as `vw analyzer` from + the `vw` CLI dispatcher, or directly via `vw-analyzer`. +- Framework: `tower-lsp` is the standard Rust LSP framework, + well-maintained and used by rust-analyzer-adjacent projects. Use it + unless there's a specific reason not to. +- Transport: stdio. The editor spawns `vw analyzer` and talks to it. +- Concurrency: file analysis runs on a worker thread pool; the protocol + handler thread stays responsive to cancellation requests. + Long-running analyses (full project re-resolution) are cancellable. +- Language scope: htcl and VHDL in one server process, dispatched + per-file via the `LanguageBackend` abstraction (see "Language + backend abstraction" above). htcl is wired up natively from phase 3; + VHDL is wired up via the `vhdl_ls` proxy in a subsequent phase, with + eventual replacement by the Oxide VHDL frontend. Cross-language + queries are first-class through the backend trait. + +### Editor integration + +VS Code is the primary target. A minimal extension: + +- Activates on `.htcl` and `.vhd`/`.vhdl` files, and on workspaces + containing `vw.toml`. +- Spawns `vw analyzer` as the language server. +- Ships a TextMate grammar for htcl syntax highlighting (VS Code's + native highlighting format). A tree-sitter grammar can come later if + we want to support editors that consume those directly (Zed, Neovim + with `nvim-treesitter`). The editor-side highlighting grammar is + separate from the LSP parser; the LSP uses winnow, while highlighting + is whatever the editor consumes. +- Provides commands: "vw: restart analyzer," "vw: update dependencies," + "vw: show IP property reference." + +Open question: relationship to any existing vw VS Code extension. If one +exists, extend it; if not, this is a new package. Either way, one +extension per project, not separate VHDL and htcl extensions. + +Other editors (Neovim, Emacs, Helix, Zed) get LSP support for free via +their generic LSP clients; we don't ship extensions for them initially, +but configuration snippets in the README are a low-cost way to support +them. + +### LSP testing strategy + +LSP regressions are easy to ship and hard to notice. Test infrastructure +from the start: + +- Snapshot tests for analysis output. Each test is a small `.htcl` fixture; + the expected output (diagnostics, symbol tables, completions at marked + positions) is a checked-in snapshot. Mismatches fail the test. +- End-to-end LSP tests using a test client that speaks the protocol. Verify + that a `textDocument/completion` request at a given position returns the + expected set of items. +- Don't test through VS Code. Test the LSP server directly; the VS Code + extension is a thin enough wrapper that manual smoke-testing is fine for + it. + +### Phasing within the LSP work + +The analyzer is built incrementally alongside the rest of the language, +with `vw analyzer` introduced as a real subcommand at phase 3 and +growing features as later phases land: + +- **Phase 3 (analyzer initial, htcl only):** `vw analyzer` binary + exists. `LanguageBackend` abstraction in place with `HtclBackend` as + the sole implementation. Provides diagnostics, document symbols, + go-to-definition for `src` targets and proc references, hover for + proc docs, completion for proc arguments. Crucially, this includes + parameter completion on IP wrapper procs imported from vw packages + — the headline IP-completion case falls out of the proc-argument + completion path. This is the point where the analyzer is genuinely + useful for htcl. +- **Phase 4 (structured wire responses):** No direct analyzer impact, + but enables typed result handling that the REPL builds on. +- **Phase 5 (VHDL via vhdl_ls proxy):** `VhdlBackend` lands as a + proxy to `vhdl_ls`. `vw analyzer` now serves both languages from a + single process; the user-facing multi-language LSP surface is in + place. +- **Phase 6 (cross-language):** htcl ↔ VHDL go-to-definition and find + references, building on the backends from phase 5. +- **Phase 8 (polish):** Find references across the workspace, rename, + workspace symbols, code actions, performance tuning, editor + extension packaging. + +Note: the analyzer benefits from being built alongside language +features rather than after them. Each language feature (modules, proc +grammar, cross-language) lands its analyzer support in the same phase +that introduces the feature. The "phase 8 polish" pass is for +analyzer-only features that don't have an underlying-language +counterpart. + +The eventual replacement of the `vhdl_ls` proxy with direct Oxide VHDL +frontend integration is a "Later" item (see implementation plan). +Because the swap stays within the `LanguageBackend` abstraction, +no htcl-side code needs to change when it happens. + +### Non-goals for the LSP + +- Debugging protocol (DAP). Out of scope; debugging happens inside Vivado. +- Semantic tokens for syntax highlighting. Editor-side grammars are cheaper + and good enough. +- Inlay hints. Possibly later; not needed for v1. +- Refactorings beyond rename and the small code-action set listed above. + +## REPL design + +The REPL is, architecturally, the same product as the analyzer with a +different presentation layer. The LSP serves an editor; the REPL serves +a TUI. Both query the same `vw-htcl` analysis. This isn't a coincidence +to exploit — it's the design. + +The traditional captive-CLI help model (Vivado's `help foo`, Cisco IOS's +`?`) was a 1990s answer to "I don't have a graphics-capable terminal but +I have screen-clearing escape codes." It conflates discovery (what +exists?), reference (what does this do?), and navigation (where am I?) +into a single text-dump idiom that clutters scrollback and answers none +of those questions well. With a modern TUI and the analyzer's data +already on hand, we can do substantially better without reimplementing +anything. + +Built with [`ratatui`](https://ratatui.rs/), the TUI library used +pervasively across Oxide. Line editing uses +[`reedline`](https://docs.rs/reedline) — Nushell's modern readline +replacement, well-suited to hint and menu rendering. Both choices are +Oxide-conventional rather than case-evaluated; same reasoning as winnow. + +### Architectural principle: history hygiene + +The defining UX commitment: anything the user explicitly ran (commands +and their results) belongs in scrollback. Anything that was a navigation +aid (completion menus, signature help, hover dialogs, help overlays) +does not. Navigation aids appear in transient overlays or inline +ghost-text and disappear when the user moves on. The scrollback is what +the user chose to do, not how they figured out what to do. + +This is the failure mode of Vivado's REPL: a `help` invocation dumps +200 lines into history, and the user's actual work is buried. Ours +won't. + +### Virtual document model + +The REPL maintains an in-memory document representing the session: +successful evaluations are appended, and the current input line is +treated as the tail. The analyzer's queries operate on (document + +current input), with the cursor positioned within the current input. + +Consequences: + +- Variables and procs defined earlier in the session are in scope for + completion, hover, and diagnostics on the current input. +- Diagnostics on the current line surface *before* the user submits it + — a typo'd argument name is flagged inline, not after Vivado returns + an error. +- Sourced modules contribute their definitions to the session document, + so completion includes everything reachable from the import graph. + +The same analyzer code that powers `vw analyzer`'s editor support +powers the REPL's interactive features. The analyzer doesn't know +whether it's serving an editor or a TUI. + +### Feature inventory + +#### Tab completion + +The primary discovery mechanism. Triggered on Tab, optionally +auto-suggested as a menu after a short typing pause. + +Completion sources match the LSP's: in-scope procs, proc arguments at +call sites (including arguments on IP wrapper procs imported from vw +packages), `@enum` values, variable names, module imports, and Vivado +builtins. + +Rendered as a popup menu *below* the input line. Arrow keys navigate; +Enter or Tab accepts; Escape dismisses. The menu does not enter +scrollback. + +#### Signature help while typing + +When the user is partway through a proc call, a non-intrusive line +*below* the input shows the proc's signature with the current argument +highlighted. The current value's `@enum` or `@range` constraint is +shown alongside. + +The signature line updates as the user types and disappears as soon as +they move past the call. Never enters scrollback. + +#### Modal help overlay + +Bound to F1 (or `?` if a more keyboard-friendly approach is preferred). +Pops a transient overlay — a centered panel or split-pane — with the +full documentation for the symbol under the cursor: proc signature, +all argument docs, attribute constraints, source location, related +procs. For IP wrappers, this surfaces the same documentation that's in +the proc's doc comments and parameter attributes — which (for +IP-XACT-sourced packages) carries the IP-XACT descriptions through to +the user. + +Dismissed with Escape. Nothing lands in scrollback. + +This is what Vivado's `help` command should have been: a brief takeover +of the screen that returns control unchanged. + +#### Inline ghost-text suggestions + +As the user types, faintly render the most likely completion in dim +text ahead of the cursor (fish-shell / Copilot style). Right-arrow or +Tab accepts. Any other key dismisses. + +For HDL workflows where proc and argument names are long and +repetitive (`tdata_num_bytes`, `axis_register_slice`), this saves real +keystrokes. Reedline supports this natively. + +#### Discoverable command palette + +Bound to Ctrl-P (or similar). Opens a fuzzy-searchable overlay listing +in-scope procs, recent commands, and (optionally) workspace symbols. +Same data the LSP uses for `workspace/symbol`. + +The failure mode of current Vivado REPLs is "I know I want to do X but +I don't remember what it's called." This is the fix. + +#### Per-instance exploration + +Dedicated mode for the common HDL workflow question: "I have an IP +instance; what are all its current properties and values in the live +Vivado design?" Triggered by `:describe ` or by a hotkey on a +variable in scope. + +Renders a sortable, filterable table built from live `report_property` +data on the instance, joined with the IP wrapper proc's parameter +documentation (so each property has its description and constraints +visible). Navigable with arrow keys; Enter on a property opens its +full documentation. Escape closes. + +Substantially better than `report_property` dumped into scrollback as +text. + +#### Pretty-printed structured results + +When `eval_structured` (phase 4) returns a typed result, the REPL +renders it as a navigable structure rather than a flat string. Timing +reports become collapsible trees; property dumps become tables; lists +of cells/pins/nets become selectable lists where each entry can be +hovered for details. + +The text representation is still available — `:plain` or a config +option turns off pretty-printing for screencasts and pipe-friendly +output. + +#### Lightweight text help fallback + +A `:help foo` command (or similar) prints help to scrollback as plain +text. Useful for SSH over slow links, grepping history, screencasts, +and copying into chat/issues. The TUI overlay is the default for +interactive use; the text command exists for cases where the overlay +isn't what's wanted. + +This is the one place we accept scrollback clutter, because it's the +user explicitly asking for it. + +### Implementation notes + +**Debouncing.** Analysis runs on keystrokes; if it takes more than +~20ms the UI feels laggy. Debounce completion and hover queries (fire +~50ms after input stability), and run analysis on a worker thread that +the ratatui frame loop polls. + +**Cancellation.** A new keystroke invalidates the previous analysis +request. The analyzer is already cancellable (LSP requirement); the +REPL inherits that. + +**History.** Persistent across sessions, stored in +`~/.local/state/vw/repl-history` (or platform-equivalent). Reedline +handles this. + +**Multiline input.** htcl procs span multiple lines; the editor must +support multi-line buffers with proper indentation. Reedline supports +this; pair with the parser to detect when a buffer is syntactically +complete vs. needs more lines. + +**Vivado worker lifecycle.** A REPL session corresponds to one +long-lived Vivado worker. Cold start happens at REPL launch (with a +spinner during the multi-second Vivado startup); the worker persists +until exit. `:restart` rebuilds the worker without exiting the REPL. + +**Module hot reload.** If a sourced module changes on disk, the REPL +detects it (file watcher), re-sources, and updates the session +document. The user keeps any session-local definitions made after the +module was first loaded. Conflicts (same name now means something +different) are surfaced as warnings. + +### Phasing within the REPL work + +The REPL doesn't need to wait for every other phase to land. It can +ship with a meaningful subset early and grow: + +- **Initial REPL (phase 7 below):** ratatui shell, reedline line + editor, tab completion, signature help, history, multi-line input, + Vivado worker integration, pretty-printed results (phase 4 has + already landed by this point). Parameter completion on IP wrapper + procs works the same as parameter completion on any other proc — + no separate path. +- **`:describe` for live instances:** lands when wired up; depends on + the structured-wire-response work from phase 4 to read live + properties cleanly. +- **Polish phase (alongside LSP phase 8):** Command palette, modal + help overlay, ghost-text suggestions, file-watcher-based module hot + reload. + +### Non-goals for the REPL + +- Mouse interaction. Keyboard-only TUI. Mouse support is an + accessibility win we can add later; not v1. +- Replacing the Vivado GUI. The REPL is for scripted/exploratory + workflows; users who need waveform viewers and floorplanning still + use Vivado proper. +- Persistent named sessions / tmux-style detach. Run inside tmux if + you want that. +- Custom keybinding configuration in v1. Pick sane defaults; expose + config later if requested. + +## Implementation plan + +The plan is organized around extending `vw` with new crates and +subcommands. The existing `vw-lib` and `vw` CLI are not restructured; we +add alongside. + +New crates introduced over the phases: + +- `vw-htcl` — htcl parser, AST, name resolution, signature checking, + TCL emission. The language layer. +- `vw-vivado` — Vivado worker spawn/connect, wire protocol, embedded + shim TCL. The execution layer. +- `vw-vivado-data` — generated database of UG835 builtin commands. + Regenerated per Vivado release; not user-edited. +- `vw-analyzer` — LSP server. Binary. +- `vw-repl` — interactive shell. Binary. + +Not in this project (separate downstream tooling): + +- IP-XACT → htcl wrapper generation. A sideband tool that reads an + IP's IP-XACT `component.xml` and emits an `.htcl` configuration + interface (a wrapper proc whose parameters match the IP's + parameters). Lives in its own repo, ships its own binary, produces + vw-consumable packages. The output is ordinary htcl that vw doesn't + need to know was generated. See "Strategic context" section. +- IP-XACT + configuration values → RSF. A separate tool that produces + the register-spec file for a specific IP instantiation. Reads the + IP-XACT memory map and the configuration values, emits RSF. Not in + vw; see "RSF generation" in Strategic Context. + +The `vw` CLI grows subcommands `run`, `check`, `repl`, `analyzer` (plus +existing `add`, `update`, `test`, etc.). + +### Phase 0: skeleton + +Goal: smallest end-to-end thing that proves the architecture. + +- New crates `vw-htcl` and `vw-vivado` created in the vw repo. +- `vw run` subcommand added to the CLI dispatcher. +- htcl parser for a minimal subset: literals, variables, `set`, `proc`, + command invocation, comments. No control flow yet. Built with + [`winnow`](https://docs.rs/winnow); see the LSP design section's + "Parser" subsection for the full rationale and requirements. +- Vivado worker spawn-and-connect logic. +- Vivado shim with `eval` op only. +- `vw run file.htcl` reads the file, sends each top-level command to + Vivado, prints results. + +Deliverable: `vw run hello.htcl` where `hello.htcl` is `puts "hello"` +prints `hello`. + +### Phase 1: module system + +Goal: `src` works. + +- Use `vw-lib` to find the project root (location of `vw.toml`). +- Implement `src` with relative and filesystem-absolute resolution. +- `@name/...` resolution: query `vw-lib` for the dependency's resolved + cache path, then index into it via the `htcl` selector. +- Module loading: parse file, execute top-level forms, track loaded set + for idempotence. +- Decide and implement namespace semantics (see open question above). +- Coordinate `vw-lib` extensions: the `htcl` selector key in dependency + entries; path dependencies if not already supported. + +Deliverable: a multi-file project loads and runs, including imports from +a `vw`-managed dependency. + +### Phase 2: proc grammar + +Goal: structured proc declarations with attributes. + +- Extend parser for the proc-arg grammar (doc comments, attributes, + names). +- AST representation for procs with metadata. +- At call time, validate keyword args against the declared signature: + required args present, no unknown args, `@enum` / `@range` / + `@requires` / `@conflicts` checked. +- Generate a TCL-side `proc` that takes positional args in canonical + order; callers pass keyword args, vw-htcl reorders them and emits a + positional call. +- Introduce `vw check` subcommand that runs analysis and reports + diagnostics without executing. + +Deliverable: the `axis_interface` example from the language design +section works end-to-end with validation, and `vw check` flags malformed +calls. + +### Phase 3: analyzer (LSP) — initial version + +Goal: editor support lands as soon as the analysis is meaningful. + +- `vw-analyzer` crate, `vw analyzer` subcommand. +- LSP server using `tower-lsp` over stdio. +- `LanguageBackend` trait introduced; `HtclBackend` is the only + implementation initially. The dispatch by file extension is in + place from the start (it just always routes to `HtclBackend`). +- Wire up the existing `vw-htcl` analysis through `HtclBackend`: + diagnostics, document symbols, hover for proc docs, go-to-definition + for `src` targets and proc references. +- Completion for proc arguments (using the signature data from phase + 2). +- VS Code extension stub: activates on `.htcl` and `vw.toml`, launches + `vw analyzer`. + +Deliverable: opening an htcl project in VS Code gives diagnostics, +hover, and basic completion. The LSP is genuinely useful for htcl +from this point forward; later phases add features and bring VHDL +into the same server. + +### Phase 4: structured wire responses + +Goal: avoid Rust-side TCL parsing for structured outputs. + +- Add `eval_structured` op to wire protocol. +- Write Vivado-shim wrappers for the initial command set + (`report_property`, `get_cells`-family, etc.). +- Rust-side types for the parsed results. + +Deliverable: `report_property` returns a typed Rust value, not a string, +in the executor. + +### Phase 5: VHDL via vhdl_ls proxy + +Goal: bring VHDL into `vw analyzer` so it serves both languages from a +single process; the user-facing surface for a unified LSP is in place. + +- `VhdlBackend` implementation that spawns `vhdl_ls` as a subprocess + and proxies LSP requests for `.vhd` / `.vhdl` files. +- Generate `vhdl_ls.toml` from `vw.toml` (reuse the existing `vw` + logic for this) and point the subprocess at it. Regenerate when + `vw.toml` changes. +- File-type dispatch in `vw-analyzer` now routes htcl files to + `HtclBackend` and VHDL files to `VhdlBackend`. +- Cross-language query surface (`find_symbol` etc.) implemented on + `VhdlBackend` via `workspace/symbol` and related `vhdl_ls` + queries. +- Cancellation, lifecycle, and error handling for the subprocess. +- Performance: confirm the proxy adds acceptable overhead. If + noticeable, profile and optimize. + +Deliverable: a single `vw analyzer` process serves htcl and VHDL. +Editors configured to use it see consistent behavior across both +languages without needing a separate `vhdl_ls` configuration. +Cross-language queries from htcl to VHDL work but aren't yet +user-facing (next phase wires up the htcl-side attributes). + +### Phase 6: cross-language analysis + +Goal: htcl ↔ VHDL navigation (the user-facing cross-language +features, building on phase 5's backend wiring). + +- `@vhdl_entity(name)` attribute on htcl procs declaring the entity + they wrap. +- `HtclBackend` resolves entity references by issuing `find_symbol` + to `VhdlBackend`. Go-to-definition surfaces the resulting location. +- Find-references on a VHDL entity surfaces both VHDL instantiations + (from `VhdlBackend`) and htcl wrappers (from `HtclBackend`'s index + of `@vhdl_entity` attributes). +- Generic-to-argument mapping: optional in this phase, depending on + how much work the `find_symbol` extension to "give me this entity's + generics" turns out to be. + +Deliverable: clicking through an htcl proc into its VHDL entity +works, in both directions. + +### Phase 7: REPL + +Goal: ship the REPL as a meaningful interactive environment. See the +dedicated "REPL design" section above for the full treatment. + +- `vw-repl` crate, `vw repl` subcommand. +- Built with `ratatui` and `reedline`. +- Initial feature set (per the REPL phasing subsection): tab completion, + signature help, persistent history, multi-line input, Vivado worker + lifecycle management, pretty-printed structured results (relies on + phase 4). + +Deliverable: a meaningfully better experience than the Vivado console +for exploring a live design — discoverable commands, inline validation, +overlay-based help that doesn't clutter scrollback. + +### Phase 8: LSP polish + +Goal: bring the analyzer up to "rust-analyzer-quality" expectations for +the features that matter most. + +- Find references across the workspace. +- Rename (cautious; abort on ambiguity). +- Workspace symbols. +- Code actions (add missing required argument, remove unused import, + convert raw `set_property -dict` to a structured call). +- Performance tuning; consider `salsa` if hand-rolled caching shows its + limits. + +Deliverable: an analyzer that meets the acceptance criteria in the LSP +design section. + +### Later (not in initial plan) + +- Type system (typed IP handles, units, phases, constraint scopes). +- Quartus backend. +- **Oxide VHDL frontend integration.** Replace the `vhdl_ls` proxy + `VhdlBackend` with a direct integration with Oxide's developing + VHDL frontend. Same `LanguageBackend` trait, different + implementation. Timing depends on the frontend's maturity; the + `LanguageBackend` abstraction exists from phase 3 specifically to + make this swap possible without rippling into htcl-side code. +- Tracing / profiling of TCL execution. +- Distributed worker pools for parallel synthesis runs. +- `vw fmt` (htcl formatter). +- Mechanism for htcl parameter doc comments to propagate into + Vivado-generated wrappers (requires Xilinx-side support; see + "Wrapper documentation" in Strategic Context). + +Not in this project at all (separate downstream tooling): + +- IP-XACT → htcl wrapper generation (a sideband tool). +- IP-XACT + configuration values → RSF generation (a separate tool; + see "RSF generation" in Strategic Context). + +## Open questions to resolve with the author + +1. **Final name for the htcl dialect.** Working name; pick something + durable before shipping anything publicly. This is just the language + name now, not a tool name. +2. **Module namespace semantics.** Global (TCL-compatible, simple) or + scoped with explicit exports (better complexity management, bigger + change)? Recommend scoped, but flag for discussion. +3. **`src` vs `use` vs `import` vs `mod` keyword.** Recommend `use` for + familiarity (Rust) and to avoid the `src/` directory collision. +4. **Shim distribution.** Ship the shim TCL embedded in the `vw-vivado` + binary, written to a temp file at worker startup? Or expect it on + disk somewhere? Embedded is simpler for users; do that unless there's + a reason not to. +5. **`vw-lib` extensions to confirm or schedule:** + - Stable Rust API for "give me the resolved cache path for dependency + X" — confirm shape. + - Generalization of the dependency selector to per-language keys + (`src` for VHDL, `htcl` for htcl), or staying with `src` plus a + parallel `htcl` field. + - Path dependencies (`path = "..."`) — present, absent, or in + progress? +6. **Cross-language wrapper attribute name.** `@vhdl_entity(name)` is + the working syntax for declaring which VHDL entity an htcl proc + wraps. Confirm or rename. +7. **Showcase IP selection.** For the Vivado-team pitch, which IPs do + we cover in the initial generated `xilinx-ip` package? The + IP-XACT → htcl generator (a separate sideband tool) produces + wrappers mechanically, but the showcase needs to demonstrate + quality at a level that earns the conversation. Pick a small set + where the generated wrappers will look genuinely good, plus one or + two complex IPs (DCMAC, CIPS) where the value of source-controlled + configuration is most visible. +8. **`vhdl_ls` proxy specifics.** Phase 5 wires up VHDL via a + subprocess proxy to `vhdl_ls`. Open: does `vhdl_ls`'s + `workspace/symbol` interface answer the cross-language queries we + need (entity location, generic lists)? If not, what's the + smallest extension to either the proxy or to `vhdl_ls` itself that + closes the gap? Confirm before phase 5 starts. +9. **Evidence-gathering for eventual types.** Not a v1 question, but + worth a habit from day one: when working with htcl, keep a log of + cases where a type system would have caught a real bug or documented + intent meaningfully. Revisit the types decision only when there's a + concrete case file to design against. + +## Non-goals + +- TCL language compatibility. We are not implementing TCL; we are implementing + a different language that happens to share TCL's value model and emits TCL + to Vivado. +- General TCL extension authorship. The Vivado shim is the only TCL we write + intentionally; it stays small. +- Replacing Vivado's interpreter in-process. We talk to it over a pipe. +- Supporting every Vivado command natively. Most commands pass through as + strings; we add structured wrappers only where they pay off. +- A package registry. Git-source dependencies cover the realistic needs; a + registry is a separate company. +- **IP-XACT awareness in vw, the analyzer, or the REPL.** IP-XACT is a + source format for *generating* htcl IP wrapper packages via a + separate sideband tool. The tooling described in this plan consumes + only htcl; it has no IP-XACT-specific code paths, data structures, + or features. See "Strategic context" for the rationale. +- **A replacement for IP-XACT.** htcl is a configuration interface + layer that sits above IP-XACT (specification) and below generated + RTL/RSF (instantiation). It does not describe ports, memory maps, + or any other aspect of an IP's structure — those remain IP-XACT's + responsibility. See "Conceptual layering" in Strategic Context. +- **Port-level analysis of generated RTL.** htcl wrappers don't + describe the ports their instantiation will emit; ports come from + the VHDL/Verilog Vivado generates. Cross-IP wiring analysis is + possible but lives in the VHDL analyzer, not in htcl. +- **Memory-map description.** Not htcl's job. Register interfaces are + generated from IP-XACT plus configuration values by a separate + pipeline targeting RSF; see "RSF generation" in Strategic Context. + +## Reference points + +- `vw`: https://github.com/oxidecomputer/vw — the host project for this + work. `vw-lib` is the existing library that handles dependency + resolution, caching, and manifest/lockfile management. The plan + extends `vw` with htcl-language support and an analyzer/repl modeled + on rust-analyzer's relationship to Cargo. +- rust-analyzer + Cargo: the architectural model. rust-analyzer reads + `Cargo.toml` / `Cargo.lock`, resolves dependencies through Cargo's + data model, and provides editor support without reimplementing the + build tool. `vw analyzer` plays the same role for `vw.toml` / + `vw.lock`. +- UG835: Vivado Design Suite Tcl Command Reference — the authoritative + source for what commands exist and what they return. +- IP-XACT (IEEE 1685): the schema for IP component metadata. *Not used + internally by vw, the analyzer, or the REPL.* IP-XACT is the source + format consumed by a separate sideband tool that generates `.htcl` + IP packages, which vw then resolves like any other dependency. +- TypeScript: model for "additive features over an existing language" + done well. Discipline: existing code keeps working (except htcl + breaks this intentionally for the proc-arg case), new features are + opt-in, output is consumable by tools that don't know about the new + features. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..30a41bc --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.97.1" +profile = "default" diff --git a/vw-agent/Cargo.toml b/vw-agent/Cargo.toml new file mode 100644 index 0000000..233e521 --- /dev/null +++ b/vw-agent/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "vw-agent" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Receives source on a vw build instance" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools"] + +[[bin]] +name = "vw-agent" +path = "src/main.rs" + +[dependencies] +vw-sync = { path = "../vw-sync" } +vw-remote = { path = "../vw-remote" } +vw-bench = { path = "../vw-bench" } +vw-lib = { path = "../vw-lib" } +rust-s3 = { version = "0.37", default-features = false, features = ["with-tokio"] } +blake3 = "1" +gethostname = "1" +rand = "0.8" +reqwest = { version = "0.13", features = ["json"] } +serde.workspace = true +serde_json.workspace = true +tokio-tungstenite.workspace = true +vw-sync-api = { path = "../vw-sync-api" } +vw-api-types-versions = { path = "../vw-api-types/versions" } +camino.workspace = true +clap.workspace = true +dropshot.workspace = true +thiserror.workspace = true +tokio.workspace = true +slog = "2.8.2" +slog-bunyan = "2.5.0" +slog-async = "2.8.0" +slog-error-chain = { git = "https://github.com/oxidecomputer/slog-error-chain", branch = "main" } + +[dev-dependencies] +tempfile.workspace = true +vw-sync = { path = "../vw-sync" } +vw-remote = { path = "../vw-remote" } +tokio-tungstenite.workspace = true +serde_json.workspace = true diff --git a/vw-agent/src/artifacts.rs b/vw-agent/src/artifacts.rs new file mode 100644 index 0000000..a1f9eff --- /dev/null +++ b/vw-agent/src/artifacts.rs @@ -0,0 +1,543 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Getting finished artifacts off the instance that built them. +//! +//! A build leaves its images in `target/image`, which synchronization never +//! touches in either direction — that is the whole point of `target`. Without +//! something like this an artifact would exist only on an instance that is, by +//! design, disposable. +//! +//! Polled rather than watched. A `.pdi` is written once at the end of a run +//! that took hours, so noticing it a few seconds late costs nothing, and a +//! poll has none of a watcher's trouble with a directory that does not exist +//! yet, is deleted by `vw clean`, and then reappears. + +use std::collections::HashMap; +use std::time::Duration; + +use camino::{Utf8Path, Utf8PathBuf}; +use slog::{info, warn, Logger}; +use vw_api_types_versions::latest::S3Credentials; + +/// Where a build leaves things worth keeping, and what to keep from each. +/// +/// Directories under the workspace's `target`, paired with the extension that +/// matters there. Everything else a build writes — checkpoints, logs, journal +/// files, the vivado project — is either enormous, only meaningful on the +/// machine that made it, or both. +/// +/// `synth`, `place` and `route` each produce an `edif` and they are different +/// netlists, which is why what goes in the bucket is keyed by the directory +/// too rather than by file name alone. +const GATHERED: [(&str, &str); 5] = [ + ("image", "pdi"), + ("reports", "rpt"), + ("synth", "edif"), + ("place", "edif"), + ("route", "edif"), +]; + +/// The directory a build writes everything to, under the workspace. +const BUILD_OUTPUT: &str = "target"; + +/// How often to look. +/// +/// Often enough that an artifact is on its way before anyone thinks to ask +/// for it. A scan of one directory costs a `readdir`, and the digest check +/// only reads a file that has actually changed. +const INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ArtifactError { + #[error("writing {0}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), + #[error("the stored artifact target is not readable")] + Corrupt, +} + +/// Remember where artifacts go, so a restart does not have to be told again. +/// +/// The instance can reboot between one build and the next, and whoever told it +/// where to put things may not think to say so a second time. +pub(crate) fn remember( + path: &Utf8Path, + credentials: &S3Credentials, +) -> Result<(), ArtifactError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| ArtifactError::Write(parent.to_owned(), e))?; + } + + let encoded = serde_json::to_string_pretty(credentials) + .map_err(|_| ArtifactError::Corrupt)?; + std::fs::write(path, encoded) + .map_err(|e| ArtifactError::Write(path.to_owned(), e))?; + + // It holds a secret key. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| ArtifactError::Write(path.to_owned(), e))?; + } + + Ok(()) +} + +/// What was remembered, if anything. +pub(crate) fn recall( + path: &Utf8Path, +) -> Result, ArtifactError> { + if !path.is_file() { + return Ok(None); + } + + let stored = std::fs::read_to_string(path) + .map_err(|e| ArtifactError::Read(path.to_owned(), e))?; + serde_json::from_str(&stored) + .map(Some) + .map_err(|_| ArtifactError::Corrupt) +} + +/// Watch `root`'s image directory and upload whatever appears in it. +/// +/// Runs until the agent stops. Takes its target from `target`, which is +/// updated whenever the service tells us where artifacts go — so an agent that +/// started before anyone said simply uploads nothing until someone does. +pub(crate) async fn synchronize( + root: Utf8PathBuf, + mut target: tokio::sync::watch::Receiver>, + log: Logger, +) { + // What has already gone, by path, with the digest that went. Keyed by + // digest rather than a timestamp so a rebuild that produces byte-identical + // output is not uploaded twice, and one that produces different bytes + // under the same name is. + let mut uploaded: HashMap = HashMap::new(); + // What each file looked like last time round, to tell a finished artifact + // from one still being written. + let mut previously: HashMap = HashMap::new(); + + loop { + tokio::select! { + () = tokio::time::sleep(INTERVAL) => {} + // A new target means the old record of what has been sent is + // about somewhere else. + changed = target.changed() => { + if changed.is_err() { + return; + } + uploaded.clear(); + } + } + + let credentials = target.borrow().clone(); + let Some(credentials) = credentials else { + continue; + }; + + let mut currently = HashMap::new(); + for Found { + path: artifact, + key, + } in artifacts(&root) + { + // An image is written over seconds, and this looks every second. + // A file whose size or timestamp moved since the last pass is + // still being written, and uploading it now would put a truncated + // artifact in the bucket under the name of a finished one — which + // is worse than not having it yet, because it looks like success. + let Some(stamp) = stamp(&artifact) else { + continue; + }; + let settled = previously.get(&artifact) == Some(&stamp); + currently.insert(artifact.clone(), stamp); + if !settled { + continue; + } + + let digest = match digest_of(&artifact) { + Ok(digest) => digest, + Err(e) => { + warn!(log, "cannot read an artifact"; + "path" => %artifact, + "error" => %e, + ); + continue; + } + }; + if uploaded.get(&artifact) == Some(&digest) { + continue; + } + + match upload(&credentials, &key, &artifact).await { + Ok(()) => { + info!(log, "uploaded an artifact"; + "path" => %artifact, + "bucket" => &credentials.bucket, + "key" => &key, + ); + uploaded.insert(artifact, digest); + } + Err(e) => { + // Left out of `uploaded`, so the next pass tries again. + // A store that is briefly unreachable should not cost an + // artifact. + warn!(log, "cannot upload an artifact"; + "path" => %artifact, + "error" => %e, + ); + } + } + } + previously = currently; + } +} + +/// What a file looks like from the outside, cheaply. +/// +/// Size and modification time together are enough to notice a file that is +/// still growing, without reading it — which matters when this runs every +/// second and the file is hundreds of megabytes. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Stamp { + size: u64, + modified: Option, +} + +fn stamp(path: &Utf8Path) -> Option { + let metadata = std::fs::metadata(path).ok()?; + Some(Stamp { + size: metadata.len(), + modified: metadata.modified().ok(), + }) +} + +/// One thing worth keeping, and the name it goes under. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct Found { + /// Where it is on this instance. + path: Utf8PathBuf, + /// What it is called in the bucket: the path under `target`, so a netlist + /// from `synth` and one from `route` stay two different objects rather + /// than one overwriting the other. + key: String, +} + +/// Everything currently sitting in the directories a build fills. +/// +/// Not recursive: each of these is written flat, and descending would pick up +/// the working state of whatever wrote them — vivado's `.runs` scratch under a +/// stage directory is neither small nor meaningful anywhere else. +fn artifacts(root: &Utf8Path) -> Vec { + let mut found = Vec::new(); + + for (directory, extension) in GATHERED { + let source = root.join(BUILD_OUTPUT).join(directory); + let Ok(entries) = std::fs::read_dir(&source) else { + // A build that has not reached this stage yet, which is the + // ordinary case for most of them most of the time. + continue; + }; + + for path in entries + .flatten() + .filter_map(|entry| Utf8PathBuf::from_path_buf(entry.path()).ok()) + .filter(|path| path.is_file()) + .filter(|path| path.extension() == Some(extension)) + { + let Some(name) = path.file_name() else { + continue; + }; + found.push(Found { + key: format!("{directory}/{name}"), + path, + }); + } + } + + found.sort(); + found +} + +/// What an artifact currently hashes to, without holding it in memory. +/// +/// An image runs to hundreds of megabytes and this runs every second; reading +/// one into a buffer to hash it would make an idle agent's memory use track +/// the size of the last build. +fn digest_of(path: &Utf8Path) -> std::io::Result { + use std::io::Read; + + let mut file = std::fs::File::open(path)?; + let mut hasher = blake3::Hasher::new(); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + + Ok(hasher.finalize().to_hex().to_string()) +} + +/// Send a specific set of files to the store, keyed by where they sit under +/// the build output directory. +/// +/// The other path here polls, because vivado writes when it likes and nothing +/// announces it. A cargo build does announce it — cargo names every file it +/// produced — so there is nothing to discover and nothing to wait for. +/// +/// Returns how many went. A failure is logged and skipped rather than +/// abandoning the rest: one unreachable moment should not cost the other +/// artifacts of the same build. +pub(crate) async fn upload_all( + root: &Utf8Path, + credentials: &S3Credentials, + artifacts: &[Utf8PathBuf], + log: &Logger, +) -> usize { + let mut sent = 0; + for artifact in artifacts { + // Keyed by its path under `target`, so a debug and a release build of + // the same name stay two objects, as do the same name built for two + // different targets. + let key = artifact + .strip_prefix(root.join(BUILD_OUTPUT)) + .map(Utf8Path::to_string) + .unwrap_or_else(|_| { + artifact.file_name().unwrap_or("artifact").to_owned() + }); + + match upload(credentials, &key, artifact).await { + Ok(()) => { + info!(log, "uploaded a build artifact"; + "path" => %artifact, + "bucket" => &credentials.bucket, + "key" => &key, + ); + sent += 1; + } + Err(e) => warn!(log, "cannot upload a build artifact"; + "path" => %artifact, + "error" => %e, + ), + } + } + sent +} + +/// Put one artifact in the bucket. +/// +/// Streamed from disk rather than read first, for the same reason the digest +/// is: the files this exists to move are large, and neither end of the wire +/// needs a copy of one in memory for it to get across. +async fn upload( + credentials: &S3Credentials, + key: &str, + path: &Utf8Path, +) -> Result<(), Box> { + let region = s3::Region::Custom { + region: credentials.region.clone(), + endpoint: credentials.endpoint.clone(), + }; + let creds = s3::creds::Credentials::new( + Some(&credentials.access_key_id), + Some(&credentials.secret_access_key), + None, + None, + None, + )?; + + // Path style because the bucket is reached by address rather than by name: + // there is no DNS inside the VPC that would resolve + // `vivado-darmok.`. + let bucket = + s3::Bucket::new(&credentials.bucket, region, creds)?.with_path_style(); + + let mut file = tokio::fs::File::open(path).await?; + let status = bucket + .put_object_stream(&mut file, format!("/{key}")) + .await? + .status_code(); + if status >= 300 { + return Err(format!("the object store answered {status}").into()); + } + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + fn scratch() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + (dir, root) + } + + fn credentials() -> S3Credentials { + S3Credentials { + endpoint: "http://127.0.0.1:3900".to_owned(), + port: 3900, + region: "garage".to_owned(), + bucket: "vivado-darmok".to_owned(), + access_key_id: "GK00000000000000000000000".to_owned(), + secret_access_key: "shhh".to_owned(), + } + } + + /// Put a file where a build would leave it. + fn build_output(root: &Utf8Path, relative: &str, contents: &str) { + let path = root.join(BUILD_OUTPUT).join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, contents).expect("write"); + } + + #[test] + fn only_what_a_build_produced_counts_as_an_artifact() { + let (_dir, root) = scratch(); + build_output(&root, "image/top.pdi", "image"); + build_output(&root, "reports/timing.rpt", "report"); + build_output(&root, "synth/design.edif", "netlist"); + // Neither small nor meaningful anywhere else. + build_output(&root, "image/top.bit", "bitstream"); + build_output(&root, "synth/design.dcp", "checkpoint"); + build_output(&root, "vw-project/project.xpr", "project"); + build_output(&root, "logs/vivado.log", "log"); + // A stage's working directory is not ours to send either. + build_output(&root, "synth/runs/inner.edif", "scratch"); + + let found: Vec = + artifacts(&root).into_iter().map(|f| f.key).collect(); + + assert_eq!( + found, + ["image/top.pdi", "reports/timing.rpt", "synth/design.edif",], + ); + } + + #[test] + fn a_netlist_from_each_stage_is_its_own_artifact() { + // `synth`, `place` and `route` all produce `design.edif` and they are + // three different netlists. Keyed by file name alone, whichever + // uploaded last would be the only one anybody could ever download. + let (_dir, root) = scratch(); + for stage in ["synth", "place", "route"] { + build_output(&root, &format!("{stage}/design.edif"), stage); + } + + let found: Vec = + artifacts(&root).into_iter().map(|f| f.key).collect(); + + assert_eq!( + found, + [ + "place/design.edif", + "route/design.edif", + "synth/design.edif" + ], + ); + } + + #[test] + fn an_instance_with_no_build_output_has_nothing_to_send() { + let (_dir, root) = scratch(); + assert!(artifacts(&root).is_empty()); + } + + #[test] + fn a_build_that_has_only_reached_synthesis_sends_what_it_has() { + // Most of the time most stages do not exist yet, and that is not a + // condition worth reporting — it is just a build in progress. + let (_dir, root) = scratch(); + build_output(&root, "synth/design.edif", "netlist"); + + let found: Vec = + artifacts(&root).into_iter().map(|f| f.key).collect(); + + assert_eq!(found, ["synth/design.edif"]); + } + + #[test] + fn where_artifacts_go_survives_a_restart() { + let (_dir, root) = scratch(); + let path = root.join("artifact-target.json"); + + remember(&path, &credentials()).expect("remember"); + let recalled = recall(&path).expect("recall").expect("something"); + + assert_eq!(recalled.bucket, "vivado-darmok"); + assert_eq!(recalled.access_key_id, "GK00000000000000000000000"); + } + + #[test] + fn nothing_remembered_is_not_an_error() { + let (_dir, root) = scratch(); + assert!(recall(&root.join("nothing.json")) + .expect("recall") + .is_none()); + } + + #[cfg(unix)] + #[test] + fn a_stored_key_is_readable_only_by_its_owner() { + use std::os::unix::fs::PermissionsExt; + let (_dir, root) = scratch(); + let path = root.join("artifact-target.json"); + + remember(&path, &credentials()).expect("remember"); + + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); + } + + #[test] + fn a_file_still_being_written_is_not_yet_an_artifact() { + // An image is written over seconds and this looks every second, so the + // difference between "finished" and "half there" is the only thing + // standing between a consumer and a truncated build. + let (_dir, root) = scratch(); + let path = root.join(BUILD_OUTPUT).join("image/top.pdi"); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + + std::fs::write(&path, "half").expect("write"); + let half = stamp(&path).expect("stamp"); + + // Still growing: this pass looks different from the last. + std::fs::write(&path, "half and then some").expect("write"); + let grown = stamp(&path).expect("stamp"); + assert_ne!(half, grown, "a file that grew should look different"); + + // Unchanged since the last pass, so it is done. + assert_eq!(stamp(&path).expect("stamp"), grown); + } + + #[test] + fn a_digest_does_not_depend_on_reading_the_whole_file_at_once() { + // The point is constant memory, but what has to be true for the change + // check to work is that the digest tracks the contents. + let (_dir, root) = scratch(); + let path = root.join("artifact.pdi"); + + std::fs::write(&path, vec![7u8; 300_000]).expect("write"); + let first = digest_of(&path).expect("digest"); + assert_eq!(digest_of(&path).expect("digest"), first, "stable"); + + std::fs::write(&path, vec![9u8; 300_000]).expect("write"); + assert_ne!(digest_of(&path).expect("digest"), first, "changed"); + } + + #[test] + fn a_secret_is_not_in_the_debug_output() { + let printed = format!("{:?}", credentials()); + assert!(!printed.contains("shhh"), "{printed}"); + assert!(printed.contains("vivado-darmok"), "{printed}"); + } +} diff --git a/vw-agent/src/error.rs b/vw-agent/src/error.rs new file mode 100644 index 0000000..8743ec1 --- /dev/null +++ b/vw-agent/src/error.rs @@ -0,0 +1,82 @@ +//! Turning the sync engine's errors into the ones the API surfaces. +//! +//! The split that matters is between a caller who sent something wrong and an +//! instance that is in no state to help. A manifest naming a path outside the +//! tree, or content that does not hash to the digest it was sent under, is the +//! caller's mistake and worth saying so precisely — retrying the same request +//! will fail the same way. A directory that cannot be written is not, and the +//! detail belongs in the log rather than in the response. +//! +//! Free functions rather than `From` impls: both the engine's errors and +//! dropshot's are foreign to this crate, so there is no impl to write. + +use dropshot::{ClientErrorStatusCode, HttpError}; + +use vw_sync::{ApplyError, StoreError}; + +pub(crate) fn apply_error(value: ApplyError) -> HttpError { + let message = value.to_string(); + match value { + // The manifest itself is wrong. + ApplyError::UnsafePath(_) => HttpError::for_bad_request(None, message), + // The manifest is fine, but it was committed before everything it + // names had been delivered. The message names the digest, so a caller + // that skipped a blob can tell which one. + ApplyError::MissingContent { .. } => HttpError::for_client_error( + None, + ClientErrorStatusCode::CONFLICT, + message, + ), + ApplyError::Store(e) => store_error(e), + // Something about this instance. + ApplyError::Scan(..) + | ApplyError::CreateDir(..) + | ApplyError::Write(..) + | ApplyError::Remove(..) => HttpError::for_internal_error(message), + } +} + +pub(crate) fn store_error(value: StoreError) -> HttpError { + let message = value.to_string(); + match value { + // Both are the caller's to fix, and both are worth refusing loudly: a + // digest that is not a digest is trying to become a path it should + // not, and content that does not match its digest would be handed back + // later under a name that lies about it. + StoreError::MalformedDigest(_) | StoreError::DigestMismatch(_) => { + HttpError::for_bad_request(None, message) + } + StoreError::CreateDir(..) + | StoreError::Write(..) + | StoreError::Read(..) + | StoreError::Empty(..) => HttpError::for_internal_error(message), + } +} + +/// Credentials that cannot be written are always this instance's problem: the +/// request carried everything it needed to, and what failed was a filesystem +/// it owns. +pub(crate) fn netrc_error(value: crate::netrc::NetrcError) -> HttpError { + HttpError::for_internal_error(value.to_string()) +} + +/// A caller asking for a generated file can get it wrong in exactly two ways: +/// naming one that is not there, and naming something that was never theirs to +/// ask for. Both are worth saying precisely — the first is a build that has not +/// run, the second is a mistake nobody should be able to make by accident. +pub(crate) fn generated_error( + value: crate::generated::GeneratedError, +) -> HttpError { + let message = value.to_string(); + match value { + crate::generated::GeneratedError::UnsafePath(_) => { + HttpError::for_bad_request(None, message) + } + crate::generated::GeneratedError::NotFound(_) => { + HttpError::for_not_found(None, message) + } + crate::generated::GeneratedError::Read(..) => { + HttpError::for_internal_error(message) + } + } +} diff --git a/vw-agent/src/garage.rs b/vw-agent/src/garage.rs new file mode 100644 index 0000000..93fc757 --- /dev/null +++ b/vw-agent/src/garage.rs @@ -0,0 +1,649 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Bringing up the object store an environment's artifacts land in. +//! +//! Run on the artifact instance. Garage is configured, started and bootstrapped +//! here rather than provisioned from outside, which means no credential is ever +//! shared in advance: the admin token is generated on this machine, written to +//! a file only this machine reads, and never leaves it. What does leave is one +//! S3 access key, handed to `vw-svc` when it asks, which passes it to the +//! instance that has artifacts to upload. +//! +//! Everything here is written to survive a restart. The instance can reboot, +//! the agent can be upgraded, and the key stays the same — it is recorded on +//! disk beside garage's own state, so nothing downstream has to be told about +//! it again. + +use camino::{Utf8Path, Utf8PathBuf}; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use slog::{info, Logger}; +use vw_api_types_versions::latest::S3Credentials; + +/// Where garage listens, and where its state lives. +#[derive(Clone, Debug)] +pub(crate) struct Settings { + /// Directory for garage's config, metadata and data. + pub(crate) dir: Utf8PathBuf, + /// The S3 API port. 3900 is garage's own default and what everything + /// downstream expects. + pub(crate) s3_port: u16, + /// The admin API port, reachable only from this machine. + pub(crate) admin_port: u16, + /// The internal RPC port garage uses to talk to itself. + pub(crate) rpc_port: u16, + /// How much of this instance's disk garage may use. + pub(crate) capacity: String, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum GarageError { + #[error("creating {0}")] + CreateDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), + #[error("garage is not installed on this instance")] + NotInstalled(#[source] std::io::Error), + #[error("garage did not become ready within {0:?}")] + NeverReady(std::time::Duration), + #[error("`garage {0}` failed: {1}")] + Command(String, String), + #[error("talking to garage's admin api")] + Admin(#[source] reqwest::Error), + #[error("garage's admin api answered {status}: {body}")] + AdminRefused { status: u16, body: String }, + #[error("garage's answer to {0} was not what was expected")] + Unexpected(String), +} + +/// How long to wait for a fresh garage to start answering. +const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +/// A running garage, and the key that opens it. +pub(crate) struct Store { + /// The key, and the bucket each kind of instance writes to. + pub(crate) credentials: S3Credentials, + /// The bucket for each kind, by the name the caller asks for. + pub(crate) buckets: std::collections::BTreeMap, + /// Kept so garage is torn down with the agent rather than outliving it. + _child: tokio::process::Child, +} + +/// Configure, start and bootstrap garage, returning the key for this +/// environment's bucket. +/// +/// Safe to run again on a machine that has already done it: the config, the +/// cluster layout and the key are each created only if absent, so a reboot +/// comes back to the same store with the same credentials. +pub(crate) async fn start( + environment: &str, + settings: &Settings, + log: &Logger, +) -> Result { + for directory in [ + &settings.dir, + &settings.dir.join("meta"), + &settings.dir.join("data"), + ] { + std::fs::create_dir_all(directory) + .map_err(|e| GarageError::CreateDir(directory.clone(), e))?; + } + + let config = settings.dir.join("garage.toml"); + let secrets = ensure_config(&config, settings)?; + + info!(log, "starting garage"; "config" => %config); + let child = tokio::process::Command::new("garage") + .args(["-c", config.as_str(), "server"]) + .kill_on_drop(true) + .spawn() + .map_err(GarageError::NotInstalled)?; + + let admin = Admin { + base: format!("http://127.0.0.1:{}", settings.admin_port), + token: secrets.admin_token.clone(), + client: reqwest::Client::new(), + }; + admin.wait_until_ready(log).await?; + ensure_layout(&config, &admin, settings, log).await?; + + let (credentials, buckets) = + ensure_credentials(environment, settings, &admin, log).await?; + + Ok(Store { + credentials, + buckets, + _child: child, + }) +} + +/// The two secrets garage's own config needs. +struct Secrets { + admin_token: String, +} + +/// Write garage's config if it is not already there, and read back the admin +/// token either way. +/// +/// Not overwritten on a restart: the admin token in a config that garage's +/// on-disk state was created under is the one that still opens it, and +/// generating a fresh one every boot would lock us out of our own store. +fn ensure_config( + path: &Utf8Path, + settings: &Settings, +) -> Result { + if path.is_file() { + let existing = std::fs::read_to_string(path) + .map_err(|e| GarageError::Read(path.to_owned(), e))?; + let admin_token = existing + .lines() + .find_map(|line| line.strip_prefix("admin_token = ")) + .map(|value| value.trim_matches('"').to_owned()) + .ok_or_else(|| { + GarageError::Unexpected(format!("{path} has no admin_token")) + })?; + return Ok(Secrets { admin_token }); + } + + let admin_token = secret(); + let rpc_secret = secret(); + let config = format!( + "metadata_dir = \"{dir}/meta\"\n\ + data_dir = \"{dir}/data\"\n\ + db_engine = \"lmdb\"\n\ + replication_factor = 1\n\ + \n\ + rpc_bind_addr = \"127.0.0.1:{rpc}\"\n\ + rpc_public_addr = \"127.0.0.1:{rpc}\"\n\ + rpc_secret = \"{rpc_secret}\"\n\ + \n\ + [s3_api]\n\ + s3_region = \"garage\"\n\ + api_bind_addr = \"[::]:{s3}\"\n\ + root_domain = \".s3.garage\"\n\ + \n\ + [admin]\n\ + api_bind_addr = \"127.0.0.1:{admin}\"\n\ + admin_token = \"{admin_token}\"\n", + dir = settings.dir, + rpc = settings.rpc_port, + s3 = settings.s3_port, + admin = settings.admin_port, + ); + + std::fs::write(path, config) + .map_err(|e| GarageError::Write(path.to_owned(), e))?; + // The file holds two secrets and garage itself refuses to read a + // world-readable one. + restrict(path)?; + + Ok(Secrets { admin_token }) +} + +/// Give a single node a share of the cluster, so garage will serve S3. +/// +/// A garage with no layout accepts no objects, which is the one step a fresh +/// instance cannot skip. Done through garage's own CLI rather than the admin +/// API: the layout request type is not the layout response type, and the CLI +/// is the interface garage documents for this. Nothing is read back from it — +/// only whether it worked — so there is no output to parse. +async fn ensure_layout( + config: &Utf8Path, + admin: &Admin, + settings: &Settings, + log: &Logger, +) -> Result<(), GarageError> { + if admin.layout_version().await? > 0 { + info!(log, "garage already has a cluster layout"); + return Ok(()); + } + + let node = admin.node_id().await?; + info!(log, "assigning a cluster layout"; "node" => &node); + + run_garage( + config, + &[ + "layout", + "assign", + "-z", + "vw", + "-c", + &settings.capacity, + &node, + ], + ) + .await?; + run_garage(config, &["layout", "apply", "--version", "1"]).await?; + + Ok(()) +} + +/// The kinds of instance that produce artifacts, and so have a bucket. +/// +/// Helios has one before it has anything to put in it. A bucket costs nothing +/// standing empty, and creating it now means the day the driver build starts +/// producing something there is nowhere for it to be missing. +pub(crate) const KINDS: [&str; 2] = ["vivado", "helios"]; + +/// What was minted, remembered so a reboot comes back to the same store. +#[derive(Serialize, Deserialize)] +struct Minted { + credentials: S3Credentials, + buckets: std::collections::BTreeMap, +} + +/// The key and buckets for this environment, created if this is the first boot. +/// +/// One key with access to every bucket rather than one key each: they are all +/// reached from inside one VPC by instances of one environment, so a second +/// key would be ceremony without a boundary behind it. +async fn ensure_credentials( + environment: &str, + settings: &Settings, + admin: &Admin, + log: &Logger, +) -> Result< + (S3Credentials, std::collections::BTreeMap), + GarageError, +> { + let path = settings.dir.join("credentials.json"); + if path.is_file() { + let stored = std::fs::read_to_string(&path) + .map_err(|e| GarageError::Read(path.clone(), e))?; + let minted: Minted = serde_json::from_str(&stored) + .map_err(|_| GarageError::Unexpected(path.to_string()))?; + info!(log, "reusing the store credentials from a previous run"; + "buckets" => minted.buckets.len(), + ); + return Ok((minted.credentials, minted.buckets)); + } + + let key = admin.ensure_key(&format!("vw-{environment}"), log).await?; + + // A bucket per environment per kind, named for both. Even though an + // artifact instance serves one environment today, a name that says which + // one keeps the objects legible if a store is ever shared. + let mut buckets = std::collections::BTreeMap::new(); + for kind in KINDS { + let bucket = format!("{kind}-{environment}"); + // Adopted if it is already there. That happens when this record was + // lost but the store's own state was not — a disk restored from a + // snapshot, a file removed by hand. Failing instead would leave an + // instance that cannot serve the artifacts sitting right there in a + // bucket, and creating a second bucket would orphan them. Neither is + // as good as picking up where we left off. + let bucket_id = admin.ensure_bucket(&bucket, log).await?; + admin.allow(&bucket_id, &key.access_key_id).await?; + buckets.insert(kind.to_owned(), bucket); + } + + let credentials = S3Credentials { + // Left for the caller to fill in: this instance cannot see which of + // its addresses another instance can reach it on. The port it can + // see, because it chose it. + endpoint: String::new(), + port: settings.s3_port, + region: "garage".to_owned(), + // Filled in per caller, which is the only thing that differs between + // one instance's view of this store and another's. + bucket: String::new(), + access_key_id: key.access_key_id, + secret_access_key: key.secret_access_key, + }; + + let minted = Minted { + credentials: credentials.clone(), + buckets: buckets.clone(), + }; + std::fs::write( + &path, + serde_json::to_string_pretty(&minted) + .map_err(|_| GarageError::Unexpected(path.to_string()))?, + ) + .map_err(|e| GarageError::Write(path.clone(), e))?; + restrict(&path)?; + + info!(log, "created the store for this environment"; + "buckets" => format!("{:?}", buckets.values().collect::>()), + ); + + Ok((credentials, buckets)) +} + +/// Run one garage subcommand against our config. +async fn run_garage( + config: &Utf8Path, + args: &[&str], +) -> Result<(), GarageError> { + let output = tokio::process::Command::new("garage") + .args(["-c", config.as_str()]) + .args(args) + .output() + .await + .map_err(GarageError::NotInstalled)?; + + if !output.status.success() { + let mut detail = String::from_utf8_lossy(&output.stderr).into_owned(); + detail.push_str(&String::from_utf8_lossy(&output.stdout)); + return Err(GarageError::Command(args.join(" "), detail)); + } + + Ok(()) +} + +/// Garage's admin API, which only this machine can reach. +struct Admin { + base: String, + token: String, + client: reqwest::Client, +} + +/// What creating a key gave us. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct CreatedKey { + access_key_id: String, + secret_access_key: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct CreatedBucket { + id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Permissions { + read: bool, + write: bool, + owner: bool, +} + +impl Admin { + async fn wait_until_ready(&self, log: &Logger) -> Result<(), GarageError> { + let deadline = std::time::Instant::now() + READY_TIMEOUT; + loop { + if self.layout_version().await.is_ok() { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + return Err(GarageError::NeverReady(READY_TIMEOUT)); + } + slog::debug!(log, "waiting for garage to come up"); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + + async fn get( + &self, + endpoint: &str, + ) -> Result { + let response = self + .client + .get(format!("{}/v2/{endpoint}", self.base)) + .bearer_auth(&self.token) + .send() + .await + .map_err(GarageError::Admin)?; + Self::decode(endpoint, response).await + } + + async fn post( + &self, + endpoint: &str, + body: &T, + ) -> Result { + let response = self + .client + .post(format!("{}/v2/{endpoint}", self.base)) + .bearer_auth(&self.token) + .json(body) + .send() + .await + .map_err(GarageError::Admin)?; + Self::decode(endpoint, response).await + } + + async fn decode( + endpoint: &str, + response: reqwest::Response, + ) -> Result { + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(GarageError::AdminRefused { + status: status.as_u16(), + body, + }); + } + response + .json() + .await + .map_err(|_| GarageError::Unexpected(endpoint.to_owned())) + } + + async fn layout_version(&self) -> Result { + let status = self.get("GetClusterStatus").await?; + status + .get("layoutVersion") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| GarageError::Unexpected("GetClusterStatus".into())) + } + + async fn node_id(&self) -> Result { + let status = self.get("GetClusterStatus").await?; + status + .get("nodes") + .and_then(|nodes| nodes.get(0)) + .and_then(|node| node.get("id")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| GarageError::Unexpected("GetClusterStatus".into())) + } + + /// The key with this name, creating one only if there is not one already. + /// + /// Adopted rather than replaced for the same reason a bucket is, plus one + /// of its own: a new key every time this record was lost would leave the + /// old ones behind with permission on the bucket, so a store would collect + /// working credentials nobody remembers issuing. Garage will hand back the + /// secret of a key it already has, which is what makes reuse possible. + async fn ensure_key( + &self, + name: &str, + log: &Logger, + ) -> Result { + if let Some(existing) = self.find_key(name).await? { + info!(log, "reusing the key this store was set up with"; + "key" => &existing.access_key_id, + ); + return Ok(existing); + } + + let created = self + .post("CreateKey", &serde_json::json!({ "name": name })) + .await?; + serde_json::from_value(created) + .map_err(|_| GarageError::Unexpected("CreateKey".into())) + } + + /// The key named `name`, secret and all, if this store has one. + async fn find_key( + &self, + name: &str, + ) -> Result, GarageError> { + let keys = self.get("ListKeys").await?; + let Some(existing) = keys.as_array().and_then(|keys| { + keys.iter().find(|key| { + key.get("name").and_then(serde_json::Value::as_str) + == Some(name) + }) + }) else { + return Ok(None); + }; + + let id = existing + .get("id") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| GarageError::Unexpected("ListKeys".into()))?; + + let full = self + .get(&format!("GetKeyInfo?id={id}&showSecretKey=true")) + .await?; + serde_json::from_value(full) + .map(Some) + .map_err(|_| GarageError::Unexpected("GetKeyInfo".into())) + } + + /// The bucket with this alias, creating it only if it is not there. + /// + /// Never destructive: an existing bucket is adopted with everything in it, + /// because the objects in it are the entire reason any of this exists. + async fn ensure_bucket( + &self, + alias: &str, + log: &Logger, + ) -> Result { + match self + .post("CreateBucket", &serde_json::json!({ "globalAlias": alias })) + .await + { + Ok(created) => { + let bucket: CreatedBucket = serde_json::from_value(created) + .map_err(|_| { + GarageError::Unexpected("CreateBucket".into()) + })?; + Ok(bucket.id) + } + // Already there, from an earlier life of this instance. + Err(GarageError::AdminRefused { status: 409, .. }) => { + let existing = self + .get(&format!("GetBucketInfo?globalAlias={alias}")) + .await?; + let bucket: CreatedBucket = serde_json::from_value(existing) + .map_err(|_| { + GarageError::Unexpected("GetBucketInfo".into()) + })?; + info!(log, "adopting a bucket that already exists"; + "bucket" => alias, + ); + Ok(bucket.id) + } + Err(e) => Err(e), + } + } + + async fn allow( + &self, + bucket_id: &str, + access_key_id: &str, + ) -> Result<(), GarageError> { + self.post( + "AllowBucketKey", + &serde_json::json!({ + "bucketId": bucket_id, + "accessKeyId": access_key_id, + "permissions": Permissions { + read: true, + write: true, + // Enough to put and get objects. Nothing on this path + // needs to reconfigure the bucket it writes to. + owner: false, + }, + }), + ) + .await?; + Ok(()) + } +} + +/// A secret nobody has to remember, in the shape garage wants. +fn secret() -> String { + let bytes: [u8; 32] = rand::thread_rng().gen(); + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Make a file readable only by its owner. +#[cfg(unix)] +fn restrict(path: &Utf8Path) -> Result<(), GarageError> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| GarageError::Write(path.to_owned(), e)) +} + +#[cfg(not(unix))] +fn restrict(_path: &Utf8Path) -> Result<(), GarageError> { + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + fn settings(dir: Utf8PathBuf) -> Settings { + Settings { + dir, + s3_port: 3900, + admin_port: 3903, + rpc_port: 3901, + capacity: "1G".to_owned(), + } + } + + #[test] + fn a_restart_keeps_the_config_it_already_had() { + // The admin token in the config is the one garage's on-disk state was + // created under. Generating a fresh one on every boot would lock this + // instance out of its own store, and everything in it. + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + std::fs::create_dir_all(&root).expect("mkdir"); + let path = root.join("garage.toml"); + let settings = settings(root); + + let first = ensure_config(&path, &settings).expect("first boot"); + let written = std::fs::read_to_string(&path).expect("read"); + + let second = ensure_config(&path, &settings).expect("second boot"); + + assert_eq!( + first.admin_token, second.admin_token, + "a restart must not mint a new admin token", + ); + assert_eq!( + written, + std::fs::read_to_string(&path).expect("read"), + "a restart must not rewrite the config at all", + ); + } + + #[cfg(unix)] + #[test] + fn the_config_holds_secrets_and_is_kept_to_itself() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + std::fs::create_dir_all(&root).expect("mkdir"); + let path = root.join("garage.toml"); + + ensure_config(&path, &settings(root)).expect("first boot"); + + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); + } + + #[test] + fn every_kind_that_builds_gets_a_bucket() { + // Helios has one before it has anything to put in it, so the day the + // driver build produces something there is nowhere for it to be + // missing. + assert!(KINDS.contains(&"vivado")); + assert!(KINDS.contains(&"helios")); + } +} diff --git a/vw-agent/src/generated.rs b/vw-agent/src/generated.rs new file mode 100644 index 0000000..7ea106c --- /dev/null +++ b/vw-agent/src/generated.rs @@ -0,0 +1,276 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The files vivado generates that a developer's own tools need. +//! +//! Not artifacts in the sense the object store holds — nobody wants to collect +//! these, and they are worthless a week later. They are the VHDL wrappers and +//! stubs that vivado writes for each configured IP, and without them a static +//! analysis of the design cannot resolve `entity ip._wrapper` or +//! `entity xil_defaultlib.`. So they have to come back to the machine the +//! developer's language server is running on, at the same paths, or "go to +//! definition" lands nowhere. +//! +//! They are fetched rather than pushed, and by exact path rather than through +//! the object store, because a check is waiting on them. Somebody typing `vw +//! check` should not be waiting on a poll interval, an upload and a download of +//! something that is sitting in a directory two hops away. + +use camino::{Utf8Path, Utf8PathBuf}; +use vw_api_types_versions::latest::{FileEntry, TreeManifest}; + +/// Where vivado leaves generated VHDL, and how to recognise it. +/// +/// Block design IP gets a wrapper per design under `target/ip`; standalone IP +/// gets a stub deep inside the vivado project's generated sources. The two +/// live nowhere near each other because vivado decides where they go, not us. +const GENERATED: [(&str, &str); 2] = + [("target/ip", ".vhd"), ("target/vw-project", "_stub.vhdl")]; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum GeneratedError { + #[error("'{0}' is not a path this instance will hand out")] + UnsafePath(String), + #[error("no generated file at '{0}'")] + NotFound(String), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), +} + +/// Write the stubs that only exist once something turns vivado's templates +/// into them. +/// +/// Vivado writes an instantiation template per standalone IP; turning that into +/// a black-box entity is a mechanical splice done in Rust, and on a local run +/// it happens right after the vivado pass. On a remote one there is nobody on +/// this side to do it, so it happens here — where the templates are. +/// +/// Idempotent and content-aware, so asking twice costs a directory walk. +pub(crate) fn prepare(root: &Utf8Path) -> usize { + vw_lib::write_ip_stubs_from_templates(root).unwrap_or(0) +} + +/// Every generated file, by the path it should have on the far end. +/// +/// Paths are relative to the workspace, so a client can write each one exactly +/// where its own tools expect to find it. +pub(crate) fn manifest(root: &Utf8Path) -> TreeManifest { + let mut entries = Vec::new(); + + for (directory, suffix) in GENERATED { + collect(root, &root.join(directory), suffix, &mut entries); + } + + entries.sort_by(|a, b| a.path.cmp(&b.path)); + TreeManifest { entries } +} + +/// Walk `directory`, adding every file whose name ends with `suffix`. +/// +/// Recursive because vivado buries a stub five levels inside its project, and +/// the block-design wrappers sit one level down under a directory per design. +/// Nothing else in these trees matches, so the depth costs only the walk. +fn collect( + root: &Utf8Path, + directory: &Utf8Path, + suffix: &str, + into: &mut Vec, +) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + + for entry in entries.flatten() { + let Ok(path) = Utf8PathBuf::from_path_buf(entry.path()) else { + continue; + }; + + if path.is_dir() { + collect(root, &path, suffix, into); + continue; + } + if !path.as_str().ends_with(suffix) { + continue; + } + let Ok(relative) = path.strip_prefix(root) else { + continue; + }; + let Ok(contents) = std::fs::read(&path) else { + continue; + }; + + into.push(FileEntry { + path: relative.to_string(), + digest: vw_sync::digest_bytes(&contents), + executable: false, + }); + } +} + +/// One generated file's contents. +/// +/// The path arrives from a caller and is joined onto this instance's tree, so +/// it is checked before it is used — and checked again against what this +/// instance is actually willing to hand out, so a well formed path to +/// somewhere else on the filesystem is refused too. +pub(crate) fn read( + root: &Utf8Path, + path: &str, +) -> Result, GeneratedError> { + let unsafe_path = || GeneratedError::UnsafePath(path.to_owned()); + + if path.is_empty() || path.starts_with('/') || path.contains('\\') { + return Err(unsafe_path()); + } + for component in path.split('/') { + if component.is_empty() || component == ".." || component == "." { + return Err(unsafe_path()); + } + } + + // Only the places generated VHDL lives, and only files that look like it. + let allowed = GENERATED.iter().any(|(directory, suffix)| { + path.starts_with(&format!("{directory}/")) && path.ends_with(suffix) + }); + if !allowed { + return Err(unsafe_path()); + } + + let full = root.join(path); + if !full.is_file() { + return Err(GeneratedError::NotFound(path.to_owned())); + } + + std::fs::read(&full).map_err(|e| GeneratedError::Read(full, e)) +} + +#[cfg(test)] +mod test { + use super::*; + + fn scratch() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + (dir, root) + } + + fn write(root: &Utf8Path, relative: &str, contents: &str) { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, contents).expect("write"); + } + + /// A tree shaped the way vivado leaves one. + fn generated_tree(root: &Utf8Path) { + write(root, "target/ip/cips/wrapper.vhd", "-- cips"); + write(root, "target/ip/dcmac/wrapper.vhd", "-- dcmac"); + write( + root, + "target/vw-project/metroid/metroid.gen/sources_1/ip/\ + primary_clock/primary_clock_stub.vhdl", + "-- primary_clock", + ); + write( + root, + "target/vw-project/metroid/metroid.gen/sources_1/ip/clk_eth/\ + clk_eth_stub.vhdl", + "-- clk_eth", + ); + } + + #[test] + fn both_kinds_of_generated_vhdl_are_found() { + // Block design wrappers and standalone stubs live nowhere near each + // other, and a check needs both to resolve the design. + let (_dir, root) = scratch(); + generated_tree(&root); + + let paths: Vec = manifest(&root) + .entries + .into_iter() + .map(|entry| entry.path) + .collect(); + + assert_eq!( + paths, + [ + "target/ip/cips/wrapper.vhd", + "target/ip/dcmac/wrapper.vhd", + "target/vw-project/metroid/metroid.gen/sources_1/ip/clk_eth/\ + clk_eth_stub.vhdl", + "target/vw-project/metroid/metroid.gen/sources_1/ip/\ + primary_clock/primary_clock_stub.vhdl", + ], + ); + } + + #[test] + fn the_rest_of_a_vivado_project_is_not_generated_vhdl() { + // The project directory is enormous and almost none of it means + // anything on another machine. + let (_dir, root) = scratch(); + generated_tree(&root); + write( + root.as_ref(), + "target/vw-project/metroid/metroid.xpr", + "proj", + ); + write( + root.as_ref(), + "target/vw-project/metroid/metroid.gen/sources_1/ip/clk_eth/\ + clk_eth.vho", + "template", + ); + write(root.as_ref(), "target/ip/cips/wrapper.dcp", "checkpoint"); + write(root.as_ref(), "target/image/top.pdi", "image"); + + let paths: Vec = manifest(&root) + .entries + .into_iter() + .map(|entry| entry.path) + .collect(); + + assert_eq!(paths.len(), 4, "only the wrappers and stubs: {paths:?}"); + } + + #[test] + fn a_workspace_with_no_generated_ip_has_nothing_to_hand_over() { + let (_dir, root) = scratch(); + assert!(manifest(&root).entries.is_empty()); + } + + #[test] + fn a_generated_file_comes_back_by_path() { + let (_dir, root) = scratch(); + generated_tree(&root); + + let contents = read(&root, "target/ip/cips/wrapper.vhd").expect("read"); + + assert_eq!(contents, b"-- cips"); + } + + #[test] + fn nothing_outside_the_generated_directories_is_handed_out() { + let (_dir, root) = scratch(); + generated_tree(&root); + write(root.as_ref(), "secrets.vhd", "-- not yours"); + write(root.as_ref(), "target/logs/vivado.log", "log"); + + for path in [ + "../secrets.vhd", + "/etc/passwd", + "target/ip/../../secrets.vhd", + // Well formed, inside the tree, and still not ours to serve. + "secrets.vhd", + "target/logs/vivado.log", + // The right directory, the wrong kind of file. + "target/vw-project/metroid/metroid.xpr", + ] { + assert!( + read(&root, path).is_err(), + "'{path}' should have been refused", + ); + } + } +} diff --git a/vw-agent/src/main.rs b/vw-agent/src/main.rs new file mode 100644 index 0000000..b81fef3 --- /dev/null +++ b/vw-agent/src/main.rs @@ -0,0 +1,1090 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Receives source on a vw build instance. +//! +//! One agent serves one environment on one instance. It takes delivery of +//! content from `vw-svc` over the rack's internal network and keeps a +//! directory matching whatever the developer's machine last said it should +//! look like — so vivado, nvc and cargo find ordinary files where they expect +//! them, with no knowledge that any of this happened. +//! +//! It is deliberately incurious. It does not know which developer it is +//! working for, whether they are allowed to be, or what is going to be built: +//! `vw-svc` settles all of that before relaying anything here. + +use camino::Utf8PathBuf; +use clap::Parser; +use dropshot::{ + ApiDescription, ConfigDropshot, HttpError, HttpResponseOk, + HttpResponseUpdatedNoContent, RequestContext, +}; +use slog::{info, o, Drain, Logger}; +use slog_error_chain::InlineErrorChain; +use std::{ + net::{IpAddr, Ipv6Addr, SocketAddr}, + sync::Arc, +}; +use vw_api_types_versions::latest::{ + CommitResult, Credentials, SyncPlan, TreeManifest, +}; +use vw_sync::Store; +use vw_sync_api::{BlobPathParam, EnvironmentPathParam, VwSyncApi}; + +mod artifacts; +mod error; +mod garage; +mod generated; +mod netrc; + +#[derive(Parser)] +#[command(name = "vw-agent")] +#[command(about = "receives source on a vw build instance")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(clap::Subcommand)] +enum Commands { + /// Run the agent + Serve(ServerArgs), + /// Run exactly one testbench into an isolated build directory. + /// + /// Hidden because it is not for people: the batch runner fans one of + /// these out per bench, the same way `vw bench` does on a developer's + /// machine. A child per bench is what makes each one's output separable + /// and each one's build directory its own — `nvc` inherits stdio, so + /// several in one process would interleave beyond recovery. + #[command(hide = true)] + BenchOne(BenchOneArgs), + /// Emit the OpenAPI spec + EmitSpec, +} + +#[derive(Parser, Clone)] +struct ServerArgs { + /// Address to listen on + #[arg(long, default_value_t = IpAddr::V6(Ipv6Addr::UNSPECIFIED))] + address: IpAddr, + + #[arg(long, default_value_t = 2729u16)] + port: u16, + + /// The environment this agent belongs to. + /// + /// Not normally given. An instance belongs to exactly one environment for + /// its whole life, and `vw-svc` already wrote that fact into the + /// instance's hostname when it created it — so the answer is on the + /// machine, and asking for it again only creates a way for the two to + /// disagree. + /// + /// Supply it to run an agent somewhere that is not one of those + /// instances, which in practice means a developer's own machine. + /// + /// Requests naming any other environment are refused however this was + /// arrived at: a request for a different environment is a routing mistake, + /// not something to serve. + #[arg(long)] + environment: Option, + + /// Which of the environment's instances this is. + /// + /// Derived from the hostname alongside the environment, and overridable + /// for the same reason. + #[arg(long)] + kind: Option, + + /// Directory the source tree is kept in. + /// + /// Fixed at startup and never derived from a request, so nothing a caller + /// sends can place a file outside it. + #[arg(long)] + root: Utf8PathBuf, + + /// Directory delivered content waits in before being put in place. + #[arg(long, default_value = "/var/lib/vw-agent/store")] + store: Utf8PathBuf, + + /// Where to remember the object store artifacts are uploaded to. + /// + /// Kept so an instance that reboots between builds still knows where its + /// output goes without being told again. + #[arg(long, default_value = "/var/lib/vw-agent/artifact-target.json")] + artifact_target: Utf8PathBuf, + + /// Directory the object store keeps its config, metadata and data in. + /// + /// Only consulted on an artifact instance, which is the only kind that + /// runs a store. + #[arg(long, default_value = "/var/lib/vw-agent/garage")] + garage_dir: Utf8PathBuf, + + /// Port the object store serves S3 on. + /// + /// Garage's own default, and what every uploader is told to expect. + #[arg(long, default_value_t = 3900u16)] + s3_port: u16, + + /// Port the object store's admin API listens on, bound to localhost. + #[arg(long, default_value_t = 3903u16)] + garage_admin_port: u16, + + /// Port the object store uses to talk to itself. + #[arg(long, default_value_t = 3901u16)] + garage_rpc_port: u16, + + /// How much of this instance's disk the object store may use. + #[arg(long, default_value = "100G")] + garage_capacity: String, + + /// Where to write the credentials a build fetches its dependencies with. + /// + /// Defaults to the `.netrc` of whoever the agent runs as. Provisioning + /// should point this at the home directory of the user builds run as when + /// that is somebody else — `/home/ubuntu/.netrc` on the vivado and + /// artifact instances, where the agent is root and the build is not. + #[arg(long)] + netrc: Option, +} + +#[derive(Parser, Clone)] +struct BenchOneArgs { + /// Workspace to run in. + #[arg(long)] + root: Utf8PathBuf, + /// The testbench entity to run. + #[arg(long)] + name: String, + /// Where nvc should build, relative to the workspace. + #[arg(long)] + build_dir: String, + /// The VHDL standard, as `nvc` spells it. + #[arg(long)] + std: String, +} + +pub struct Context { + environment: String, + /// Where finished artifacts are uploaded, once anyone has said. + artifact_target: tokio::sync::watch::Sender< + Option, + >, + /// Where that answer is kept across restarts. + artifact_target_path: Utf8PathBuf, + /// The object store this instance runs, if it is the one that runs it. + /// + /// Held so garage lives as long as the agent does, and so the key can be + /// handed to whoever asks for it. Distinct from `store` below, which is + /// where delivered source waits — this one is where finished artifacts go. + object_store: Option, + root: Utf8PathBuf, + store: Store, + /// Where the credentials for fetching dependencies are kept. + netrc: Utf8PathBuf, + /// Held while a tree is being made to match a manifest. + /// + /// Two machines synchronizing the same environment is not a conflict worth + /// reporting — the later one wins — but it is worth making sure the loser + /// does not leave half of its tree interleaved with half of the winner's. + materializing: tokio::sync::Mutex<()>, +} + +pub struct Agent {} + +impl VwSyncApi for Agent { + type Context = Arc; + + async fn sync_plan( + rqctx: RequestContext, + path_params: dropshot::Path, + body: dropshot::TypedBody, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let manifest = body.into_inner(); + let plan = vw_sync::missing(&ctx.root, &ctx.store, &manifest) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot work out what is missing"; + InlineErrorChain::new(e), + ); + }) + .map_err(error::apply_error)?; + + info!(rqctx.log, "planned a sync"; + "wanted" => manifest.entries.len(), + "missing" => plan.missing.len(), + ); + + Ok(HttpResponseOk(plan)) + } + + async fn sync_blob( + rqctx: RequestContext, + path_params: dropshot::Path, + body: dropshot::UntypedBody, + ) -> Result { + let ctx = rqctx.context(); + let params = path_params.into_inner(); + ctx.check_environment(¶ms.environment, &rqctx.log)?; + + ctx.store + .put(¶ms.digest, body.as_bytes()) + .inspect_err(|e| { + slog::error!(rqctx.log, "rejected delivered content"; + "digest" => %params.digest, + InlineErrorChain::new(e), + ); + }) + .map_err(error::store_error)?; + + Ok(HttpResponseUpdatedNoContent()) + } + + async fn sync_commit( + rqctx: RequestContext, + path_params: dropshot::Path, + body: dropshot::TypedBody, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + let manifest = body.into_inner(); + + let _guard = ctx.materializing.lock().await; + let result = vw_sync::apply(&ctx.root, &ctx.store, &manifest) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot make the tree match"; + "root" => %ctx.root, + InlineErrorChain::new(e), + ); + }) + .map_err(error::apply_error)?; + + info!(rqctx.log, "tree synchronized"; + "created" => result.created, + "updated" => result.updated, + "deleted" => result.deleted, + "unchanged" => result.unchanged, + ); + + Ok(HttpResponseOk(result)) + } + + async fn sync_clear( + rqctx: RequestContext, + path_params: dropshot::Path, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + // The same lock a commit takes: this is a commit, of an empty + // manifest, and it would be no better to interleave with one than two + // commits would be with each other. + let _guard = ctx.materializing.lock().await; + let result = vw_sync::clear(&ctx.root, &ctx.store) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot clear the tree"; + "root" => %ctx.root, + InlineErrorChain::new(e), + ); + }) + .map_err(error::apply_error)?; + + info!(rqctx.log, "tree cleared"; + "deleted" => result.deleted, + ); + + Ok(HttpResponseOk(result)) + } + + async fn generated_manifest( + rqctx: RequestContext, + path_params: dropshot::Path, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + // Finish vivado's work before reporting it: the stubs do not exist + // until the templates are spliced, and on a remote run this is the + // only place that happens. + let written = generated::prepare(&ctx.root); + let manifest = generated::manifest(&ctx.root); + + info!(rqctx.log, "reporting generated ip"; + "stubs_written" => written, + "files" => manifest.entries.len(), + ); + + Ok(HttpResponseOk(manifest)) + } + + async fn generated_file( + rqctx: RequestContext, + path_params: dropshot::Path, + query: dropshot::Query< + vw_api_types_versions::latest::GeneratedFileQuery, + >, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + let wanted = query.into_inner().path; + + let contents = generated::read(&ctx.root, &wanted) + .inspect_err(|e| { + slog::warn!(rqctx.log, "refusing a generated file"; + "path" => &wanted, + InlineErrorChain::new(e), + ); + }) + .map_err(error::generated_error)?; + + Ok(HttpResponseOk( + dropshot::Body::with_content(contents).into(), + )) + } + + async fn get_artifact_target( + rqctx: RequestContext, + path_params: dropshot::Path, + ) -> Result< + HttpResponseOk, + HttpError, + > { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let current = ctx.artifact_target.borrow().clone(); + let Some(current) = current else { + slog::debug!(rqctx.log, "no artifact target has been set"); + return Err(HttpError::for_not_found( + None, + String::from( + "this instance has not been told where its \ + artifacts go", + ), + )); + }; + + Ok(HttpResponseOk(current)) + } + + async fn put_artifact_target( + rqctx: RequestContext, + path_params: dropshot::Path, + body: dropshot::TypedBody, + ) -> Result { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + let credentials = body.into_inner(); + + artifacts::remember(&ctx.artifact_target_path, &credentials) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot remember the artifact target"; + "path" => %ctx.artifact_target_path, + InlineErrorChain::new(e), + ); + }) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + info!(rqctx.log, "artifacts now go to a store"; + "bucket" => &credentials.bucket, + "endpoint" => &credentials.endpoint, + ); + + // Waking the uploader, which will send anything already built. + let _ = ctx.artifact_target.send(Some(credentials)); + + Ok(HttpResponseUpdatedNoContent()) + } + + async fn get_object_store( + rqctx: RequestContext, + path_params: dropshot::Path, + query: dropshot::Query, + ) -> Result< + HttpResponseOk, + HttpError, + > { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let Some(store) = ctx.object_store.as_ref() else { + slog::warn!( + rqctx.log, + "asked for an object store this instance \ + does not run" + ); + return Err(HttpError::for_not_found( + None, + String::from( + "this instance does not run an object store; the artifact \ + instance does", + ), + )); + }; + + let kind = query + .into_inner() + .kind + .unwrap_or(vw_api_types_versions::latest::TargetKind::Vivado) + .to_string(); + + let Some(bucket) = store.buckets.get(&kind) else { + slog::warn!(rqctx.log, "asked for a bucket that does not exist"; + "kind" => &kind, + ); + return Err(HttpError::for_not_found( + None, + format!("this store has no bucket for '{kind}'"), + )); + }; + + info!(rqctx.log, "handing out the object store key"; + "bucket" => bucket, + ); + + let mut credentials = store.credentials.clone(); + credentials.bucket = bucket.clone(); + + Ok(HttpResponseOk(credentials)) + } + + async fn clean_build_output( + rqctx: RequestContext, + path_params: dropshot::Path, + ) -> Result< + HttpResponseOk, + HttpError, + > { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + // The same lock a commit takes. Removing the build output while a + // tree is being written would not corrupt either — they are disjoint + // — but a build starting in between would see half a world. + let _guard = ctx.materializing.lock().await; + let cleaned = vw_sync::clean(&ctx.root) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot remove the build output"; + "root" => %ctx.root, + InlineErrorChain::new(e), + ); + }) + .map_err(error::apply_error)?; + + info!(rqctx.log, "build output removed"; + "existed" => cleaned.existed, + "bytes" => cleaned.bytes, + ); + + Ok(HttpResponseOk(vw_api_types_versions::latest::CleanResult { + existed: cleaned.existed, + bytes: cleaned.bytes, + })) + } + + async fn driver_build( + rqctx: RequestContext, + path_params: dropshot::Path, + query: dropshot::Query, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let query = query.into_inner(); + let params = vw_remote::BuildParams { + release: query.release, + args: query.arguments(), + }; + + info!(rqctx.log, "building the driver"; + "root" => %ctx.root, + "release" => params.release, + "args" => params.args.join(" "), + ); + + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + websock.into_inner(), + tokio_tungstenite::tungstenite::protocol::Role::Server, + None, + ) + .await; + + // Where the artifacts go is settled before the build starts, so the + // upload can happen the moment cargo is done rather than after the + // developer's command has already returned. + let target = ctx.artifact_target.borrow().clone(); + let root = ctx.root.clone(); + let upload_log = rqctx.log.clone(); + let upload: vw_remote::driver::Uploader = + Box::new(move |produced: Vec| { + let target = target.clone(); + let root = root.clone(); + let log = upload_log.clone(); + Box::pin(async move { + match target { + Some(credentials) => { + artifacts::upload_all( + &root, + &credentials, + &produced, + &log, + ) + .await + } + None => { + slog::warn!( + log, + "nowhere to put the driver's artifacts; this \ + instance has not been told where its store is" + ); + 0 + } + } + }) + }); + + let result = + vw_remote::driver::serve(socket, &ctx.root, params, upload).await; + + match &result { + Ok(produced) => info!(rqctx.log, "driver build finished"; + "artifacts" => produced.len(), + ), + Err(e) => slog::error!(rqctx.log, "driver build failed"; + InlineErrorChain::new(e), + ), + } + + result.map(|_| ()).map_err(Into::into) + } + + async fn bench_session( + rqctx: RequestContext, + path_params: dropshot::Path, + query: dropshot::Query, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let query = query.into_inner(); + // The instance decides how many run at once when the client does not + // say, because the instance is the machine doing the work and knows + // what it has. + let concurrency = + query.concurrency.map(|n| n as usize).unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + }); + let request = vw_bench::Request { + filter: query.filter.clone(), + standard: query + .standard + .clone() + .unwrap_or_else(|| "2019".to_owned()), + concurrency, + ignore: query.ignored(), + }; + + info!(rqctx.log, "running testbenches"; + "root" => %ctx.root, + "filter" => query.filter.as_deref().unwrap_or("-"), + "concurrency" => concurrency, + ); + + // One child per bench, and the child is this same binary. It already + // knows how to run exactly one bench into its own directory, which is + // what the hidden `bench-one` mode is for. + let exe = std::env::current_exe()?; + let root = ctx.root.clone(); + let standard = request.standard.clone(); + let launch: vw_bench::Launch = + std::sync::Arc::new(move |name: &str, build_dir: &str| { + let mut command = tokio::process::Command::new(&exe); + command.args([ + "bench-one", + "--root", + root.as_str(), + "--name", + name, + "--build-dir", + build_dir, + "--std", + &standard, + ]); + command + }); + + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + websock.into_inner(), + tokio_tungstenite::tungstenite::protocol::Role::Server, + None, + ) + .await; + + let result = + vw_remote::bench::serve(socket, &ctx.root, request, launch).await; + + match &result { + Ok(()) => info!(rqctx.log, "testbenches finished"), + Err(e) => slog::error!(rqctx.log, "testbench run failed"; + InlineErrorChain::new(e), + ), + } + + result.map_err(Into::into) + } + + async fn vivado_session( + rqctx: RequestContext, + path_params: dropshot::Path, + query: dropshot::Query< + vw_api_types_versions::latest::VivadoSessionQuery, + >, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + + let query = query.into_inner(); + let params = vw_remote::SessionParams { + part: query.part.clone(), + variant: query.variant.clone(), + info_with_stack: query.info_with_stack, + verbose: query.verbose, + }; + + info!(rqctx.log, "starting a vivado session"; + "root" => %ctx.root, + "part" => query.part.as_deref().unwrap_or("-"), + "variant" => query.variant.as_deref().unwrap_or("-"), + ); + + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + websock.into_inner(), + tokio_tungstenite::tungstenite::protocol::Role::Server, + None, + ) + .await; + + let result = vw_remote::serve(socket, &ctx.root, params).await; + + match &result { + Ok(()) => info!(rqctx.log, "vivado session finished"), + Err(e) => slog::error!(rqctx.log, "vivado session failed"; + InlineErrorChain::new(e), + ), + } + + result.map_err(Into::into) + } + + async fn put_credentials( + rqctx: RequestContext, + path_params: dropshot::Path, + body: dropshot::TypedBody, + ) -> Result { + let ctx = rqctx.context(); + ctx.check_environment( + &path_params.into_inner().environment, + &rqctx.log, + )?; + let credentials = body.into_inner(); + + netrc::write(&ctx.netrc, &credentials) + .inspect_err(|e| { + slog::error!(rqctx.log, "cannot write the credentials file"; + "path" => %ctx.netrc, + InlineErrorChain::new(e), + ); + }) + .map_err(error::netrc_error)?; + + // The login and the path, never the token. + info!(rqctx.log, "credentials in place"; + "user" => &credentials.user, + "path" => %ctx.netrc, + ); + + Ok(HttpResponseUpdatedNoContent()) + } +} + +impl Context { + /// Refuse a request meant for a different environment. + fn check_environment( + &self, + environment: &str, + log: &Logger, + ) -> Result<(), HttpError> { + if environment == self.environment { + return Ok(()); + } + + slog::warn!(log, "request for an environment this agent does not serve"; + "wanted" => environment, + "serving" => &self.environment, + ); + Err(HttpError::for_not_found( + None, + format!( + "this agent serves '{}', not '{environment}'", + self.environment + ), + )) + } +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + match cli.command { + Commands::Serve(args) => serve(args).await, + Commands::BenchOne(args) => bench_one(args).await, + Commands::EmitSpec => emit_spec(), + } +} + +/// Run one testbench and exit with whether it passed. +/// +/// Output goes to this process's own stdout and stderr, where the parent +/// captures it — that is the whole point of being a separate process. +async fn bench_one(args: BenchOneArgs) { + let standard = match args.std.parse::() { + Ok(standard) => standard, + Err(e) => { + eprintln!("{e}"); + std::process::exit(1); + } + }; + + let result = vw_lib::run_testbench( + &args.root, + args.name, + standard, + true, + &[], + false, + false, + &args.build_dir, + ) + .await; + + if let Err(e) = result { + eprintln!("{e}"); + std::process::exit(1); + } +} + +/// The kinds of instance an environment has. +const KINDS: [&str; 3] = ["vivado", "helios", "artifact"]; + +/// What this machine is, according to its own name. +/// +/// `vw-svc` names each instance `-` when it creates it, so +/// a box that calls itself `helios-darmok` is the helios half of `darmok` and +/// there is nothing further to look up. Environment names are validated to +/// contain no hyphen precisely so this splits without ambiguity. +/// +/// Returns nothing when the hostname says nothing — a developer's laptop is +/// not called `vivado-something` — in which case the flags have to say. +fn identity_from_hostname() -> Option<(String, String)> { + read_hostname().as_deref().and_then(parse_identity) +} + +/// This machine's name. +/// +/// The running hostname rather than the file, because the running one is what +/// the machine currently answers to. `/etc/hostname` is the persisted +/// intention and can disagree with it — after a `hostnamectl`, or inside a +/// namespace — and when the two differ the live one is the truth. +fn read_hostname() -> Option { + gethostname::gethostname() + .into_string() + .ok() + .filter(|name| !name.trim().is_empty()) + .or_else(|| std::fs::read_to_string("/etc/hostname").ok()) +} + +/// Split `-` into its parts, if that is what this is. +fn parse_identity(hostname: &str) -> Option<(String, String)> { + let (kind, environment) = hostname.trim().split_once('-')?; + if !KINDS.contains(&kind) || environment.is_empty() { + return None; + } + Some((kind.to_owned(), environment.to_owned())) +} + +async fn serve(args: ServerArgs) { + let bare = logger(); + + // What this instance is, taken from its own name unless told otherwise. + let derived = identity_from_hostname(); + // Which environment is the part nothing can be assumed about — guessing + // would attach this agent to somebody's environment uninvited. + let Some(environment) = args + .environment + .clone() + .or_else(|| derived.as_ref().map(|(_, e)| e.clone())) + else { + slog::error!(bare, "cannot tell which environment this is"; + "detail" => "this machine is not named `-`, \ + which is how vw-svc names an environment's instances. \ + Pass --environment to run an agent somewhere else.", + ); + std::process::exit(1); + }; + + // Which half of it is safe to assume, and vivado is what an agent that was + // never told has always been. + let kind = args + .kind + .clone() + .or_else(|| derived.as_ref().map(|(k, _)| k.clone())) + .unwrap_or_else(|| String::from("vivado")); + + let log = bare.new(o!( + "environment" => environment.clone(), + "kind" => kind.clone(), + )); + if args.environment.is_none() && args.kind.is_none() { + info!(log, "took this instance's identity from its hostname"); + } + + // Both are made now rather than on the first request, so a directory that + // cannot be created is a startup failure naming it rather than a confusing + // error in the middle of somebody's first sync. + for directory in [&args.root, &args.store] { + if let Err(e) = std::fs::create_dir_all(directory) { + slog::error!(log, "cannot create a directory the agent needs"; + "path" => %directory, + InlineErrorChain::new(&e), + ); + std::process::exit(1); + } + } + + // Resolved at startup rather than when the first credentials arrive, so + // an agent with nowhere to put them says so now instead of halfway + // through somebody's first sync. + let netrc = match netrc_path(&args) { + Some(path) => path, + None => { + slog::error!(log, "cannot work out where to write credentials"; + "detail" => "HOME is not set and --netrc was not given", + ); + std::process::exit(1); + } + }; + + // The artifact instance is the one that holds the object store, so it is + // the one that brings it up. Nothing is pre-shared: the admin credential + // is generated here and stays here, and only an S3 key ever leaves. + let store = if kind == "artifact" { + match garage::start( + &environment, + &garage::Settings { + dir: args.garage_dir.clone(), + s3_port: args.s3_port, + admin_port: args.garage_admin_port, + rpc_port: args.garage_rpc_port, + capacity: args.garage_capacity.clone(), + }, + &log, + ) + .await + { + Ok(store) => Some(store), + Err(e) => { + slog::error!(log, "cannot bring up the object store"; + InlineErrorChain::new(&e), + ); + std::process::exit(1); + } + } + } else { + None + }; + + // Where artifacts go, recovered from the last time anyone said. An agent + // that has never been told simply uploads nothing. + let remembered = artifacts::recall(&args.artifact_target) + .inspect_err(|e| { + slog::warn!(log, "cannot read the remembered artifact target"; + InlineErrorChain::new(e), + ); + }) + .unwrap_or(None); + if let Some(target) = &remembered { + info!(log, "artifacts go to a store this instance was told about"; + "bucket" => &target.bucket, + ); + } + let (artifact_target, artifact_changes) = + tokio::sync::watch::channel(remembered); + + // Only where builds happen. An artifact instance holds the store rather + // than filling it, and helios does not produce images. + if kind == "vivado" { + tokio::spawn(artifacts::synchronize( + args.root.clone(), + artifact_changes, + log.new(o!("task" => "artifacts")), + )); + } + + let context = Arc::new(Context { + environment: environment.clone(), + artifact_target, + artifact_target_path: args.artifact_target.clone(), + object_store: store, + root: args.root.clone(), + store: Store::new(args.store.clone()), + netrc: netrc.clone(), + materializing: tokio::sync::Mutex::new(()), + }); + + info!(log, "serving source synchronization"; + "root" => %args.root, + "store" => %args.store, + "netrc" => %netrc, + ); + + let server = + dropshot::ServerBuilder::new(api_description(), context, log.clone()) + .config(ConfigDropshot { + bind_address: SocketAddr::new(args.address, args.port), + // A manifest is one JSON document listing every file in the tree, and + // a blob is one whole source file. Neither is large, but the default + // of a few kilobytes is smaller than either. + default_request_body_max_bytes: 128 * 1024 * 1024, + ..Default::default() + }) + .start(); + + let server = match server { + Ok(server) => server, + Err(e) => { + slog::error!(log, "cannot start the agent"; + InlineErrorChain::new(&e), + ); + std::process::exit(1); + } + }; + + info!(log, "listening on http://{}", server.local_addr()); + + if let Err(e) = server.await { + slog::error!(log, "agent stopped"; "error" => e); + std::process::exit(1); + } +} + +/// Where credentials go, if it can be worked out at all. +fn netrc_path(args: &ServerArgs) -> Option { + if let Some(path) = &args.netrc { + return Some(path.clone()); + } + std::env::var("HOME") + .ok() + .map(|home| Utf8PathBuf::from(home).join(".netrc")) +} + +fn emit_spec() { + let api = api_description(); + let spec = api.openapi("VW agent API", vw_sync_api::latest_version()); + spec.write(&mut std::io::stdout()) + .expect("write spec to stdout"); +} + +pub fn api_description() -> ApiDescription> { + vw_sync_api::vw_sync_api_mod::api_description::() + .expect("the api description is built from a trait that compiles") +} + +fn logger() -> Logger { + let drain = slog_bunyan::new(std::io::stdout()).build().fuse(); + let drain = slog_async::Async::new(drain) + .chan_size(0x8000) + .build() + .fuse(); + Logger::root(drain, o!()) +} + +#[cfg(test)] +mod identity_test { + use super::*; + + #[test] + fn an_instance_knows_itself_by_its_name() { + // Exactly the names `vw-svc` gives the three instances it creates. + assert_eq!( + parse_identity("vivado-darmok"), + Some(("vivado".to_owned(), "darmok".to_owned())), + ); + assert_eq!( + parse_identity("helios-darmok"), + Some(("helios".to_owned(), "darmok".to_owned())), + ); + assert_eq!( + parse_identity("artifact-jalad\n"), + Some(("artifact".to_owned(), "jalad".to_owned())), + ); + } + + #[test] + fn a_machine_that_is_not_one_of_ours_says_so() { + // A developer's own machine, or anything else that happens to have a + // hyphen in its name. Guessing here would attach an agent to an + // environment nobody asked for. + for hostname in + ["runabout", "my-laptop", "vivado", "vivado-", "-darmok", ""] + { + assert!( + parse_identity(hostname).is_none(), + "'{hostname}' should not look like an instance", + ); + } + } +} diff --git a/vw-agent/src/netrc.rs b/vw-agent/src/netrc.rs new file mode 100644 index 0000000..12bb425 --- /dev/null +++ b/vw-agent/src/netrc.rs @@ -0,0 +1,237 @@ +//! Putting the caller's Github credentials where a build will find them. +//! +//! Sources arrive from a developer's machine, but dependencies do not — the +//! instance fetches those itself, and needs credentials to do it. A `.netrc` +//! is how those get supplied because git, cargo and curl all already read one; +//! nothing on the instance has to be taught anything new. + +use camino::{Utf8Path, Utf8PathBuf}; +use vw_api_types_versions::latest::Credentials; + +/// The hosts a build fetches from. +/// +/// `api.github.com` as well as `github.com` because a dependency fetched +/// through the API — a release artifact, a tarball of a tag — goes to the +/// other host and would otherwise be an anonymous request against a private +/// repository. +const HOSTS: [&str; 2] = ["github.com", "api.github.com"]; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum NetrcError { + #[error("creating {0}")] + CreateDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("reading the owner of {0}")] + Owner(Utf8PathBuf, #[source] std::io::Error), +} + +/// Write `credentials` to the netrc at `path`. +/// +/// Replaces whatever was there. A token is reissued or rotated without any +/// ceremony on the instance's part, and a netrc that has fallen behind the +/// caller's real credentials is worth nothing. +pub(crate) fn write( + path: &Utf8Path, + credentials: &Credentials, +) -> Result<(), NetrcError> { + let mut contents = String::new(); + for host in HOSTS { + contents.push_str(&format!( + "machine {host}\n login {}\n password {}\n", + credentials.user, credentials.token, + )); + } + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| NetrcError::CreateDir(parent.to_owned(), e))?; + } + + write_private(path, contents.as_bytes())?; + take_ownership_from_parent(path)?; + + Ok(()) +} + +/// Write `contents` to `path` so that only its owner can ever read it. +/// +/// The permissions are set as the file is created rather than after it is +/// written. Creating it first and fixing the mode afterwards would leave a +/// window — brief, but on a machine several people can reach — in which a live +/// credential sits in a world readable file. +#[cfg(unix)] +fn write_private(path: &Utf8Path, contents: &[u8]) -> Result<(), NetrcError> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + + // Removed rather than truncated, so the mode below applies to a file this + // call created. An existing netrc might have any permissions at all. + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(NetrcError::Write(path.to_owned(), e)), + } + + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .map_err(|e| NetrcError::Write(path.to_owned(), e))?; + + file.write_all(contents) + .map_err(|e| NetrcError::Write(path.to_owned(), e)) +} + +#[cfg(not(unix))] +fn write_private(path: &Utf8Path, contents: &[u8]) -> Result<(), NetrcError> { + std::fs::write(path, contents) + .map_err(|e| NetrcError::Write(path.to_owned(), e)) +} + +/// Give the netrc the same owner as the directory holding it. +/// +/// The agent runs as root, but a build does not: on the vivado and artifact +/// instances it runs as `ubuntu`. A root owned netrc in ubuntu's home is +/// readable by nobody except root — which is to say, unreadable by exactly the +/// process it exists for. Taking ownership from the enclosing directory gets +/// this right on helios too, where both are root and nothing changes. +#[cfg(unix)] +fn take_ownership_from_parent(path: &Utf8Path) -> Result<(), NetrcError> { + use std::os::unix::fs::MetadataExt; + + let Some(parent) = path.parent() else { + return Ok(()); + }; + + let owner = std::fs::metadata(parent) + .map_err(|e| NetrcError::Owner(parent.to_owned(), e))?; + let (uid, gid) = (owner.uid(), owner.gid()); + + let current = std::fs::metadata(path) + .map_err(|e| NetrcError::Owner(path.to_owned(), e))?; + if current.uid() == uid && current.gid() == gid { + return Ok(()); + } + + // Only root may hand a file to somebody else. An agent running as an + // ordinary user in a directory it does not own cannot fix this, and + // failing the whole request over it would be worse than writing a netrc + // that the intended reader may still be able to use. + std::os::unix::fs::chown(path, Some(uid), Some(gid)) + .map_err(|e| NetrcError::Owner(path.to_owned(), e)) +} + +#[cfg(not(unix))] +fn take_ownership_from_parent(_path: &Utf8Path) -> Result<(), NetrcError> { + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + fn credentials() -> Credentials { + Credentials { + user: "picard".to_owned(), + token: "ghp_darmokandjalad".to_owned(), + } + } + + fn scratch() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = camino::Utf8Path::from_path(dir.path()) + .expect("utf8") + .to_owned(); + (dir, root) + } + + #[test] + fn a_netrc_names_both_github_hosts() { + let (_dir, root) = scratch(); + let path = root.join(".netrc"); + + write(&path, &credentials()).expect("write netrc"); + + let contents = std::fs::read_to_string(&path).expect("read netrc"); + assert_eq!( + contents, + "machine github.com\n login picard\n password \ + ghp_darmokandjalad\nmachine api.github.com\n login picard\n \ + password ghp_darmokandjalad\n", + ); + } + + #[cfg(unix)] + #[test] + fn nobody_but_the_owner_can_read_a_netrc() { + use std::os::unix::fs::PermissionsExt; + let (_dir, root) = scratch(); + let path = root.join(".netrc"); + + write(&path, &credentials()).expect("write netrc"); + + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); + } + + #[cfg(unix)] + #[test] + fn a_permissive_netrc_already_there_does_not_stay_permissive() { + use std::os::unix::fs::PermissionsExt; + let (_dir, root) = scratch(); + let path = root.join(".netrc"); + std::fs::write(&path, "old").expect("write"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) + .expect("chmod"); + + write(&path, &credentials()).expect("write netrc"); + + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); + } + + #[test] + fn a_rotated_token_replaces_the_old_one() { + let (_dir, root) = scratch(); + let path = root.join(".netrc"); + write(&path, &credentials()).expect("first"); + + write( + &path, + &Credentials { + user: "picard".to_owned(), + token: "ghp_temba".to_owned(), + }, + ) + .expect("second"); + + let contents = std::fs::read_to_string(&path).expect("read netrc"); + assert!(contents.contains("ghp_temba")); + assert!( + !contents.contains("ghp_darmokandjalad"), + "the old token is still there: {contents}", + ); + } + + #[test] + fn a_home_directory_that_is_not_there_yet_is_created() { + let (_dir, root) = scratch(); + let path = root.join("home/ubuntu/.netrc"); + + write(&path, &credentials()).expect("write netrc"); + + assert!(path.is_file()); + } + + #[test] + fn a_token_is_not_in_the_debug_output() { + // Everything else here is careful about the file; this is about the + // other way a credential escapes, which is a log line. + let printed = format!("{:?}", credentials()); + + assert!(!printed.contains("ghp_darmokandjalad"), "{printed}"); + assert!(printed.contains("picard"), "{printed}"); + } +} diff --git a/vw-agent/tests/sync.rs b/vw-agent/tests/sync.rs new file mode 100644 index 0000000..c719338 --- /dev/null +++ b/vw-agent/tests/sync.rs @@ -0,0 +1,567 @@ +// The agent driven the way `vw-svc` will drive it: over HTTP, against a real +// filesystem, with no shortcuts through the engine underneath. +// +// The engine's own behaviour is covered in `vw-sync`. What is worth checking +// here is everything the HTTP layer adds — that a manifest survives the round +// trip, that content is verified on arrival, that a request for the wrong +// environment is refused, and that a caller who lies about a path or a digest +// gets a refusal rather than a file somewhere it should not be. + +use std::net::TcpListener; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use camino::{Utf8Path, Utf8PathBuf}; +use reqwest::StatusCode; +use tempfile::TempDir; +use vw_api_types_versions::latest::{ + CommitResult, Credentials, Digest, FileEntry, SyncPlan, TreeManifest, +}; + +const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); +const ENVIRONMENT: &str = "darmok"; + +/// A running agent with a tree and a content store of its own. +struct Agent { + child: Child, + base_url: String, + client: reqwest::Client, + _dir: TempDir, + root: Utf8PathBuf, + netrc: Utf8PathBuf, +} + +impl Agent { + async fn start() -> Agent { + let dir = TempDir::new().expect("scratch directory"); + let base = Utf8Path::from_path(dir.path()).expect("utf8 temp dir"); + let root = base.join("tree"); + let netrc = base.join("home/.netrc"); + let port = free_port(); + + let child = Command::new(env!("CARGO_BIN_EXE_vw-agent")) + .arg("serve") + .args(["--address", "127.0.0.1"]) + .args(["--port", &port.to_string()]) + .args(["--environment", ENVIRONMENT]) + .args(["--root", root.as_str()]) + .args(["--store", base.join("store").as_str()]) + .args(["--netrc", netrc.as_str()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn vw-agent"); + + let mut agent = Agent { + child, + base_url: format!("http://127.0.0.1:{port}"), + client: reqwest::Client::new(), + _dir: dir, + root, + netrc, + }; + agent.wait_until_ready().await; + agent + } + + async fn wait_until_ready(&mut self) { + let deadline = Instant::now() + STARTUP_TIMEOUT; + loop { + if self + .plan_raw(ENVIRONMENT, &TreeManifest::default()) + .await + .is_ok() + { + return; + } + if let Some(status) = + self.child.try_wait().expect("check on vw-agent") + { + panic!("vw-agent exited during startup: {status}"); + } + assert!( + Instant::now() < deadline, + "vw-agent never started listening on {}", + self.base_url, + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + async fn plan_raw( + &self, + environment: &str, + manifest: &TreeManifest, + ) -> reqwest::Result { + self.client + .post(format!( + "{}/environment/{environment}/sync/plan", + self.base_url + )) + .json(manifest) + .send() + .await + } + + async fn plan(&self, manifest: &TreeManifest) -> SyncPlan { + let response = self + .plan_raw(ENVIRONMENT, manifest) + .await + .expect("plan request"); + assert_eq!(response.status(), StatusCode::OK); + response.json().await.expect("decode plan") + } + + async fn put_blob( + &self, + digest: &Digest, + contents: &[u8], + ) -> reqwest::Response { + self.client + .put(format!( + "{}/environment/{ENVIRONMENT}/sync/blob/{digest}", + self.base_url + )) + .body(contents.to_vec()) + .send() + .await + .expect("blob request") + } + + async fn commit_raw(&self, manifest: &TreeManifest) -> reqwest::Response { + self.client + .post(format!( + "{}/environment/{ENVIRONMENT}/sync/commit", + self.base_url + )) + .json(manifest) + .send() + .await + .expect("commit request") + } + + async fn clear_raw(&self, environment: &str) -> reqwest::Response { + self.client + .delete(format!("{}/environment/{environment}/sync", self.base_url)) + .send() + .await + .expect("clear request") + } + + async fn clear(&self) -> CommitResult { + let response = self.clear_raw(ENVIRONMENT).await; + assert_eq!(response.status(), StatusCode::OK); + response.json().await.expect("decode clear result") + } + + async fn put_credentials( + &self, + environment: &str, + credentials: &Credentials, + ) -> reqwest::Response { + self.client + .put(format!( + "{}/environment/{environment}/credentials", + self.base_url + )) + .json(credentials) + .send() + .await + .expect("credentials request") + } + + async fn clean_raw(&self, environment: &str) -> reqwest::Response { + self.client + .delete(format!( + "{}/environment/{environment}/build-output", + self.base_url + )) + .send() + .await + .expect("clean request") + } + + async fn commit(&self, manifest: &TreeManifest) -> CommitResult { + let response = self.commit_raw(manifest).await; + assert_eq!(response.status(), StatusCode::OK); + response.json().await.expect("decode commit result") + } + + /// A full synchronization of `files`, returning what the commit did and + /// how many blobs actually crossed the wire. + async fn sync(&self, files: &[(&str, &str)]) -> (CommitResult, usize) { + let manifest = manifest_of(files); + let plan = self.plan(&manifest).await; + + for digest in &plan.missing { + let contents = files + .iter() + .find(|(_, body)| { + vw_sync::digest_bytes(body.as_bytes()) == *digest + }) + .expect("the plan asked for something in the manifest") + .1; + let response = self.put_blob(digest, contents.as_bytes()).await; + assert_eq!(response.status(), StatusCode::NO_CONTENT); + } + + (self.commit(&manifest).await, plan.missing.len()) + } + + fn contents(&self, path: &str) -> String { + std::fs::read_to_string(self.root.join(path)) + .unwrap_or_else(|e| panic!("reading {path}: {e}")) + } + + fn paths(&self) -> Vec { + let mut paths: Vec = vw_sync::scan(&self.root) + .expect("scan tree") + .entries + .into_iter() + .map(|entry| entry.path) + .collect(); + paths.sort(); + paths + } +} + +impl Drop for Agent { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn manifest_of(files: &[(&str, &str)]) -> TreeManifest { + TreeManifest { + entries: files + .iter() + .map(|(path, contents)| FileEntry { + path: (*path).to_owned(), + digest: vw_sync::digest_bytes(contents.as_bytes()), + executable: false, + }) + .collect(), + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("read ephemeral port") + .port() +} + +#[tokio::test] +async fn a_tree_arrives_over_http() { + let agent = Agent::start().await; + + let (result, uploaded) = agent + .sync(&[ + ("hdl/top.vhd", "entity top is end;"), + ("vw.toml", "[workspace]"), + ]) + .await; + + assert_eq!(uploaded, 2); + assert_eq!(result.created, 2); + assert_eq!(agent.paths(), ["hdl/top.vhd", "vw.toml"]); + assert_eq!(agent.contents("hdl/top.vhd"), "entity top is end;"); +} + +#[tokio::test] +async fn a_second_sync_of_the_same_tree_sends_nothing() { + let agent = Agent::start().await; + let files = [("hdl/top.vhd", "entity top is end;")]; + + agent.sync(&files).await; + let (result, uploaded) = agent.sync(&files).await; + + assert_eq!(uploaded, 0); + assert_eq!(result.unchanged, 1); +} + +#[tokio::test] +async fn an_edit_sends_only_the_edited_file() { + let agent = Agent::start().await; + agent + .sync(&[ + ("hdl/top.vhd", "entity top is end;"), + ("hdl/other.vhd", "entity other is end;"), + ]) + .await; + + let (result, uploaded) = agent + .sync(&[ + ("hdl/top.vhd", "entity top is end; -- edited"), + ("hdl/other.vhd", "entity other is end;"), + ]) + .await; + + assert_eq!(uploaded, 1); + assert_eq!(result.updated, 1); + assert_eq!(result.unchanged, 1); +} + +#[tokio::test] +async fn a_rename_crosses_no_wire() { + let agent = Agent::start().await; + let body = "x".repeat(50_000); + agent.sync(&[("hdl/big.vhd", body.as_str())]).await; + + let (result, uploaded) = + agent.sync(&[("hdl/renamed.vhd", body.as_str())]).await; + + assert_eq!(uploaded, 0, "the content is already on the instance"); + assert_eq!(result.created, 1); + assert_eq!(result.deleted, 1); + assert_eq!(agent.paths(), ["hdl/renamed.vhd"]); +} + +#[tokio::test] +async fn content_that_does_not_match_its_digest_is_refused() { + let agent = Agent::start().await; + let honest = vw_sync::digest_bytes(b"the real thing"); + + let response = agent.put_blob(&honest, b"something else").await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn a_digest_that_is_not_a_digest_is_refused() { + let agent = Agent::start().await; + + // The digest becomes a filename in the content store, so anything that is + // not 64 hex characters has to stop at the door. + // + // `..` is not in this list because it never reaches a handler: URL + // normalization removes dot segments before routing, so the request + // arrives at a path that matches nothing. The store refuses it anyway, and + // `vw-sync` tests that directly — this is about what survives the wire. + for hostile in ["not-hex", &"f".repeat(63), &"F".repeat(64), "0"] { + let response = agent + .put_blob(&Digest(hostile.to_owned()), b"payload") + .await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "'{hostile}' should be refused", + ); + } +} + +#[tokio::test] +async fn a_manifest_cannot_write_outside_the_tree() { + let agent = Agent::start().await; + let payload = "payload"; + let digest = vw_sync::digest_bytes(payload.as_bytes()); + agent.put_blob(&digest, payload.as_bytes()).await; + + for path in ["../escaped.vhd", "/etc/passwd", "hdl/../../escaped.vhd"] { + let response = agent.commit_raw(&manifest_of(&[(path, payload)])).await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "'{path}' should be refused", + ); + } +} + +#[tokio::test] +async fn committing_before_delivering_is_refused() { + let agent = Agent::start().await; + + // Nothing has been uploaded, so the commit cannot be satisfied. It should + // say so rather than write a tree with holes in it. + let response = agent + .commit_raw(&manifest_of(&[("hdl/top.vhd", "entity top is end;")])) + .await; + + assert_eq!(response.status(), StatusCode::CONFLICT); + assert!(agent.paths().is_empty()); +} + +#[tokio::test] +async fn a_request_for_another_environment_is_refused() { + let agent = Agent::start().await; + + // An agent belongs to one environment. Being asked about a different one + // means something upstream routed badly, and serving it would put one + // developer's source on another's instance. + let response = agent + .plan_raw("jalad", &TreeManifest::default()) + .await + .expect("plan request"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn a_build_output_directory_survives_synchronization() { + let agent = Agent::start().await; + agent.sync(&[("hdl/top.vhd", "entity top is end;")]).await; + + // What a build leaves behind afterwards. + std::fs::create_dir_all(agent.root.join("target/synth")).expect("mkdir"); + std::fs::write(agent.root.join("target/synth/top.dcp"), "checkpoint") + .expect("write"); + + agent + .sync(&[("hdl/top.vhd", "entity top is end; -- edited")]) + .await; + + assert_eq!(agent.contents("target/synth/top.dcp"), "checkpoint"); +} + +#[tokio::test] +async fn a_cleared_agent_asks_for_the_whole_tree_again() { + let agent = Agent::start().await; + let files = [ + ("hdl/top.vhd", "entity top is end;"), + ("vw.toml", "[workspace]"), + ]; + agent.sync(&files).await; + + // What `vw cloud sync --force` does: rather than argue with the instance + // about what it has, leave it with nothing to argue about. + let cleared = agent.clear().await; + assert_eq!(cleared.deleted, 2); + assert!(agent.paths().is_empty()); + + let (result, uploaded) = agent.sync(&files).await; + assert_eq!(uploaded, 2, "everything should be sent again"); + assert_eq!(result.created, 2); + assert_eq!(result.unchanged, 0); + assert_eq!(agent.contents("hdl/top.vhd"), "entity top is end;"); +} + +#[tokio::test] +async fn clearing_leaves_a_build_where_it_stands() { + let agent = Agent::start().await; + agent.sync(&[("hdl/top.vhd", "entity top is end;")]).await; + std::fs::create_dir_all(agent.root.join("target/synth")).expect("mkdir"); + std::fs::write(agent.root.join("target/synth/top.dcp"), "checkpoint") + .expect("write"); + + agent.clear().await; + + assert_eq!(agent.contents("target/synth/top.dcp"), "checkpoint"); +} + +#[tokio::test] +async fn clearing_another_environment_is_refused() { + let agent = Agent::start().await; + agent.sync(&[("hdl/top.vhd", "entity top is end;")]).await; + + // Of everything relayed here this is the one worth being surest about: + // serving it would delete a developer's tree on somebody else's say-so. + let response = agent.clear_raw("jalad").await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(agent.paths(), ["hdl/top.vhd"]); +} + +fn credentials(token: &str) -> Credentials { + Credentials { + user: "picard".to_owned(), + token: token.to_owned(), + } +} + +#[tokio::test] +async fn credentials_land_where_a_build_will_find_them() { + let agent = Agent::start().await; + + let response = agent + .put_credentials(ENVIRONMENT, &credentials("ghp_darmokandjalad")) + .await; + + assert_eq!(response.status(), StatusCode::NO_CONTENT); + let contents = std::fs::read_to_string(&agent.netrc).expect("read netrc"); + assert!(contents.contains("machine github.com"), "{contents}"); + assert!(contents.contains("login picard"), "{contents}"); + assert!( + contents.contains("password ghp_darmokandjalad"), + "{contents}" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_credentials_file_is_readable_only_by_its_owner() { + use std::os::unix::fs::PermissionsExt; + let agent = Agent::start().await; + + agent + .put_credentials(ENVIRONMENT, &credentials("ghp_darmokandjalad")) + .await; + + let mode = std::fs::metadata(&agent.netrc) + .expect("stat netrc") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); +} + +#[tokio::test] +async fn credentials_for_another_environment_are_refused() { + let agent = Agent::start().await; + + // The instance would otherwise take a stranger's token and hand it to + // whatever it builds next. + let response = agent + .put_credentials("jalad", &credentials("ghp_darmokandjalad")) + .await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!agent.netrc.exists()); +} + +#[tokio::test] +async fn clearing_a_tree_does_not_take_the_credentials_with_it() { + // `--force` throws away the source tree. Credentials are not source, and + // an instance that lost them would fail its next fetch for no reason the + // developer could see. + let agent = Agent::start().await; + agent + .put_credentials(ENVIRONMENT, &credentials("ghp_darmokandjalad")) + .await; + agent.sync(&[("hdl/top.vhd", "entity top is end;")]).await; + + agent.clear().await; + + assert!(agent.netrc.is_file()); +} + +#[tokio::test] +async fn build_output_can_be_removed_over_http() { + let agent = Agent::start().await; + agent.sync(&[("hdl/top.vhd", "entity top is end;")]).await; + std::fs::create_dir_all(agent.root.join("target/synth")).expect("mkdir"); + std::fs::write(agent.root.join("target/synth/top.dcp"), "checkpoint") + .expect("write"); + + let response = agent.clean_raw(ENVIRONMENT).await; + + assert_eq!(response.status(), StatusCode::OK); + let result: serde_json::Value = response.json().await.expect("decode"); + assert_eq!(result["existed"], true); + assert!(result["bytes"].as_u64().expect("bytes") > 0); + assert!(!agent.root.join("target").exists()); + // The source is still here, so the next sync has nothing to do. + assert_eq!(agent.paths(), ["hdl/top.vhd"]); +} + +#[tokio::test] +async fn cleaning_another_environment_is_refused() { + let agent = Agent::start().await; + std::fs::create_dir_all(agent.root.join("target")).expect("mkdir"); + std::fs::write(agent.root.join("target/keep.me"), "output").expect("write"); + + let response = agent.clean_raw("jalad").await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(agent.root.join("target/keep.me").exists()); +} diff --git a/vw-analyzer/Cargo.toml b/vw-analyzer/Cargo.toml new file mode 100644 index 0000000..392b965 --- /dev/null +++ b/vw-analyzer/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "vw-analyzer" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Multi-language LSP server for the vw HDL workflow (htcl native; VHDL via vhdl_ls proxy in a later phase)" + +[[bin]] +name = "vw-analyzer" +path = "src/main.rs" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +vw-lib = { path = "../vw-lib" } +camino.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +async-trait.workspace = true +tower-lsp.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +vhdl_lang.workspace = true +vhdl_ls.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-analyzer/src/backend.rs b/vw-analyzer/src/backend.rs new file mode 100644 index 0000000..7973b2d --- /dev/null +++ b/vw-analyzer/src/backend.rs @@ -0,0 +1,225 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! [`LanguageBackend`] — per-language analysis surface consumed by the +//! LSP server. +//! +//! Even though only [`HtclBackend`](crate::HtclBackend) exists today, +//! defining the trait from day one is the architectural commitment +//! described in the project plan: VHDL via a `vhdl_ls` proxy (phase 5) +//! and a future direct Oxide-VHDL-frontend integration both slot in as +//! additional implementations without changing the server or +//! cross-language htcl code. + +use async_trait::async_trait; +use tower_lsp::lsp_types::{ + CompletionItem, Diagnostic, DidChangeWatchedFilesParams, DocumentSymbol, + Hover, Location, Position, SignatureHelp, SymbolInformation, Url, + WorkspaceEdit, +}; + +#[async_trait] +pub trait LanguageBackend: Send + Sync { + /// Language id (`"htcl"`, `"vhdl"`, ...) — used for tracing and + /// dispatch. + fn language_id(&self) -> &str; + + /// Whether this backend handles the given file. Default: match by + /// extension. + fn handles(&self, uri: &Url) -> bool; + + /// Update the backend's view of `uri`'s contents. Called on + /// `did_open` and every `did_change`. The backend should treat + /// this as the new authoritative source and may eagerly compute + /// (and cache) analysis results. + async fn set_text(&self, uri: Url, text: String); + + /// Trigger an immediate re-index of `uri` — no debounce, no + /// wait. Called from `did_save` so `Ctrl-s` in the editor + /// forces a fresh diagnostic sweep even after a small edit + /// that hadn't yet reached the `set_text` debounce window. + /// Backends without a debounce can leave this as the no-op + /// default: their `set_text` already committed synchronously. + async fn save(&self, _uri: &Url) {} + + /// Block until the next full re-index / re-analysis of `uri` + /// commits. Called by the server AFTER `set_text` from a + /// detached task so it can wrap the wait in an LSP + /// `window/workDoneProgress` notification — that's how the + /// editor's "indexing…" spinner comes back on. + /// + /// Default implementation returns immediately (backends without + /// a background reindex signal don't have anything to wait on; + /// the progress spinner just flashes briefly and disappears). + /// Backends that do have a background rebuild (like + /// [`crate::HtclBackend`]) override this to await their + /// indexer's commit notification. + async fn wait_for_reindex(&self, _uri: &Url) {} + + /// Whether this backend publishes diagnostics itself (via a + /// side channel — e.g. an embedded `vhdl_ls::VHDLServer` + /// bridged into the LSP `Client` through a custom + /// `RpcChannel`). + /// + /// The default `false` matches the `HtclBackend` model: + /// diagnostics are pulled through + /// [`LanguageBackend::diagnostics`] after every text edit and + /// the LSP server-side plumbing publishes them to the client. + /// + /// Backends that return `true` are trusted to have already + /// fired `textDocument/publishDiagnostics` before their async + /// LSP-notification handlers return; the outer server will + /// not call `diagnostics()` after `set_text` for those URIs + /// and will not clobber the freshly-pushed diagnostics with a + /// pull-based empty vec. + fn pushes_diagnostics(&self) -> bool { + false + } + + /// Editor-supplied workspace roots (from LSP `rootUri` / + /// `workspaceFolders`, plus updates via + /// `didChangeWorkspaceFolders`). Backends may use them as + /// fallback dep-lookup sources when a file being analyzed sits + /// outside the nearest `vw.toml` — e.g. a goto-def landed the + /// user in a dep cache dir whose own workspace doesn't declare + /// the same deps as the editor's root. Default impl is a no-op + /// because most backends won't care. + async fn set_workspace_roots(&self, _roots: Vec) {} + + /// The editor observed one of the paths this backend registered + /// as a "watched file" changing on disk. Called from + /// [`LanguageServer::did_change_watched_files`]. Backends that + /// don't register watched-file patterns can leave the default + /// no-op in place. + /// + /// Used by the VHDL backend to re-render its `vhdl_lang::Config` + /// whenever the workspace's `vw.toml`, `vw.lock`, or + /// `ip/**/*.htcl` layout shifts (e.g. a shell-side `vw update` + /// completes): the workspace's `VHDLServer` gets `set_config`'d + /// with the fresh enumeration, no editor restart required. + async fn did_change_watched_files( + &self, + _params: &DidChangeWatchedFilesParams, + ) { + } + + /// Forget any state for `uri`. + async fn close(&self, uri: &Url); + + /// Diagnostics for the current text of `uri`. The server pushes + /// these to the editor via `textDocument/publishDiagnostics` after + /// every text update. + async fn diagnostics(&self, uri: &Url) -> Vec; + + /// Workspace-wide diagnostics, keyed by file URI. The server + /// serves these to the editor via `workspace/diagnostic` (LSP + /// 3.17 pull-based workspace diagnostics — Helix's `space-D` + /// picker consumes them). + /// + /// Backends compute these from the workspace view attached to + /// every open document — a validator diagnostic that landed in + /// an imported file's region gets routed back to that file's + /// URI. Files that no open document transitively `src`s stay + /// silent; that's a soft edge of the model but a good default + /// (an isolated file with no reachable entry is unlikely to be + /// what the user is looking for). + /// + /// Default impl returns empty so backends without workspace- + /// diagnostic support don't have to stub it out. + async fn workspace_diagnostics(&self) -> Vec<(Url, Vec)> { + Vec::new() + } + + /// Document symbols ("outline view") for `uri`. + async fn document_symbols(&self, uri: &Url) -> Vec; + + /// Workspace-wide symbol search for the LSP `workspace/symbol` + /// request. `query` is the user's filter text; the backend should + /// return at minimum every symbol whose name matches `query` (case- + /// insensitive substring is fine — the editor applies its own fuzzy + /// scoring on top). An empty `query` should return every symbol the + /// backend knows about (capped at a sensible upper bound to keep + /// the response small enough to render). + async fn workspace_symbols(&self, query: &str) -> Vec; + + /// Hover content for the construct at `position`. Returns `None` + /// if the cursor isn't on anything the backend has something to + /// say about. + async fn hover(&self, uri: &Url, position: Position) -> Option; + + /// Definition site for the reference at `position`. Returns + /// `None` if the cursor isn't on a known reference. Returns + /// possibly multiple locations because, in general, a name may + /// have several defining sites (overloads, conditional + /// definitions); the Phase 2 htcl backend only returns one. + async fn goto_definition( + &self, + uri: &Url, + position: Position, + ) -> Vec; + + /// Completion items for the cursor at `position`. Empty when the + /// backend has nothing to offer in that context. + async fn completion( + &self, + uri: &Url, + position: Position, + ) -> Vec; + + /// Signature help for the call enclosing `position`. `None` when + /// the cursor isn't inside a call the backend recognizes. + async fn signature_help( + &self, + uri: &Url, + position: Position, + ) -> Option; + + /// Compute the workspace edit that renames the identifier at + /// `position` to `new_name`. `None` when the cursor isn't on a + /// renamable symbol, when `new_name` is invalid for the target + /// language, or when the rename would need to touch symbols the + /// backend can't safely reach (e.g. cross-file references). + /// + /// Default `None` so language backends without rename support + /// (or where rename hasn't landed yet) don't have to stub the + /// method out; the server treats the response as "not + /// supported here" and the editor shows an unobtrusive error. + async fn rename( + &self, + _uri: &Url, + _position: Position, + _new_name: &str, + ) -> Option { + None + } + + /// All locations that reference the symbol at `position`, + /// including `position`'s own location. `include_declaration` + /// mirrors the LSP `ReferenceContext` flag — when `false` the + /// backend should omit the decl span from the response. + /// + /// Returns an empty vec when the cursor isn't on a known + /// symbol. Default impl returns empty so backends without a + /// references implementation don't have to stub it out. + async fn references( + &self, + _uri: &Url, + _position: Position, + _include_declaration: bool, + ) -> Vec { + Vec::new() + } +} + +/// A symbol surfaced from a backend, language-neutral. Backends that +/// need richer fields can build [`DocumentSymbol`] directly; this is +/// a convenience for the common cases that fit a flat name+kind+span. +#[derive(Clone, Debug)] +pub struct SymbolInfo { + pub name: String, + pub kind: tower_lsp::lsp_types::SymbolKind, + pub detail: Option, + pub range: tower_lsp::lsp_types::Range, + pub selection_range: tower_lsp::lsp_types::Range, +} diff --git a/vw-analyzer/src/htcl_backend.rs b/vw-analyzer/src/htcl_backend.rs new file mode 100644 index 0000000..9b9b209 --- /dev/null +++ b/vw-analyzer/src/htcl_backend.rs @@ -0,0 +1,4029 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl [`LanguageBackend`] — native, in-process, using `vw-htcl`. + +use std::collections::HashMap; +use std::fmt::Write; +use std::sync::Arc; + +use async_trait::async_trait; +use tokio::sync::{watch, RwLock}; +use tower_lsp::lsp_types::{ + CompletionItem, CompletionItemKind, Diagnostic, DiagnosticSeverity, + DocumentSymbol, Documentation, Hover, HoverContents, InsertTextFormat, + Location, MarkupContent, MarkupKind, ParameterInformation, ParameterLabel, + Position, Range, SignatureHelp, SignatureInformation, SymbolInformation, + SymbolKind, TextEdit, Url, WorkspaceEdit, +}; +use tracing::debug; +use vw_htcl::{ + definition_at, find_references_in, hover_at, identify_at, parse, + signature_help_at, validate_with_all_extras_and_vars, Attribute, + AttributeValue, CommandKind, Completion, CompletionKind, HoverTarget, + LineCol, LineIndex, ParseOutput, ProcArg, ProcSignature, ReferenceTarget, + Severity, Span, Stmt, +}; + +use crate::backend::LanguageBackend; + +#[derive(Default)] +pub struct HtclBackend { + docs: Arc>>, + /// Editor-supplied workspace roots (LSP `rootUri` / + /// `workspaceFolders`). Consulted as a fallback when the file + /// currently being analyzed sits outside the enclosing + /// `vw.toml` — e.g. after a goto-def has taken the user into a + /// dep-cache dir. Without this, dep names declared only in the + /// editor-root workspace fail to resolve and every `@name/…` + /// import in the visited file goes dead. + workspace_roots: Arc>>, +} + +/// Cached analysis for one open document. Populated by the +/// background indexer spawned from `set_text`, read by every +/// request handler. Parses + workspace-view + diagnostics all +/// bundled together — the source strings live in +/// `view.view_source` and `local_text`, and every span in +/// `parsed_view`/`parsed_local` indexes into them, so passing the +/// whole `Arc` around keeps span interpretation safe. +pub(crate) struct DocAnalysis { + /// Document text at index time — sources for every span in + /// `parsed_local`, and for line/col translation via + /// `local_line_index`. + pub local_text: String, + /// Concatenated workspace view (local text + every + /// transitively `src`d file). Source for spans in + /// `parsed_view` and for `line_index`. + pub view: crate::workspace::WorkspaceView, + pub parsed_local: ParseOutput, + pub parsed_view: ParseOutput, + pub local_line_index: LineIndex, + /// Diagnostics computed at index time — parse errors from the + /// local doc plus validator errors from the workspace view + /// filtered to local-file spans. Served verbatim by + /// `LanguageBackend::diagnostics`. + pub diagnostics: Vec, + /// Same validator errors, but for spans that land in + /// TRANSITIVELY-imported files. Each entry is + /// `(origin_file_uri, diagnostic_with_file_local_range)`. + /// The `workspace/diagnostic` handler routes these back to the + /// files that actually contain the error, giving the editor a + /// workspace-wide picker (`space-D` in Helix) even for files + /// the user hasn't opened. + pub cross_file_diagnostics: Vec<(Url, Diagnostic)>, +} + +struct DocState { + text: String, + /// Monotonic per-URI counter bumped on every `set_text`. The + /// spawned indexer captures the generation it was created for + /// and only pushes its result to the watch channel if the + /// counter still matches — newer set_text having bumped it + /// means the newer indexer will supersede us. + generation: u64, + /// Watch channel carrying the latest completed analysis. + /// `None` while indexing is in flight; `Some(Arc<..>)` once + /// the indexer commits. `set_text` sends `None` to invalidate. + /// Request handlers subscribe and `.changed().await` until + /// `borrow()` returns `Some`. + tx: watch::Sender>>, + /// Handle to the in-flight indexer. `set_text` aborts the + /// previous handle before spawning a new one — keeps only ONE + /// index running at a time so rapid typing doesn't back up + /// (each keystroke's stale index runs to completion under the + /// generation guard, but the ABORT drops it at the next .await + /// point which shortens the wall-clock for the freshest text). + index_task: Option>, +} + +impl HtclBackend { + pub fn new() -> Self { + Self::default() + } + + /// Test-only convenience: update text and wait for the indexer + /// to commit. Production `set_text` is deliberately fire-and- + /// forget so keystrokes never block; tests want synchronous + /// behavior so their assertions run against a committed + /// analysis. + #[cfg(test)] + pub(crate) async fn set_text_sync(&self, uri: Url, text: String) { + use crate::backend::LanguageBackend as _; + self.set_text(uri.clone(), text).await; + self.wait_for_reindex(&uri).await; + } + + /// Snapshot of the editor-supplied workspace roots. Callers + /// pass this into [`crate::workspace::build_view`] etc. as + /// fallback dep-lookup roots — see `workspace_roots` + /// on the struct for the rationale. Cloned so the lock isn't + /// held across the (potentially I/O-heavy) view build. + async fn workspace_roots_snapshot(&self) -> Vec { + self.workspace_roots.read().await.clone() + } + + /// Preload analyses for every entry point `vw check` would + /// discover under each workspace root. These files land in + /// the docs map as "virtual-open" entries — the editor never + /// sent `did_open` for them, but their committed analysis + /// participates in `workspace_diagnostics` the same way an + /// actually-open file does. That's what lets space-D show + /// warnings in files the user hasn't visited yet. + /// + /// Entry-point set mirrors `vw-cli`'s `discover_check_targets`: + /// `/design.htcl`, `/module.htcl`, `/ip/module.htcl`, + /// and every discovered `bench/**/*.htcl` test. If an entry is + /// missing on disk (empty workspace) or the read fails, + /// silently skip that entry — a preload failure must never + /// block LSP startup or hide the on-open editor experience for + /// files that DO exist. + /// + /// Skips entries already in the docs map so this can be called + /// repeatedly (e.g. `did_change_workspace_folders`) without + /// clobbering the live buffer of a file the user is editing. + async fn preload_workspace_targets(&self, roots: &[std::path::PathBuf]) { + use crate::backend::LanguageBackend as _; + for root in roots { + let root_utf8 = + match camino::Utf8PathBuf::from_path_buf(root.clone()) { + Ok(p) => p, + Err(_) => continue, + }; + let ws = match vw_lib::find_workspace_dir(root_utf8.as_std_path()) { + Some(ws) => ws, + None => continue, + }; + let mut targets: Vec = Vec::new(); + if let Some(design) = vw_lib::find_design_file(&ws) { + targets.push(design.into_std_path_buf()); + } + let module = ws.join("module.htcl"); + if module.is_file() { + targets.push(module.into_std_path_buf()); + } + let ip_module = ws.join("ip/module.htcl"); + if ip_module.is_file() { + targets.push(ip_module.into_std_path_buf()); + } + if let Ok(tests) = vw_lib::list_htcl_tests(&ws) { + targets.extend(tests); + } + for target in targets { + let Ok(uri) = Url::from_file_path(&target) else { + continue; + }; + { + let docs = self.docs.read().await; + if docs.contains_key(&uri) { + continue; + } + } + let Ok(text) = std::fs::read_to_string(&target) else { + continue; + }; + debug!(%uri, "preloading workspace target"); + self.set_text(uri, text).await; + } + } + } + + /// Snapshot the current in-memory text for `uri`. Used by + /// completion / hover / signature-help handlers that need the + /// CURRENT text (what the user just typed) rather than the + /// analysis snapshot's stale copy. Cheap: one hashmap read + a + /// String clone. + pub(crate) async fn current_text(&self, uri: &Url) -> Option { + let docs = self.docs.read().await; + docs.get(uri).map(|s| s.text.clone()) + } + + /// Resolve a `src` import path from `entry_file`'s directory, + /// honoring the editor-supplied root fallback. + async fn resolve_import( + &self, + entry_file: &std::path::Path, + raw: &str, + ) -> Option { + let roots = self.workspace_roots_snapshot().await; + crate::workspace::resolve_import(entry_file, raw, &roots) + } + + /// Return the cached analysis for `uri`, awaiting the in-flight + /// indexer if one is currently running. Returns `None` when the + /// document isn't tracked (never `set_text`'d or already + /// `close`d). + /// + /// Contract with `set_text`: every time a new set_text fires + /// on this URI, the watch channel receives `Some(...)` only + /// after the indexer for THAT set_text's text commits under + /// the generation guard. So `analysis_for` naturally waits for + /// the LATEST index. Older in-flight indexers that finish + /// under a stale generation are silently discarded — waiters + /// don't get bogus results. + pub(crate) async fn analysis_for( + &self, + uri: &Url, + ) -> Option> { + // Non-blocking snapshot: return whatever's currently in the + // watch channel — `Some(previous_analysis)` while a rebuild + // is in flight (serve-stale), `None` before the very first + // indexer has committed. Callers that CAN work with a stale + // snapshot (completion, hover, goto-def, references) use + // this directly; callers that need a fresh commit + // (diagnostics publish + progress indicator) use + // `wait_for_reindex` explicitly. + // + // We intentionally do NOT `await` a `changed()` here: rapid + // typing plus a 7s workspace-validate means every keystroke + // aborts the in-flight indexer, and if we haven't yet had a + // first commit, waiting would block indefinitely. Returning + // `None` immediately lets handlers degrade gracefully to a + // local-only analysis or empty results. + let docs = self.docs.read().await; + let state = docs.get(uri)?; + let out = state.tx.borrow().clone(); + out + } + + /// Build a fresh `DocAnalysis` for `text` synchronously + /// (parses + workspace view + validate). Called from the + /// spawned indexer task. Extracted so it can also be invoked + /// synchronously from tests without needing the async task + /// plumbing. + pub(crate) fn build_analysis( + uri: &Url, + text: String, + workspace_roots: &[std::path::PathBuf], + ) -> Arc { + let view = crate::workspace::build_view(uri, &text, workspace_roots); + let parsed_local = parse(&text); + let local_line_index = LineIndex::new(&text); + let parsed_view = parse(&view.view_source); + let line_index = LineIndex::new(&view.view_source); + + let mut diagnostics = Vec::new(); + // Target-compatibility check. For LSP diagnostics we anchor + // the message on the `src @` statement in the local + // file — much more informative than a whole-file gutter + // marker. Only fires when the entry workspace declares a + // target-part; libraries (no target) are no-op. + if let Ok(file_path) = uri.to_file_path() { + if let Some(ws) = + file_path.parent().and_then(vw_lib::find_workspace_dir) + { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + // LSP checks the DEFAULT part only — cheap and + // reflects what `vw run` would boot with. In + // variant-mode workspaces the default variant's + // part is what runs by default; in part-mode + // workspaces we fall back to the default target + // part. The CLI's `vw check --all-parts` / + // `--all-variants` covers the wider matrix on + // demand. + let resolved_part: Option = + if !cfg.workspace.variants.is_empty() { + cfg.workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.part.clone()) + } else { + cfg.workspace + .default_target_part() + .ok() + .flatten() + .map(|p| p.to_string()) + }; + if let Some(target_part) = resolved_part { + let target_part = target_part.as_str(); + let dep_targets = vw_lib::collect_dep_targets(&ws); + let mismatches = vw_lib::check_target_compatibility( + Some(target_part), + &dep_targets, + ); + // Anchor each mismatch on the specific + // `src @` line in the LOCAL file. + for m in &mismatches { + for stmt in &parsed_local.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let vw_htcl::CommandKind::Src(src) = &cmd.kind + else { + continue; + }; + let Some(raw) = &src.path else { continue }; + // Match `@` or `@/...`. + let dep_name = raw + .strip_prefix('@') + .and_then(|rest| { + rest.split_once('/') + .map(|(n, _)| n) + .or(Some(rest)) + }) + .unwrap_or(""); + if dep_name != m.dep { + continue; + } + let (start, end) = + local_line_index.range(src.path_span); + let hint = target_mismatch_families_hint(m); + let (severity, message) = match m.kind { + vw_lib::TargetMismatchKind::NotSupported => ( + DiagnosticSeverity::ERROR, + format!( + "target-part `{}` matches dep \ + `{}`'s `not-supported` list \ + — Xilinx has attested the IP \ + is not usable on this part \ + ({})", + m.target_part, m.dep, hint, + ), + ), + vw_lib::TargetMismatchKind::Unblessed => ( + DiagnosticSeverity::WARNING, + format!( + "target-part `{}` isn't blessed \ + by dep `{}` ({}); the IP may \ + still work but Xilinx hasn't \ + blessed the combination", + m.target_part, m.dep, hint, + ), + ), + }; + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message, + ..Default::default() + }); + } + } + } + } + } + } + let mut cross_file_diagnostics: Vec<(Url, Diagnostic)> = Vec::new(); + // Local parse errors. + for err in &parsed_local.errors { + let (start, end) = local_line_index.range(err.span); + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(DiagnosticSeverity::ERROR), + source: Some("vw-htcl".into()), + message: err.message.clone(), + ..Default::default() + }); + } + // Precompute a per-imported-file `LineIndex` on demand. + // Kept keyed by region index so multiple diagnostics + // landing in the same file only build the index once. + let mut import_line_indexes: HashMap = HashMap::new(); + // Workspace-view validator. Diagnostics whose span sits + // in the local prefix land in `diagnostics`; those that + // land in an imported region get retranslated into that + // file's own line/col and stashed for the workspace- + // diagnostic path. + for d in validate_with_all_extras_and_vars( + &parsed_view.document, + &view.view_source, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + &view.dep_names, + ) { + let severity = match d.severity { + Severity::Error => DiagnosticSeverity::ERROR, + Severity::Warning => DiagnosticSeverity::WARNING, + }; + if d.span.start < view.local_len { + let (start, end) = line_index.range(d.span); + diagnostics.push(Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message: d.message, + ..Default::default() + }); + continue; + } + // Locate which imported file the span belongs to and + // translate its offset into that file's coordinates. + // Skip diagnostics that don't land in any tracked + // region — shouldn't happen with the current view + // builder, but the model allows for view-source + // regions unmapped to files (empty for now). + let Some((region_idx, region, file_offset_start)) = + view.imports.iter().enumerate().find_map(|(i, r)| { + (d.span.start >= r.start && d.span.start < r.end) + .then(|| (i, r, d.span.start - r.start)) + }) + else { + continue; + }; + let file_offset_end = d.span.end.saturating_sub(region.start); + let file_text_range = + &view.view_source[region.start as usize..region.end as usize]; + let li = import_line_indexes + .entry(region_idx) + .or_insert_with(|| LineIndex::new(file_text_range)); + let (start, end) = li.range(vw_htcl::Span { + start: file_offset_start, + end: file_offset_end.min(region.end - region.start), + }); + cross_file_diagnostics.push(( + region.file_uri.clone(), + Diagnostic { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + severity: Some(severity), + source: Some("vw-htcl".into()), + message: d.message, + ..Default::default() + }, + )); + } + + Arc::new(DocAnalysis { + local_text: text, + view, + parsed_local, + parsed_view, + local_line_index, + diagnostics, + cross_file_diagnostics, + }) + } + + /// Spawn a fresh indexer task for `uri` + `text`. Returns the + /// `JoinHandle` so the caller can install it into `DocState` + /// (and abort it on a subsequent update). `debounce` gates how + /// long the task waits before starting the ~7s build — used + /// by `set_text` (250ms, to coalesce rapid typing) and by + /// `save` (0ms, so `Ctrl-s` re-checks immediately). + /// + /// The `spawn_blocking` inner run to completion even if the + /// outer task is aborted; the generation guard at commit time + /// discards any superseded result. + fn spawn_indexer( + &self, + uri: Url, + text: String, + generation: u64, + debounce: std::time::Duration, + ) -> tokio::task::JoinHandle<()> { + spawn_indexer_task( + self.docs.clone(), + self.workspace_roots.clone(), + uri, + text, + generation, + debounce, + ) + } +} + +/// Standalone version of `HtclBackend::spawn_indexer`. Split out so +/// the fan-out (`reindex_importers_of`, called from the indexer's +/// own commit path) can spawn follow-up indexers without needing a +/// live reference to the surrounding `HtclBackend` — the tokio task +/// only ever holds the two `Arc`s the indexer itself needs. +fn spawn_indexer_task( + docs: Arc>>, + workspace_roots: Arc>>, + uri: Url, + text: String, + generation: u64, + debounce: std::time::Duration, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + if !debounce.is_zero() { + tokio::time::sleep(debounce).await; + } + let roots = workspace_roots.read().await.clone(); + let uri_inner = uri.clone(); + let analysis = match tokio::task::spawn_blocking(move || { + HtclBackend::build_analysis(&uri_inner, text, &roots) + }) + .await + { + Ok(a) => a, + Err(_) => return, + }; + // Commit under the generation guard. Superseded results + // are discarded — the `set_text` handler that bumped past + // us installed a fresher indexer that WILL commit. + let committed = { + let docs_r = docs.read().await; + let Some(state) = docs_r.get(&uri) else { + return; + }; + if state.generation != generation { + debug!( + uri = %uri, + generation, + current = state.generation, + "index superseded, discarded", + ); + return; + } + debug!(uri = %uri, generation, "index committed"); + state.tx.send_replace(Some(analysis.clone())); + true + }; + // Fan out to open documents whose CURRENT analysis + // imports `uri`. Reason: an import's disk / open-buffer + // content changed (that's what just committed here), so + // any doc that transitively `src`s it now has stale + // symbol/hover data for that region. Without this fan-out, + // editing `ip/clock.htcl` leaves `ip/module.htcl`'s hover + // pointing at the pre-edit analysis until the user + // touches module.htcl itself. Fan-out uses the same + // 0ms-debounce path `save()` takes so the ripple lands + // fast; cascading (A imports B imports C, C edits) still + // converges because each level's fan-out only fires ONCE + // per commit (there's no re-entry: A's rebuild doesn't + // re-fire B's, since B's imports of A haven't changed). + if committed { + reindex_importers_of(&docs, &workspace_roots, &uri).await; + } + }) +} + +/// Enqueue a fresh indexer for every open doc whose currently- +/// committed analysis lists `changed` in `view.imports`. Called from +/// the indexer commit path so downstream files pick up upstream +/// edits (doc comments, signatures, new procs) without waiting for +/// the downstream file to be edited itself. +/// +/// Skips docs that don't have a committed analysis yet (they'll +/// pick up the change on their first commit anyway) and skips the +/// changed URI itself (it just committed). +async fn reindex_importers_of( + docs: &Arc>>, + workspace_roots: &Arc>>, + changed: &Url, +) { + // Collect the (uri, text, new_generation, prev_task) tuples under + // the write lock, then release the lock BEFORE awaiting aborts + // and spawning new indexers — other handlers can proceed + // concurrently. Matches the pattern in `set_text`. + let mut to_spawn: Vec<( + Url, + String, + u64, + Option>, + )> = Vec::new(); + { + let mut docs_w = docs.write().await; + // Snapshot the URIs first — we can't hold two live borrows + // (one to iterate, one to `get_mut`) at once. + let candidates: Vec = docs_w + .iter() + .filter(|(u, _)| *u != changed) + .filter_map(|(u, state)| { + let analysis = state.tx.borrow(); + analysis + .as_ref() + .filter(|a| { + a.view.imports.iter().any(|i| &i.file_uri == changed) + }) + .map(|_| u.clone()) + }) + .collect(); + for u in candidates { + let Some(state) = docs_w.get_mut(&u) else { + continue; + }; + state.generation += 1; + let prev = state.index_task.take(); + to_spawn.push((u, state.text.clone(), state.generation, prev)); + } + } + for (u, text, gen, prev) in to_spawn { + if let Some(h) = prev { + h.abort(); + } + let handle = spawn_indexer_task( + docs.clone(), + workspace_roots.clone(), + u.clone(), + text, + gen, + std::time::Duration::ZERO, + ); + let mut docs_w = docs.write().await; + if let Some(state) = docs_w.get_mut(&u) { + if state.generation == gen { + state.index_task = Some(handle); + } else { + handle.abort(); + } + } else { + handle.abort(); + } + } +} + +#[async_trait] +impl LanguageBackend for HtclBackend { + fn language_id(&self) -> &str { + "htcl" + } + + fn handles(&self, uri: &Url) -> bool { + uri.path().ends_with(".htcl") + } + + async fn set_text(&self, uri: Url, text: String) { + debug!(%uri, bytes = text.len(), "set_text"); + // Capture the previous indexer task (if any) so we can abort + // it AFTER releasing the write lock — abort() itself is + // cheap but keeping the lock held for it stalls other + // handlers wanting to read the docs map. + let (tx, generation, prev_task) = { + let mut docs = self.docs.write().await; + match docs.get_mut(&uri) { + Some(state) => { + state.generation += 1; + state.text = text.clone(); + // Serve-stale-while-rebuild: do NOT clear the + // previous analysis. Reads (`analysis_for`, + // via completion/hover/goto-def/references) + // will see the pre-keystroke snapshot + // immediately instead of waiting the ~7s a + // fresh `build_analysis` takes on a large + // workspace (the metroid tree hits ~7s in + // the validator alone). Diagnostics update + // one commit behind — an acceptable tradeoff + // for interactive latency. When the freshly + // spawned indexer commits, `send_replace(Some + // (new))` swaps the snapshot in-place with + // no observable gap. + let prev = state.index_task.take(); + (state.tx.clone(), state.generation, prev) + } + None => { + let (tx, _rx) = watch::channel(None); + docs.insert( + uri.clone(), + DocState { + text: text.clone(), + generation: 1, + tx: tx.clone(), + index_task: None, + }, + ); + (tx, 1, None) + } + } + }; + if let Some(handle) = prev_task { + handle.abort(); + } + + let handle = self.spawn_indexer( + uri.clone(), + text, + generation, + std::time::Duration::from_millis(250), + ); + + // Store the handle so a subsequent set_text can abort us. + let mut docs = self.docs.write().await; + if let Some(state) = docs.get_mut(&uri) { + if state.generation == generation { + state.index_task = Some(handle); + } else { + // A newer set_text landed between our two lock + // acquisitions. Kill our task, the newer set_text + // has already installed its own. + handle.abort(); + } + } else { + // Doc was closed while we were spawning. Kill our task. + handle.abort(); + } + let _ = tx; // silence unused warning when watching sends aren't used further + } + + async fn set_workspace_roots(&self, roots: Vec) { + *self.workspace_roots.write().await = roots.clone(); + // Preload the same set of files `vw check` scans so + // workspace-wide diagnostics cover the whole tree — not + // just the import graph reachable from the docs the + // user has explicitly opened. Without this the space-D + // picker misses warnings in leaf files (e.g. `ip/gtm.htcl`, + // `ip/dcmac.htcl`) until the user opens each one; with + // it, opening ANY file in the workspace surfaces the + // full-workspace picture on first analysis. + // + // Preload runs BEHIND `set_text`'s 250ms debounce (it + // just enqueues indexers) so this call returns fast; the + // per-file builds run on the indexer's spawn_blocking + // thread pool in the background. + self.preload_workspace_targets(&roots).await; + } + + async fn save(&self, uri: &Url) { + // Save is the user's explicit "I'm done for now" signal — + // skip the debounce entirely so their edit is checked + // immediately. Bump the generation so any in-flight + // debounced indexer from the last `set_text` gets + // superseded when this one commits. Text is whatever's + // currently stored; Helix sends `did_change` before + // `did_save` on save operations so the buffer's already + // in sync. + let (text, generation, prev_task) = { + let mut docs = self.docs.write().await; + let Some(state) = docs.get_mut(uri) else { + return; + }; + state.generation += 1; + let prev = state.index_task.take(); + (state.text.clone(), state.generation, prev) + }; + if let Some(handle) = prev_task { + handle.abort(); + } + let handle = self.spawn_indexer( + uri.clone(), + text, + generation, + std::time::Duration::ZERO, + ); + let mut docs = self.docs.write().await; + if let Some(state) = docs.get_mut(uri) { + if state.generation == generation { + state.index_task = Some(handle); + } else { + handle.abort(); + } + } else { + handle.abort(); + } + } + + async fn wait_for_reindex(&self, uri: &Url) { + // Subscribe to the doc's analysis-watch channel, then wait + // for the NEXT commit — i.e. the indexer task's + // `send_replace(Some(new))` at the end of `set_text`'s + // spawned future. Used by the server to wrap the wait in + // an LSP `workDoneProgress` notification so the editor's + // "indexing…" spinner reflects the actual rebuild + // duration, not just a fire-and-forget millisecond. + // + // `mark_unchanged` is the key call: `subscribe()` seeds the + // receiver at the current sender value (which may be the + // stale-serve analysis we've KEPT alive across `set_text` + // — see the serve-stale comment there). Without + // `mark_unchanged` the immediate `changed().await` would + // return instantly on that already-seen value and the + // spinner would flash for a millisecond instead of + // spanning the whole rebuild. + // + // If the sender is dropped (doc closed), or the current + // watch has never held a value (uri unknown), we return + // immediately — no rebuild to wait on. + let mut rx = { + let docs = self.docs.read().await; + match docs.get(uri) { + Some(state) => state.tx.subscribe(), + None => return, + } + }; + rx.mark_unchanged(); + let _ = rx.changed().await; + } + + async fn close(&self, uri: &Url) { + let prev_task = { + let mut docs = self.docs.write().await; + docs.remove(uri).and_then(|s| s.index_task) + }; + if let Some(handle) = prev_task { + handle.abort(); + } + } + + async fn diagnostics(&self, uri: &Url) -> Vec { + // All diagnostics precomputed at index time. No work here. + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + analysis.diagnostics.clone() + } + + async fn workspace_diagnostics(&self) -> Vec<(Url, Vec)> { + // Walk every open document's committed analysis. Each + // carries its own local diagnostics AND the retranslated + // diagnostics from every file it transitively `src`s. + // + // Snapshot the URI list first so we can drop the docs + // read lock before calling `analysis_for` on each (which + // takes its own read). + let uris: Vec = self.docs.read().await.keys().cloned().collect(); + // First-call gate: preloaded entry points can still be + // building the FIRST time this runs (initialize → preload + // spawns indexers, and the user opens a file before those + // indexers commit). Without this wait, the fan-out + // published from the user-open reindex reads a partial + // `workspace_diagnostics` snapshot — files whose preload + // hasn't committed yet contribute NOTHING, so the picker + // shows an incomplete picture until the user happens to + // open another file (which triggers a fresh fan-out + // AFTER preloads have finished). + // + // The bounded wait is per-URI: subscribe to that doc's + // analysis-watch channel and await the first non-`None` + // value. Already-committed docs return instantly. + // Preloads that fail to commit (indexer panic, disk read + // error) never publish a `Some` — 5 s is a generous + // ceiling that keeps the picker responsive even in that + // pathological case (the failing URI is simply omitted). + for uri in &uris { + let rx = { + let docs = self.docs.read().await; + docs.get(uri).map(|s| s.tx.subscribe()) + }; + let Some(mut rx) = rx else { continue }; + if rx.borrow().is_some() { + continue; + } + let _ = tokio::time::timeout( + std::time::Duration::from_secs(5), + rx.wait_for(|v| v.is_some()), + ) + .await; + } + let roots = self.workspace_roots_snapshot().await; + // Open-doc set — used below to decide who owns a URI's + // diagnostics. When a file is open, its own analysis was + // built from the live buffer and is authoritative; + // cross-file diagnostics from OTHER open files targeting + // the same URI reflect whatever those files last saw on + // disk, which is stale as soon as this file gets edited. + // Ignoring them fixes the stuck-diagnostic bug where + // fixing `cips.htcl` doesn't clear its markers until every + // file that `src`s it also reindexes. + let open_uris: std::collections::HashSet = + uris.iter().cloned().collect(); + // Group by origin URI so a file `src`d by multiple open + // docs doesn't get its diagnostics duplicated; when the + // same file surfaces via more than one analysis, we keep + // just the first non-empty set. In practice the analyses + // agree because the validator is deterministic per input. + let mut by_uri: HashMap> = HashMap::new(); + for uri in &uris { + let Some(analysis) = self.analysis_for(uri).await else { + continue; + }; + // Every open document contributes an entry — even an + // EMPTY one — so the editor clears stale errors when + // the user fixes them. Without this, a file that + // *used* to have errors keeps showing them until it's + // reopened. `insert` (not `or_insert_with`) so a + // freshly-committed analysis wins over a stale entry + // some earlier iteration's cross-file merge left + // behind. + by_uri.insert(uri.clone(), analysis.diagnostics.clone()); + // Seed an empty entry for every imported file that + // sits inside the workspace. This is what makes fixed + // errors CLEAR: after the fix, `cross_file_diagnostics` + // has no entry for that file, but the fan-out still + // sees the URI (from `view.imports`) and publishes an + // empty payload — the editor's cached "there were + // errors here" state gets overwritten with "no + // errors." Without this seed, cleared files wouldn't + // reappear in the output map at all. + for import in &analysis.view.imports { + if !roots.is_empty() + && !uri_under_roots(&import.file_uri, &roots) + { + continue; + } + by_uri.entry(import.file_uri.clone()).or_default(); + } + for (u, d) in &analysis.cross_file_diagnostics { + // Only surface diagnostics from files that sit + // inside the editor's own workspace roots. Deps + // (`~/.vw/deps`, the amd/ trees, etc.) get walked + // by `build_view` for symbol resolution, but the + // user isn't editing them from this workspace — + // reporting a dep-side error in `space-D` is + // noise. When no roots are set (e.g. the file was + // opened standalone), fall through: nothing to + // filter against. + if !roots.is_empty() && !uri_under_roots(u, &roots) { + continue; + } + // Do NOT overlay cross-file diagnostics onto files + // that have their own open analysis — that + // analysis was just rebuilt from the live buffer + // and supersedes whatever the srcing file's stale + // analysis remembers. Skipping this was the entire + // reason old errors stuck around in Helix after + // the user fixed them: any parent doc's cached + // analysis kept re-injecting the same diagnostic + // on every workspace/diagnostic tick until that + // parent reindexed. + if open_uris.contains(u) { + continue; + } + by_uri.entry(u.clone()).or_default().push(d.clone()); + } + } + by_uri.into_iter().collect() + } + + async fn document_symbols(&self, uri: &Url) -> Vec { + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + let parsed = &analysis.parsed_local; + let line_index = &analysis.local_line_index; + let mut symbols = Vec::new(); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + let name = proc.name.clone().unwrap_or_else(|| "".into()); + let (cmd_start, cmd_end) = line_index.range(cmd.span); + let (name_start, name_end) = line_index.range(proc.name_span); + let detail = if cmd.doc_comments.is_empty() { + None + } else { + Some(cmd.doc_comments.join("\n")) + }; + #[allow(deprecated)] + symbols.push(DocumentSymbol { + name, + detail, + kind: SymbolKind::FUNCTION, + tags: None, + deprecated: None, + range: Range { + start: lc_to_pos(cmd_start), + end: lc_to_pos(cmd_end), + }, + selection_range: Range { + start: lc_to_pos(name_start), + end: lc_to_pos(name_end), + }, + children: None, + }); + } + symbols + } + + async fn workspace_symbols(&self, query: &str) -> Vec { + // Cap the response so a wide picker scroll doesn't pay for + // thousands of entries when the user hasn't narrowed yet. The + // editor applies its own scoring on top, so any reasonable + // ceiling keeps the UX responsive. + const MAX_RESULTS: usize = 500; + + let needle = query.to_ascii_lowercase(); + // Snapshot the list of open URIs — we release the docs + // lock before calling `analysis_for` on each so an + // in-flight indexer's write-lock acquisition doesn't + // deadlock against our read lock. + let uris: Vec = self.docs.read().await.keys().cloned().collect(); + // Files we've already harvested — dedupe so a header imported + // by multiple open docs doesn't double up. Keyed on the URI as + // a string for hashability. + let mut seen_files: HashMap = HashMap::new(); + let mut out: Vec = Vec::new(); + + for uri in &uris { + let Some(analysis) = self.analysis_for(uri).await else { + continue; + }; + // Visit the open doc itself first, then everything it + // transitively `src`s. `build_view` already canonicalizes + // paths during the walk, so the import file_uris are + // stable across docs. + if seen_files.insert(uri.to_string(), ()).is_none() { + collect_workspace_symbols( + uri, + &analysis.local_text, + &needle, + &mut out, + MAX_RESULTS, + ); + if out.len() >= MAX_RESULTS { + return out; + } + } + + for import in &analysis.view.imports { + let key = import.file_uri.to_string(); + if seen_files.insert(key, ()).is_some() { + continue; + } + let text = &analysis.view.view_source + [import.start as usize..import.end as usize]; + collect_workspace_symbols( + &import.file_uri, + text, + &needle, + &mut out, + MAX_RESULTS, + ); + if out.len() >= MAX_RESULTS { + return out; + } + } + } + out + } + + async fn goto_definition( + &self, + uri: &Url, + position: Position, + ) -> Vec { + let Some(analysis) = self.analysis_for(uri).await else { + return Vec::new(); + }; + let line_index = &analysis.local_line_index; + let offset = line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + + // Special case: cursor on a `src @dep/foo` path → jump to the + // imported file. Resolved through the same `vw-lib` machinery + // the CLI uses, so editor and CLI agree on the same target. + let parsed_local = &analysis.parsed_local; + if let Some(import) = src_import_at(&parsed_local.document, offset) { + if let Some(raw) = import.path.as_deref() { + let Ok(file_path) = uri.to_file_path() else { + return Vec::new(); + }; + if let Some(resolved) = + self.resolve_import(&file_path, raw).await + { + if let Ok(target_uri) = Url::from_file_path(resolved) { + return vec![Location { + uri: target_uri, + range: Range::default(), + }]; + } + } + } + return Vec::new(); + } + + // General case: resolve against the workspace view so calls to + // imported procs jump to the right file. + let view = &analysis.view; + let parsed_view = &analysis.parsed_view; + let Some(target_span) = + definition_at(&parsed_view.document, &view.view_source, offset) + else { + return Vec::new(); + }; + + // Translate the target span back to its source file: local + // file when in the original region, otherwise the imported + // file whose appended region contains it. + match view.locate(target_span.start) { + None => { + // Local hit — line_index is over analysis.local_text. + let (start, end) = analysis.local_line_index.range(target_span); + vec![Location { + uri: uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }] + } + Some((region, _)) => { + // Read the imported file's text so we can build a + // file-local line index. (Already on disk; cheap.) + let Ok(import_path) = region.file_uri.to_file_path() else { + return Vec::new(); + }; + let Ok(import_text) = std::fs::read_to_string(&import_path) + else { + return Vec::new(); + }; + let import_index = LineIndex::new(&import_text); + let local_start = target_span.start - region.start; + let local_end = target_span.end - region.start; + let (s, e) = import_index + .range(vw_htcl::Span::new(local_start, local_end)); + vec![Location { + uri: region.file_uri.clone(), + range: Range { + start: lc_to_pos(s), + end: lc_to_pos(e), + }, + }] + } + } + } + + async fn hover(&self, uri: &Url, position: Position) -> Option { + // Same strategy as completion: serve hover off the CURRENT + // in-memory text so what the user's cursor is on maps to + // real content. When a workspace analysis is available + // (usually — 99% of hover requests fire between typing + // bursts, when there IS a committed snapshot), we consult + // it for cross-file lookups; when there isn't (fresh file, + // still-building initial index), we degrade to local-only + // — still useful for hovering over locally-defined procs. + let current_text = self.current_text(uri).await.unwrap_or_default(); + let current_line_index = LineIndex::new(¤t_text); + let offset = current_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + let current_parsed = vw_htcl::parse(¤t_text); + + // Prefer the workspace snapshot's parsed_view (it contains + // the imported proc definitions we want to hover through). + // Fall back to the CURRENT local parse when nothing's + // committed yet. + let stale = self.analysis_for(uri).await; + let (hover_doc, hover_source, hover_offset, doc_for_comments) = + if let Some(a) = stale.as_ref() { + // The offset was computed against CURRENT text. It + // maps 1:1 into the workspace view AS LONG AS the + // stale view's local prefix is a prefix of the + // current text (typical: adds/dels midway through + // the line only shift bytes past that point, but + // the workspace view is only ~correct in the local + // prefix anyway). For hover, the miscarriage is + // harmless — worst case we hover on the wrong + // token and return None. + ( + &a.parsed_view.document, + a.view.view_source.as_str(), + offset, + &a.parsed_view.document, + ) + } else { + ( + ¤t_parsed.document, + current_text.as_str(), + offset, + ¤t_parsed.document, + ) + }; + let target = hover_at(hover_doc, hover_source, hover_offset)?; + // Prefer LOCAL line index for translating spans → line/col: + // that's what Helix expects. + let (start, end) = current_line_index.range(target.span()); + // The proc's own doc comments live on the surrounding Command, + // not on its `Proc` payload — fetch them up here so the + // formatters can stay focused on shape, not lookup plumbing. + let proc_doc_comments = match &target { + HoverTarget::ProcDef { proc, .. } => { + proc_doc_comments_for(doc_for_comments, proc) + } + HoverTarget::CallSite { proc_name, .. } => { + proc_doc_comments_by_name(doc_for_comments, proc_name) + } + _ => Vec::new(), + }; + let markdown = format_hover(&target, &proc_doc_comments); + Some(Hover { + contents: HoverContents::Markup(MarkupContent { + kind: MarkupKind::Markdown, + value: markdown, + }), + range: Some(Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }), + }) + } + + async fn completion( + &self, + uri: &Url, + position: Position, + ) -> Vec { + // Serve completion off the CURRENT in-memory text (what the + // user just typed), not the stale-cached workspace analysis's + // local_text. Otherwise cmdline::analyze scans backward from + // an offset in TEXT THAT DOESN'T CONTAIN WHAT WAS JUST TYPED + // — the `partial` comes out blank or wrong, and completion + // silently returns nothing. This is the "typed `-preset` and + // got no enum values" symptom. + // + // Cross-file proc lookups (`gtwiz_versal::configure`, etc.) + // still come from the stale workspace analysis via + // `parsed_view` — those signatures don't change while the + // user types locally, so stale is fine. + let current_text = self.current_text(uri).await.unwrap_or_default(); + let current_line_index = LineIndex::new(¤t_text); + let offset = current_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + // Suppress completion inside comments. Helix's LSP client + // auto-triggers on typing, so writing prose inside a + // `#`-line otherwise pops a dropdown on every space — + // pure noise. The check: on the line up to the cursor, + // the first non-whitespace char is `#`. That covers both + // standalone `# text` and `## doc comment` lines, plus + // mid-command `# comment` continuations (which in htcl + // are still line-anchored — the comment starts at + // column-0-after-ws, same shape). It doesn't catch `#` + // inside strings / brackets — those don't start comments + // in htcl anyway, so no false positives on real code. + if in_line_comment(¤t_text, offset) { + return Vec::new(); + } + let current_parsed = vw_htcl::parse(¤t_text); + + // `src ` is filesystem-aware, so it takes its own + // path before we fall back to the htcl-level analyzer. + let line = vw_htcl::cmdline::analyze(¤t_text, offset); + if crate::src_complete::is_src_path_context(&line) { + if let Ok(entry_file) = uri.to_file_path() { + let resolver = crate::workspace::build_resolver(&entry_file); + return crate::src_complete::src_path_completions( + &entry_file, + &line, + ¤t_line_index, + &resolver, + ); + } + } + + // Grab the workspace analysis if it exists (stale or fresh). + // If we've never had one commit, we complete against just + // the local file — better than blocking indefinitely. + let analysis = self.analysis_for(uri).await; + let workspace_docs: Vec<&vw_htcl::Document> = analysis + .as_ref() + .map(|a| vec![&a.parsed_view.document]) + .unwrap_or_default(); + + vw_htcl::complete_at_with_extras( + ¤t_parsed.document, + ¤t_text, + offset, + &workspace_docs, + ) + .into_iter() + .map(|c| completion_item(c, ¤t_line_index)) + .collect() + } + + async fn signature_help( + &self, + uri: &Url, + position: Position, + ) -> Option { + let analysis = self.analysis_for(uri).await?; + let offset = analysis.local_line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + // Workspace view so signatures of imported procs surface, and + // so the cmdline scan can step into a `[ … ]` substitution + // (the parser now carries a `body` inside `CmdSubst` and the + // scan already treats `[` as a command boundary). + let view = &analysis.view; + let parsed = &analysis.parsed_view; + let help = + signature_help_at(&parsed.document, &view.view_source, offset)?; + Some(signature_help_response(&help)) + } + + async fn rename( + &self, + uri: &Url, + position: Position, + new_name: &str, + ) -> Option { + let (target, _decl_uri) = + self.identify_target_at(uri, position).await?; + // Build a per-URI edit list for every file the target + // reaches. For file-local kinds (Local, ProcArg) this + // resolves to just the current file; for cross-file + // kinds it walks every `.htcl` file under the workspace + // root. + let per_file = self.collect_reference_spans(uri, &target).await; + if per_file.is_empty() { + return None; + } + let replacement = rename_replacement_for(&target, new_name)?; + let mut changes: HashMap> = HashMap::new(); + for (file_uri, text, spans) in per_file { + let line_index = LineIndex::new(&text); + let edits: Vec = spans + .into_iter() + .map(|span| span_to_text_edit(span, &line_index, &replacement)) + .collect(); + if !edits.is_empty() { + changes.insert(file_uri, edits); + } + } + if changes.is_empty() { + return None; + } + Some(WorkspaceEdit { + changes: Some(changes), + document_changes: None, + change_annotations: None, + }) + } + + async fn references( + &self, + uri: &Url, + position: Position, + include_declaration: bool, + ) -> Vec { + let Some((target, _)) = self.identify_target_at(uri, position).await + else { + return Vec::new(); + }; + let per_file = self.collect_reference_spans(uri, &target).await; + let mut locations = Vec::new(); + for (file_uri, text, mut spans) in per_file { + let line_index = LineIndex::new(&text); + if !include_declaration { + // Best-effort decl filter: the target's own decl + // is contained inside the ref set (procs' name- + // span, types' name-span, enum-variant name- + // spans). The reference finder returns them all; + // remove those that lie inside the current + // target's OWN declaration span when the target + // came from this file. For cross-file callers + // there's no ambiguity — the decl is only in the + // decl file. + spans.retain(|s| { + !span_looks_like_decl(&target, *s, &file_uri, uri) + }); + } + for span in spans { + let (start, end) = line_index.range(span); + locations.push(Location { + uri: file_uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }); + } + } + locations + } +} + +impl HtclBackend { + /// Identify the reference target at `position` in `uri`. Also + /// returns the URI that owned the identification so the + /// per-file collector knows which document to treat as the + /// origin (matters for the file-local Local/ProcArg kinds). + async fn identify_target_at( + &self, + uri: &Url, + position: Position, + ) -> Option<(ReferenceTarget, Url)> { + let analysis = self.analysis_for(uri).await?; + let line_index = &analysis.local_line_index; + let offset = line_index.offset_of(LineCol { + line: position.line, + character: position.character, + }); + let parsed = &analysis.parsed_local; + let target = + identify_at(&parsed.document, &analysis.local_text, offset)?; + Some((target, uri.clone())) + } + + /// For each file the target reaches, return `(uri, text, + /// spans)`. File-local targets stay in the origin file; cross- + /// file targets get every `.htcl` file under the workspace + /// root read and scanned. + async fn collect_reference_spans( + &self, + origin: &Url, + target: &ReferenceTarget, + ) -> Vec<(Url, String, Vec)> { + match target { + ReferenceTarget::Local { .. } | ReferenceTarget::ProcArg { .. } => { + let Some(analysis) = self.analysis_for(origin).await else { + return Vec::new(); + }; + let spans = find_references_in( + &analysis.parsed_local.document, + &analysis.local_text, + target, + ); + if spans.is_empty() { + Vec::new() + } else { + vec![(origin.clone(), analysis.local_text.clone(), spans)] + } + } + ReferenceTarget::Proc { .. } + | ReferenceTarget::Type { .. } + | ReferenceTarget::EnumVariant { .. } => { + let mut files = self.workspace_htcl_files(origin).await; + // Fallback: no workspace root (test URIs, files + // opened outside a `vw.toml` tree, etc.) → operate + // on the origin file only. The rename still + // works locally; users adopting the LSP outside a + // workspace get local-only semantics until they + // set up a `vw.toml`. + if files.is_empty() { + if let Some(analysis) = self.analysis_for(origin).await { + files.push(( + origin.clone(), + analysis.local_text.clone(), + )); + } + } + let mut out = Vec::new(); + for (file_uri, text) in files { + let parsed = parse(&text); + let spans = + find_references_in(&parsed.document, &text, target); + if !spans.is_empty() { + out.push((file_uri, text, spans)); + } + } + out + } + } + } + + /// Enumerate every `.htcl` file under the workspace root that + /// contains `origin`. Reads their current on-disk content — + /// for files also open in the editor this may be one tick + /// behind, but that's the cost of not requiring the editor to + /// pre-open every workspace file. Skips typical non-workspace + /// directories (`target/`, `.git/`, `.vw/`). + /// + /// The origin file itself is served from the in-memory + /// analysis so unsaved edits round-trip through the rename. + async fn workspace_htcl_files(&self, origin: &Url) -> Vec<(Url, String)> { + let Ok(origin_path) = origin.to_file_path() else { + return Vec::new(); + }; + let Some(origin_utf8) = camino::Utf8Path::from_path(&origin_path) + else { + return Vec::new(); + }; + let Some(root) = crate::workspace::workspace_root(origin_utf8) else { + return Vec::new(); + }; + let mut paths: Vec = Vec::new(); + walk_htcl_files(root.as_std_path(), &mut paths); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + let mut out: Vec<(Url, String)> = Vec::new(); + for path in paths { + let canonical = + path.canonicalize().unwrap_or_else(|_| path.clone()); + if !visited.insert(canonical.clone()) { + continue; + } + let file_uri = match Url::from_file_path(&canonical) { + Ok(u) => u, + Err(_) => continue, + }; + // For the origin file, prefer the in-memory analysis + // text so unsaved edits round-trip. + let text = if file_uri == *origin { + if let Some(analysis) = self.analysis_for(&file_uri).await { + analysis.local_text.clone() + } else { + std::fs::read_to_string(&canonical).unwrap_or_default() + } + } else { + std::fs::read_to_string(&canonical).unwrap_or_default() + }; + if text.is_empty() { + continue; + } + out.push((file_uri, text)); + } + out + } +} + +/// Recursively walk `dir` collecting `.htcl` files. Skips `target/`, +/// `.git/`, `.vw/`, `node_modules/` at any depth. Silently swallows +/// I/O errors on individual directories — a permission-denied +/// subtree just contributes nothing to the results. +/// True when `uri`'s filesystem path lies under any of `roots`. +/// Non-file URIs and paths that don't resolve into any root fall +/// through as `false` — the caller's default behavior is "not in +/// the workspace, don't fan out." Both sides get canonicalized so +/// a symlinked workspace root matches a real-path URI (`Path:: +/// starts_with` is purely lexical). Canonicalization failures +/// (missing files, permission errors) fall back to the lexical +/// compare, which still catches the common case. +/// Same helper as vw-cli's — split blessed vs. banned family lists +/// so a dep whose `[targets]` only carries a `not-supported` list +/// doesn't get misreported as "declared families: versal" when +/// the versal families there are BANNED, not blessed. +fn target_mismatch_families_hint(m: &vw_lib::TargetMismatch) -> String { + match ( + m.supported_families.is_empty(), + m.not_supported_families.is_empty(), + ) { + (true, true) => { + "no `[targets]` families declared — the dep has patterns \ + but none carry family names" + .to_string() + } + (false, true) => { + format!("blessed families: {}", m.supported_families.join(", ")) + } + (true, false) => { + format!( + "no blessed families — only `not-supported` entries for {}", + m.not_supported_families.join(", "), + ) + } + (false, false) => { + format!( + "blessed families: {}; also `not-supported` entries for {}", + m.supported_families.join(", "), + m.not_supported_families.join(", "), + ) + } + } +} + +fn uri_under_roots(uri: &Url, roots: &[std::path::PathBuf]) -> bool { + let Ok(path) = uri.to_file_path() else { + return false; + }; + let canonical_path = path.canonicalize().unwrap_or(path); + roots.iter().any(|r| { + let canonical_root = r.canonicalize().unwrap_or_else(|_| r.clone()); + canonical_path.starts_with(&canonical_root) + }) +} + +fn walk_htcl_files(dir: &std::path::Path, out: &mut Vec) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(t) => t, + Err(_) => continue, + }; + if file_type.is_dir() { + let name = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + if matches!(name, "target" | ".git" | ".vw" | "node_modules") { + continue; + } + walk_htcl_files(&path, out); + } else if file_type.is_file() + && path.extension().and_then(|s| s.to_str()) == Some("htcl") + { + out.push(path); + } + } +} + +/// Pick the exact text to substitute at each rename span for a +/// given target. Preserves namespace prefixes when the user typed +/// a bare replacement name. +fn rename_replacement_for( + target: &ReferenceTarget, + new_name: &str, +) -> Option { + if new_name.is_empty() { + return None; + } + // Validate: bare identifier or `ns::segment(::segment)*`. + for seg in new_name.split("::") { + if seg.is_empty() { + return None; + } + let mut bytes = seg.bytes(); + let first = bytes.next().unwrap(); + if !(first.is_ascii_alphabetic() || first == b'_') { + return None; + } + if !bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return None; + } + } + Some(match target { + ReferenceTarget::Proc { name } + if name.contains("::") && !new_name.contains("::") => + { + let ns = name.rsplit_once("::").map(|(n, _)| n).unwrap_or(""); + format!("{ns}::{new_name}") + } + ReferenceTarget::EnumVariant { enum_name, .. } + if !new_name.contains("::") => + { + format!("{enum_name}::{new_name}") + } + _ => new_name.to_string(), + }) +} + +/// Map a source `Span` + replacement text to an LSP `TextEdit`. +fn span_to_text_edit( + span: Span, + line_index: &LineIndex, + new_text: &str, +) -> TextEdit { + let (start, end) = line_index.range(span); + TextEdit { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + new_text: new_text.to_string(), + } +} + +/// Best-effort filter for the `!include_declaration` case. Skips +/// spans that plausibly correspond to a decl site by name-matching +/// the target's shape. This is imperfect (a proc named `X` and a +/// call `X` are indistinguishable at the span level), but the LSP +/// clients that pass `include_declaration=false` usually just want +/// to hide the decl in the results — an occasional inclusion is +/// benign. +fn span_looks_like_decl( + _target: &ReferenceTarget, + _span: Span, + _file_uri: &Url, + _origin_uri: &Url, +) -> bool { + // Placeholder — the LSP protocol says clients CAN filter locally, + // and most do. Returning false means we always include; safer + // than accidentally dropping too much. + false +} + +// (`rename_edit_to_lsp` removed — the rename handler now emits +// `TextEdit`s directly via `span_to_text_edit` on the raw +// reference spans, so the intermediate `RenameEdit` type isn't +// crossed over anymore.) + +// --- completion / signature-help formatters ------------------------------- + +fn completion_item(c: Completion, line_index: &LineIndex) -> CompletionItem { + let kind = match c.kind { + CompletionKind::Proc => CompletionItemKind::FUNCTION, + CompletionKind::Flag => CompletionItemKind::FIELD, + CompletionKind::EnumValue => CompletionItemKind::ENUM_MEMBER, + CompletionKind::Constructor => CompletionItemKind::CONSTRUCTOR, + }; + let (start, end) = line_index.range(c.replace); + let insert = c.insert_text.clone().unwrap_or_else(|| c.label.clone()); + let text_edit = TextEdit { + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + new_text: insert, + }; + let insert_text_format = if c.snippet { + InsertTextFormat::SNIPPET + } else { + InsertTextFormat::PLAIN_TEXT + }; + CompletionItem { + label: c.label, + kind: Some(kind), + detail: c.detail, + documentation: c.documentation.map(|value| { + Documentation::MarkupContent(MarkupContent { + kind: MarkupKind::Markdown, + value, + }) + }), + insert_text_format: Some(insert_text_format), + text_edit: Some(tower_lsp::lsp_types::CompletionTextEdit::Edit( + text_edit, + )), + ..Default::default() + } +} + +fn signature_help_response(help: &vw_htcl::SignatureHelp<'_>) -> SignatureHelp { + // Build the rendered signature label and, in lockstep, the + // [start, end) offsets each parameter occupies within it so the + // editor highlights the active one. Names are identifiers, so + // UTF-16 and char counts coincide. + let mut label = help.proc_name.clone(); + let mut parameters = Vec::with_capacity(help.signature.args.len()); + for arg in &help.signature.args { + label.push(' '); + let start = label.chars().count() as u32; + label.push('-'); + label.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + label.push_str(": "); + label.push_str(&render_type(ty)); + } + let end = label.chars().count() as u32; + parameters.push(ParameterInformation { + label: ParameterLabel::LabelOffsets([start, end]), + documentation: vw_htcl::doc::brief(&arg.doc_comments) + .map(Documentation::String), + }); + } + // Append the return type to the signature label when present. + // Renders as `proc-name -arg1 -arg2 → bd_cell`. + if let Some(ty) = help.signature.return_type.as_ref() { + label.push_str(" → "); + label.push_str(&render_type(ty)); + } + + let reflowed = vw_htcl::doc::reflow_doc_comments(help.doc_comments); + let documentation = (!reflowed.is_empty()).then_some({ + Documentation::MarkupContent(MarkupContent { + kind: MarkupKind::Markdown, + value: reflowed, + }) + }); + + #[allow(deprecated)] // `active_parameter` field on SignatureInformation + let info = SignatureInformation { + label, + documentation, + parameters: Some(parameters), + active_parameter: help.active_parameter, + }; + + SignatureHelp { + signatures: vec![info], + active_signature: Some(0), + active_parameter: help.active_parameter, + } +} + +// --- src import lookup ---------------------------------------------------- + +/// If the cursor at `offset` is on the path word of a `src ` +/// statement, return that import. Used by `goto_definition` to jump +/// to the imported module. +fn src_import_at( + document: &vw_htcl::Document, + offset: u32, +) -> Option<&vw_htcl::SrcImport> { + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + if import.path_span.contains(offset) { + return Some(import); + } + } + None +} + +// --- doc-comment lookup --------------------------------------------------- + +fn proc_doc_comments_for( + document: &vw_htcl::Document, + proc: &vw_htcl::Proc, +) -> Vec { + proc_doc_comments_for_in(&document.stmts, proc).unwrap_or_default() +} + +fn proc_doc_comments_for_in( + stmts: &[Stmt], + proc: &vw_htcl::Proc, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(p) + // Pointer-identity match: `proc` was looked up out + // of this same parse, so its address inside the AST + // is unique. + if std::ptr::eq(p, proc) => { + return Some(cmd.doc_comments.clone()); + } + CommandKind::NamespaceEval(ns) => { + if let Some(found) = proc_doc_comments_for_in(&ns.body, proc) { + return Some(found); + } + } + _ => {} + } + } + None +} + +fn proc_doc_comments_by_name( + document: &vw_htcl::Document, + name: &str, +) -> Vec { + proc_doc_comments_by_name_in(&document.stmts, "", name).unwrap_or_default() +} + +fn proc_doc_comments_by_name_in( + stmts: &[Stmt], + prefix: &str, + name: &str, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(p) => { + let Some(decl_name) = p.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + decl_name.to_string() + } else { + format!("{prefix}::{decl_name}") + }; + if qualified == name { + return Some(cmd.doc_comments.clone()); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(ns_name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + ns_name.to_string() + } else { + format!("{prefix}::{ns_name}") + }; + if let Some(found) = + proc_doc_comments_by_name_in(&ns.body, &nested, name) + { + return Some(found); + } + } + _ => {} + } + } + None +} + +/// Render a type expression in the canonical user-facing form — +/// `dict`, `list`, etc. Used by hover and +/// signature-help so the displayed type matches what the user +/// would write in source. +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + match ty { + vw_htcl::TypeExpr::Named { name, .. } => name.clone(), + vw_htcl::TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(",")) + } + vw_htcl::TypeExpr::Qualified { + namespace, variant, .. + } => { + format!("{namespace}::{variant}") + } + } +} + +// --- markdown formatters -------------------------------------------------- + +fn format_hover(target: &HoverTarget, proc_doc_comments: &[String]) -> String { + match target { + HoverTarget::ProcDef { proc, .. } => format_proc( + proc.name.as_deref().unwrap_or(""), + proc.signature.as_ref(), + proc_doc_comments, + ), + HoverTarget::CallSite { + proc_name, + signature, + .. + } => format_proc(proc_name, Some(signature), proc_doc_comments), + HoverTarget::ProcArgDef { arg, .. } + | HoverTarget::CallArg { arg, .. } => format_arg(arg), + HoverTarget::LocalVar { name, ty, .. } => { + format_local_var(name, ty.as_ref()) + } + HoverTarget::EnumDef { decl, .. } => format_enum(decl), + HoverTarget::TypeDef { decl, .. } => format_type_def(decl), + } +} + +fn format_type_def(decl: &vw_htcl::TypeDecl) -> String { + let name = decl.name.as_deref().unwrap_or(""); + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match decl.underlying.as_ref() { + Some(ty) => writeln!(out, "type {name} = {}", render_type(ty)).unwrap(), + None => writeln!(out, "type {name} = ").unwrap(), + } + writeln!(out, "```").unwrap(); + out +} + +fn format_enum(decl: &vw_htcl::EnumDecl) -> String { + let mut out = String::new(); + let name = decl.name.as_deref().unwrap_or(""); + writeln!(out, "```htcl").unwrap(); + writeln!(out, "enum {name} = {{").unwrap(); + for v in &decl.variants { + match v.payload.as_ref() { + Some(p) => { + writeln!(out, " {}: {}", v.name, render_type(p)).unwrap() + } + None => writeln!(out, " {}", v.name).unwrap(), + } + } + writeln!(out, "}}").unwrap(); + writeln!(out, "```").unwrap(); + out.push_str("\nTagged sum type. The compiler auto-generates "); + out.push_str("constructors (`::`), repr, and "); + out.push_str("`tag`/`payload` accessors. See "); + out.push_str("docs/htcl-enums.md for the full semantics.\n"); + out +} + +fn format_local_var(name: &str, ty: Option<&vw_htcl::TypeExpr>) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match ty { + Some(t) => writeln!(out, "${name}: {}", render_type(t)).unwrap(), + None => writeln!(out, "${name}").unwrap(), + } + writeln!(out, "```").unwrap(); + out.push_str("\nLocal variable.\n"); + out +} + +fn format_proc( + name: &str, + signature: Option<&ProcSignature>, + proc_doc_comments: &[String], +) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + // Include the return type in the proc header when annotated: + // proc foo → string + // Unannotated procs render unchanged (`proc foo`). + let return_ty = signature.and_then(|s| s.return_type.as_ref()); + match return_ty { + Some(ty) => { + writeln!(out, "proc {name} → {}", render_type(ty)).unwrap(); + } + None => { + writeln!(out, "proc {name}").unwrap(); + } + } + writeln!(out, "```").unwrap(); + let reflowed = vw_htcl::doc::reflow_doc_comments(proc_doc_comments); + if !reflowed.is_empty() { + out.push('\n'); + out.push_str(&reflowed); + out.push('\n'); + } + if let Some(sig) = signature { + if !sig.args.is_empty() { + out.push_str("\n### Parameters\n\n"); + for arg in &sig.args { + match arg.type_annotation.as_ref() { + Some(ty) => { + write!(out, "- `-{}: {}`", arg.name, render_type(ty)) + .unwrap(); + } + None => { + write!(out, "- `-{}`", arg.name).unwrap(); + } + } + let reflowed = + vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + let mut paragraphs = reflowed.split("\n\n"); + if let Some(brief) = paragraphs.next().filter(|s| !s.is_empty()) + { + write!(out, " — {brief}").unwrap(); + } + out.push('\n'); + for extra in paragraphs.filter(|s| !s.is_empty()) { + writeln!(out, " {extra}").unwrap(); + } + for attr in &arg.attributes { + writeln!(out, " - `{}`", format_attribute(attr)).unwrap(); + } + } + } + } + out +} + +fn format_arg(arg: &ProcArg) -> String { + let mut out = String::new(); + writeln!(out, "```htcl").unwrap(); + match arg.type_annotation.as_ref() { + Some(ty) => { + writeln!(out, "-{}: {}", arg.name, render_type(ty)).unwrap() + } + None => writeln!(out, "-{}", arg.name).unwrap(), + } + writeln!(out, "```").unwrap(); + let reflowed = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + if !reflowed.is_empty() { + out.push('\n'); + out.push_str(&reflowed); + out.push('\n'); + } + if !arg.attributes.is_empty() { + out.push('\n'); + for attr in &arg.attributes { + writeln!(out, "- `{}`", format_attribute(attr)).unwrap(); + } + } + out +} + +fn format_attribute(attr: &Attribute) -> String { + if attr.values.is_empty() { + format!("@{}", attr.name) + } else { + let values: Vec = + attr.values.iter().map(format_attribute_value).collect(); + format!("@{}({})", attr.name, values.join(", ")) + } +} + +fn format_attribute_value(v: &AttributeValue) -> String { + v.to_tcl_literal() +} + +fn lc_to_pos(lc: LineCol) -> Position { + Position { + line: lc.line, + character: lc.character, + } +} + +/// True when `offset` lies on a line whose first non-whitespace +/// byte before the cursor is `#`. That's the shape of every htcl +/// comment — standalone `# text`, `## doc comment`, and mid-command +/// `# inline comment` continuations all start `#…` at column 0 +/// of their line. Used by the completion handler to suppress its +/// dropdown while the user is writing prose in a comment. +/// +/// Bytes only (ASCII-fast); no UTF-8 walking. `\n` bounds the +/// backward scan so we never look past the current line. `#` inside +/// a string / bracket on a preceding line can't reach here because +/// the scan stops at the enclosing `\n` first. +fn in_line_comment(source: &str, offset: u32) -> bool { + let bytes = source.as_bytes(); + let end = (offset as usize).min(bytes.len()); + let line_start = bytes[..end] + .iter() + .rposition(|&b| b == b'\n') + .map(|i| i + 1) + .unwrap_or(0); + for &b in &bytes[line_start..end] { + match b { + b' ' | b'\t' | b'\r' => continue, + b'#' => return true, + _ => return false, + } + } + false +} + +/// Parse one htcl `text` and push every `proc` / `type` / `enum` +/// declaration whose name contains `needle` (case-insensitive, empty +/// `needle` matches all) into `out`. Stops as soon as `out` reaches +/// `cap` entries so a `workspace/symbol` request never assembles an +/// unbounded response. Variants of an enum are emitted as siblings +/// with `container_name` set to the enum, matching how +/// rust-analyzer surfaces variants in the workspace picker. +fn collect_workspace_symbols( + uri: &Url, + text: &str, + needle: &str, + out: &mut Vec, + cap: usize, +) { + let parsed = parse(text); + let line_index = LineIndex::new(text); + for stmt in &parsed.document.stmts { + if out.len() >= cap { + return; + } + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + push_symbol( + uri, + &line_index, + name, + proc.name_span, + SymbolKind::FUNCTION, + None, + needle, + out, + ); + } + } + CommandKind::TypeDecl(td) => { + if let Some(name) = td.name.as_deref() { + push_symbol( + uri, + &line_index, + name, + td.name_span, + SymbolKind::STRUCT, + None, + needle, + out, + ); + } + } + CommandKind::EnumDecl(ed) => { + let enum_name = ed.name.as_deref(); + if let Some(name) = enum_name { + push_symbol( + uri, + &line_index, + name, + ed.name_span, + SymbolKind::ENUM, + None, + needle, + out, + ); + } + for v in &ed.variants { + if out.len() >= cap { + return; + } + push_symbol( + uri, + &line_index, + &v.name, + v.name_span, + SymbolKind::ENUM_MEMBER, + enum_name.map(str::to_string), + needle, + out, + ); + } + } + _ => {} + } + } +} + +#[allow(clippy::too_many_arguments)] +fn push_symbol( + uri: &Url, + line_index: &LineIndex, + name: &str, + span: vw_htcl::Span, + kind: SymbolKind, + container_name: Option, + needle: &str, + out: &mut Vec, +) { + if !needle.is_empty() && !name.to_ascii_lowercase().contains(needle) { + return; + } + let (start, end) = line_index.range(span); + #[allow(deprecated)] + out.push(SymbolInformation { + name: name.to_string(), + kind, + tags: None, + deprecated: None, + location: Location { + uri: uri.clone(), + range: Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + }, + }, + container_name, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn uri() -> Url { + Url::parse("file:///tmp/x.htcl").unwrap() + } + + #[tokio::test] + async fn handles_htcl_extension() { + let backend = HtclBackend::new(); + assert!(backend.handles(&uri())); + assert!(!backend.handles(&Url::parse("file:///tmp/x.vhd").unwrap())); + } + + #[tokio::test] + async fn diagnostics_for_unterminated_string() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "puts \"oops\nputs ok\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!(!diags.is_empty(), "expected at least one diagnostic"); + assert_eq!(diags[0].severity, Some(DiagnosticSeverity::ERROR)); + assert!(diags[0].message.contains("unterminated string")); + } + + #[tokio::test] + async fn document_symbols_include_proc() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "## greet someone\nproc greet {name} { puts hi }\n".into(), + ) + .await; + let symbols = backend.document_symbols(&uri()).await; + assert_eq!(symbols.len(), 1); + assert_eq!(symbols[0].name, "greet"); + assert_eq!(symbols[0].kind, SymbolKind::FUNCTION); + assert_eq!(symbols[0].detail.as_deref(), Some("greet someone")); + } + + #[tokio::test] + async fn workspace_symbols_surface_procs_types_and_enum_variants() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc greet {name} { puts hi }\n\ + type Foo = int\n\ + enum Color = {\n Red\n Green\n Blue\n}\n" + .into(), + ) + .await; + let all = backend.workspace_symbols("").await; + let names: Vec<&str> = all.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"greet"), "{names:?}"); + assert!(names.contains(&"Foo"), "{names:?}"); + assert!(names.contains(&"Color"), "{names:?}"); + assert!(names.contains(&"Red"), "{names:?}"); + let red = all.iter().find(|s| s.name == "Red").unwrap(); + assert_eq!(red.kind, SymbolKind::ENUM_MEMBER); + assert_eq!(red.container_name.as_deref(), Some("Color")); + + // Substring filter, case-insensitive. + let filtered = backend.workspace_symbols("gre").await; + assert!(filtered.iter().any(|s| s.name == "greet")); + assert!(filtered.iter().any(|s| s.name == "Green")); + assert!(!filtered.iter().any(|s| s.name == "Foo")); + } + + #[tokio::test] + async fn validator_diagnostics_surface_in_lsp() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc axis {\n @enum(1, 2, 4) width\n} { puts $width }\n\ + axis -width 3\n" + .into(), + ) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!( + diags.iter().any(|d| d.message.contains("@enum")), + "{:?}", + diags + ); + } + + /// Unused-variable warnings from the `vw-htcl::unused` pass + /// reach LSP clients with `DiagnosticSeverity::WARNING` and + /// point at the offending decl. Underscore-prefixed names are + /// exempt. + #[tokio::test] + async fn unused_var_warning_surfaces_in_lsp() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc f {unused_arg} { return 1 }\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + let warnings: Vec<_> = diags + .iter() + .filter(|d| d.severity == Some(DiagnosticSeverity::WARNING)) + .filter(|d| d.message.contains("unused proc arg")) + .collect(); + assert_eq!(warnings.len(), 1, "{:?}", diags); + assert!( + warnings[0].message.contains("unused_arg"), + "{:?}", + warnings[0] + ); + } + + #[tokio::test] + async fn unused_var_underscore_prefix_suppresses_lsp_warning() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc f {_ignored} { return 1 }\n".into()) + .await; + let diags = backend.diagnostics(&uri()).await; + assert!( + !diags.iter().any(|d| d.message.contains("unused")), + "{:?}", + diags + ); + } + + /// Rename produces a WorkspaceEdit whose TextEdits, when + /// applied in reverse order, transform the source correctly. + /// Covers the end-to-end LSP path: cursor → offset → rename_at → + /// edits → LSP `WorkspaceEdit`. + #[tokio::test] + async fn rename_local_via_lsp() { + let backend = HtclBackend::new(); + // `mode` is a local; renaming it should update the decl and + // the two `$mode` refs. + let src = "\ +proc f {} { + set mode fast + puts $mode + return $mode +} +"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `m` of `set mode` (line 1, column 6). 0-indexed. + let workspace_edit = backend + .rename( + &uri(), + Position { + line: 1, + character: 6, + }, + "kind", + ) + .await + .expect("rename should succeed"); + let changes = workspace_edit.changes.expect("expected changes"); + let text_edits = changes.get(&uri()).expect("edits for this file"); + assert_eq!(text_edits.len(), 3, "{text_edits:?}"); + // Apply edits from tail to head to preserve earlier offsets. + let mut renamed = src.to_string(); + let mut edits = text_edits.clone(); + edits.sort_by_key(|e| (e.range.start.line, e.range.start.character)); + for edit in edits.iter().rev() { + let start = position_to_offset(&renamed, edit.range.start); + let end = position_to_offset(&renamed, edit.range.end); + renamed.replace_range(start..end, &edit.new_text); + } + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + assert!(!renamed.contains("mode"), "{renamed}"); + } + + /// Proc-name rename now works within the local file — the + /// declaration span and every call site in the same document + /// get rewritten. Cross-file callers (in other `.htcl` files + /// the LSP hasn't yet been asked about) still need the + /// workspace-scan variant. + #[tokio::test] + async fn rename_proc_name_via_lsp_covers_decl_and_call() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "proc greet {} { puts hi }\ngreet\n".into()) + .await; + // Cursor on the `g` of the proc's own name. + let result = backend + .rename( + &uri(), + Position { + line: 0, + character: 5, + }, + "hello", + ) + .await; + let ws = result.expect("expected an edit set"); + let changes = ws.changes.expect("expected changes map"); + let edits = changes.get(&uri()).expect("edits for the local uri"); + assert_eq!(edits.len(), 2, "decl + one call site"); + } + + #[tokio::test] + async fn references_returns_all_local_call_sites() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "proc greet {} { puts hi }\ngreet\nproc other {} { greet }\n" + .into(), + ) + .await; + let locs = backend + .references( + &uri(), + Position { + line: 0, + character: 5, + }, + true, + ) + .await; + // 3 hits: decl name + top-level call + nested call in `other`. + assert_eq!(locs.len(), 3, "{locs:?}"); + for loc in &locs { + assert_eq!(loc.uri, uri()); + } + } + + #[tokio::test] + async fn references_on_type_covers_annotations() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "type MyThing = string\nproc a {v: MyThing} MyThing { }\nproc b {v: MyThing} { }\n" + .into(), + ) + .await; + // Cursor on `MyThing` at the type decl (char 5..12 = "MyThing"). + let locs = backend + .references( + &uri(), + Position { + line: 0, + character: 5, + }, + true, + ) + .await; + // decl + a's arg-type + a's return-type + b's arg-type = 4. + assert_eq!(locs.len(), 4, "{locs:?}"); + } + + #[tokio::test] + async fn rename_type_covers_all_annotations() { + let backend = HtclBackend::new(); + backend + .set_text_sync( + uri(), + "type MyThing = string\nproc a {v: MyThing} MyThing { }\n" + .into(), + ) + .await; + let ws = backend + .rename( + &uri(), + Position { + line: 0, + character: 5, + }, + "YourThing", + ) + .await + .expect("edit set"); + let changes = ws.changes.expect("changes"); + let edits = changes.get(&uri()).expect("local edits"); + // Same 3 hits: decl + arg-type + return-type. + assert_eq!(edits.len(), 3, "{edits:?}"); + for e in edits { + assert_eq!(e.new_text, "YourThing"); + } + } + + /// Utility: convert an LSP `Position` (line + UTF-16 char offset, + /// but at ASCII we treat as byte offset) into a byte index in the + /// given source. Used to apply text edits in tests. + fn position_to_offset(source: &str, pos: Position) -> usize { + let mut cur_line = 0u32; + let mut cur_col = 0u32; + for (idx, byte) in source.bytes().enumerate() { + if cur_line == pos.line && cur_col == pos.character { + return idx; + } + if byte == b'\n' { + cur_line += 1; + cur_col = 0; + } else { + cur_col += 1; + } + } + source.len() + } + + #[tokio::test] + async fn hover_on_call_site_shows_signature() { + let backend = HtclBackend::new(); + let src = "\ +## Greet someone by name.\n\ +proc greet {\n\ + ## Who to greet.\n\ + @default(\"world\") name\n\ +} { puts \"hi $name\" }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `g` of the call-site `greet`. Line indices + // are 0-based. + let hover = backend + .hover( + &uri(), + Position { + line: 5, + character: 0, + }, + ) + .await + .expect("hover should return content"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!("expected markup"), + }; + assert!(body.contains("proc greet"), "{body}"); + assert!(body.contains("Greet someone by name."), "{body}"); + assert!(body.contains("### Parameters"), "{body}"); + assert!(body.contains("-name"), "{body}"); + assert!(body.contains("Who to greet."), "{body}"); + assert!(body.contains("@default"), "{body}"); + } + + #[tokio::test] + async fn hover_on_call_arg_shows_arg_doc() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {\n\ + ## Who to greet.\n\ + @default(\"world\") name\n\ +} { puts hi }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Position cursor on `-name` of the call site (line 4 in the + // 0-indexed scheme). + let hover = backend + .hover( + &uri(), + Position { + line: 4, + character: 7, + }, + ) + .await + .expect("hover should return content"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!("expected markup"), + }; + assert!(body.contains("-name"), "{body}"); + assert!(body.contains("Who to greet."), "{body}"); + assert!(body.contains("@default"), "{body}"); + // Shouldn't include the proc-level header. + assert!(!body.contains("### Parameters"), "{body}"); + } + + #[tokio::test] + async fn hover_outside_known_construct_returns_none() { + let backend = HtclBackend::new(); + backend + .set_text_sync(uri(), "puts hello world\n".into()) + .await; + let hover = backend + .hover( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(hover.is_none()); + } + + #[tokio::test] + async fn goto_definition_jumps_call_to_proc_decl() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {\n name\n} { puts hi }\n\ +greet -name there\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the `g` of the call-site `greet` (line 3). + let locs = backend + .goto_definition( + &uri(), + Position { + line: 3, + character: 0, + }, + ) + .await; + assert_eq!(locs.len(), 1); + // Decl name `greet` is on line 0 at character 5. + assert_eq!(locs[0].range.start.line, 0); + assert_eq!(locs[0].range.start.character, 5); + } + + #[tokio::test] + async fn goto_definition_resolves_attribute_ident() { + let backend = HtclBackend::new(); + let src = "\ +proc f {\n has_a\n @requires(has_a) has_b\n} { }\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on `has_a` inside `@requires(has_a)`. + let locs = backend + .goto_definition( + &uri(), + Position { + line: 2, + character: 13, + }, + ) + .await; + assert_eq!(locs.len(), 1); + // Decl `has_a` is on line 1 at character 2. + assert_eq!(locs[0].range.start.line, 1); + assert_eq!(locs[0].range.start.character, 2); + } + + #[tokio::test] + async fn completion_offers_proc_names_in_command_position() { + let backend = HtclBackend::new(); + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gr\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor at end of `gr` on line 2. + let items = backend + .completion( + &uri(), + Position { + line: 2, + character: 2, + }, + ) + .await; + let mut labels: Vec = + items.iter().map(|i| i.label.clone()).collect(); + labels.sort(); + assert_eq!(labels, vec!["greet", "grumble"]); + assert_eq!(items[0].kind, Some(CompletionItemKind::FUNCTION)); + } + + #[test] + fn in_line_comment_recognizes_standalone_and_indented_comments() { + // Cursor after `# hello` on a standalone comment line. + let src = "# hello world\nputs hi\n"; + assert!(in_line_comment(src, 8), "cursor after `#` should count"); + assert!(!in_line_comment(src, 0), "cursor before `#` should not"); + // Indented comment (common inline_comment shape inside a + // multi-line call). + let src = " # inner\nputs hi\n"; + assert!(in_line_comment(src, 5), "cursor after indent + `#`"); + // Doc comments (`##`) share the same shape — start with `#`. + let src = "## doc\n"; + assert!(in_line_comment(src, 4)); + } + + #[test] + fn in_line_comment_returns_false_on_code_line() { + // A `#` appearing later inside a code line (e.g., inside a + // quoted string) is NOT the start of a comment in htcl — + // the check requires `#` to be the FIRST non-ws char of + // the line before the cursor. + let src = "puts \"hi # not comment\"\n"; + // Cursor right after `#`. + assert!(!in_line_comment(src, 10)); + // Cursor mid-code, no `#` on the line. + assert!(!in_line_comment(src, 3)); + } + + #[test] + fn in_line_comment_scan_stops_at_line_boundary() { + // A `#` on the PREVIOUS line must not leak into the + // current line's classification — the backward scan must + // stop at `\n`. + let src = "# prev line\nfoo bar\n"; + // Cursor on the code line at `bar`. + let off = src.find("bar").unwrap() as u32; + assert!(!in_line_comment(src, off)); + } + + #[tokio::test] + async fn completion_returns_empty_when_cursor_is_inside_a_comment() { + // Regression: without this suppression, Helix's LSP client + // auto-triggers completion on every keystroke inside a + // `#` comment line, so writing prose pops a dropdown on + // each space. The suppression fires when the cursor sits + // on a line whose first non-ws char before the cursor is + // `#`. + let backend = HtclBackend::new(); + let src = "\ +proc greet {} { }\n\ +# writing a note about greet\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor at end of the comment line (line 1, char 27). + let items = backend + .completion( + &uri(), + Position { + line: 1, + character: 27, + }, + ) + .await; + assert!( + items.is_empty(), + "expected no completions inside comment, got {items:?}", + ); + // Sanity: completion on the next (code) line still fires. + let src = "\ +proc greet {} { }\n\ +# writing a note about greet\n\ +gr\n"; + backend.set_text_sync(uri(), src.into()).await; + let items = backend + .completion( + &uri(), + Position { + line: 2, + character: 2, + }, + ) + .await; + assert!( + items.iter().any(|i| i.label == "greet"), + "expected greet in completions on code line, got {items:?}", + ); + } + + #[tokio::test] + async fn completion_offers_flags_in_argument_position() { + let backend = HtclBackend::new(); + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg \n"; + backend.set_text_sync(uri(), src.into()).await; + // Line 4, just after `cfg ` (character 4). + let items = backend + .completion( + &uri(), + Position { + line: 4, + character: 4, + }, + ) + .await; + let mut labels: Vec = + items.iter().map(|i| i.label.clone()).collect(); + labels.sort(); + assert_eq!(labels, vec!["-depth", "-width"]); + assert_eq!(items[0].kind, Some(CompletionItemKind::FIELD)); + } + + #[tokio::test] + async fn signature_help_highlights_active_parameter() { + let backend = HtclBackend::new(); + let src = "\ +## Configure the bus.\n\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -depth \n"; + backend.set_text_sync(uri(), src.into()).await; + // Line 5, after `cfg -depth ` (character 11). + let help = backend + .signature_help( + &uri(), + Position { + line: 5, + character: 11, + }, + ) + .await + .expect("signature help expected"); + assert_eq!(help.active_parameter, Some(1)); + let info = &help.signatures[0]; + assert!(info.label.starts_with("cfg "), "{}", info.label); + assert_eq!(info.parameters.as_ref().unwrap().len(), 2); + match &info.documentation { + Some(Documentation::MarkupContent(m)) => { + assert!(m.value.contains("Configure the bus."), "{}", m.value); + } + other => panic!("expected markup documentation, got {other:?}"), + } + } + + #[tokio::test] + async fn signature_help_includes_return_type_arrow() { + let backend = HtclBackend::new(); + let src = "\ +proc make_widget {} bd_cell { return foo }\n\ +make_widget \n"; + backend.set_text_sync(uri(), src.into()).await; + let help = backend + .signature_help( + &uri(), + Position { + line: 1, + character: 12, + }, + ) + .await + .expect("signature help expected"); + let info = &help.signatures[0]; + // Label should carry the `→ bd_cell` suffix. + assert!(info.label.contains("→ bd_cell"), "{}", info.label); + } + + #[tokio::test] + async fn hover_on_enum_decl_shows_variants() { + let backend = HtclBackend::new(); + let src = "\ +enum Property = {\n Scalar: string\n Nested: int\n}\n"; + backend.set_text_sync(uri(), src.into()).await; + // Cursor on the enum name (line 0, col 5: 'Property'). + let hover = backend + .hover( + &uri(), + Position { + line: 0, + character: 7, + }, + ) + .await + .expect("hover on enum decl name"); + if let HoverContents::Markup(MarkupContent { value, .. }) = + hover.contents + { + assert!(value.contains("enum Property"), "{value}"); + assert!(value.contains("Scalar: string"), "{value}"); + assert!(value.contains("Nested: int"), "{value}"); + } else { + panic!("expected Markup hover"); + } + } + + #[tokio::test] + async fn hover_proc_def_includes_return_type() { + let backend = HtclBackend::new(); + let src = "\ +## Builds a widget.\n\ +proc make_widget {} dict { return {} }\n"; + backend.set_text_sync(uri(), src.into()).await; + // Hover on the proc name `make_widget` at line 1. + let hover = backend + .hover( + &uri(), + Position { + line: 1, + character: 8, + }, + ) + .await + .expect("hover expected on proc def"); + if let HoverContents::Markup(MarkupContent { value, .. }) = + hover.contents + { + assert!( + value.contains("→ dict"), + "expected return type in hover: {value}" + ); + } else { + panic!("expected Markup hover contents"); + } + } + + #[tokio::test] + async fn signature_help_none_outside_call() { + let backend = HtclBackend::new(); + backend.set_text_sync(uri(), "puts hi\n".into()).await; + let help = backend + .signature_help( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(help.is_none()); + } + + #[tokio::test] + async fn goto_definition_unknown_returns_empty() { + let backend = HtclBackend::new(); + backend.set_text_sync(uri(), "puts hello\n".into()).await; + let locs = backend + .goto_definition( + &uri(), + Position { + line: 0, + character: 0, + }, + ) + .await; + assert!(locs.is_empty()); + } + + // --- cross-file (workspace view) tests -------------------------------- + + /// Build a temp workspace with a `lib.htcl` defining `greet` and + /// a `main.htcl` that imports it. Returns the backend with both + /// files already opened and the URIs. + async fn temp_workspace_with_import() -> ( + tempfile::TempDir, + HtclBackend, + Url, // main.htcl + Url, // lib.htcl + ) { + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("lib.htcl"); + std::fs::write( + &lib_path, + "## Greet someone.\n\ +proc greet {\n ## Who to greet.\n who\n} { puts \"hi $who\" }\n", + ) + .unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\ngreet -who world\n"; + std::fs::write(&main_path, main_src).unwrap(); + + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + (dir, backend, main_uri, lib_uri) + } + + #[tokio::test] + async fn goto_on_src_import_jumps_to_imported_file() { + let (_dir, backend, main_uri, lib_uri) = + temp_workspace_with_import().await; + // Cursor on the `l` of `src lib` (line 0, col 4). + let locs = backend + .goto_definition( + &main_uri, + Position { + line: 0, + character: 4, + }, + ) + .await; + assert_eq!(locs.len(), 1); + assert_eq!(locs[0].uri, lib_uri); + } + + #[tokio::test] + async fn goto_on_call_to_imported_proc_jumps_to_lib() { + let (_dir, backend, main_uri, lib_uri) = + temp_workspace_with_import().await; + // Cursor on `greet` at line 1. + let locs = backend + .goto_definition( + &main_uri, + Position { + line: 1, + character: 0, + }, + ) + .await; + assert_eq!(locs.len(), 1, "{locs:?}"); + assert_eq!(locs[0].uri, lib_uri); + // The declaration of `greet` is on lib.htcl line 1 col 5. + assert_eq!(locs[0].range.start.line, 1); + assert_eq!(locs[0].range.start.character, 5); + } + + /// Regression: a call from inside an `if { … }` body should + /// still find its proc's declaration. The parser leaves the + /// brace-body as an opaque word, so without an explicit + /// reparse pass in [`vw_htcl::goto`] the search never reaches + /// the nested call. + #[tokio::test] + async fn goto_from_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + target -x 1\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + let locs = backend + .goto_definition( + &uri, + Position { + line: 3, + character: 4, + }, + ) + .await; + assert!( + !locs.is_empty(), + "goto-def from inside `if {{…}}` body failed" + ); + } + + /// Regression: a call from inside `[…]` command substitution + /// inside `if {…} { … }` — the double-nested shape the IP + /// wrapper's `if {$bd} { set cell [create_bd_cell …] } + /// else { set cell [create_ip …] }` scaffold produces. The + /// reparse pass has to also run `populate_procs` so the + /// inner CmdSubst.body gets filled in. + #[tokio::test] + async fn goto_from_cmdsubst_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + set cell [target -x 1]\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + // Cursor on `target` inside `[target -x 1]` on line 3. + // Line 3 is ` set cell [target -x 1]`; `target` starts + // at col 17. + let locs = backend + .goto_definition( + &uri, + Position { + line: 3, + character: 17, + }, + ) + .await; + assert!( + !locs.is_empty(), + "goto-def from inside `[[…]]`-inside-`if` failed" + ); + } + + /// Companion to [`goto_from_cmdsubst_inside_if_body`] for hover. + #[tokio::test] + async fn hover_from_cmdsubst_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "## Target proc doc.\n\ + proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + set cell [target -x 1]\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + // Cursor on `target` inside `[target -x 1]` on line 4. + let hover = backend + .hover( + &uri, + Position { + line: 4, + character: 17, + }, + ) + .await; + assert!( + hover.is_some(), + "hover from inside `[[…]]`-inside-`if` returned None" + ); + } + + /// Same regression as [`goto_from_inside_if_body`], but for + /// hover — the two share the "reparse brace-body" fix in + /// [`vw_htcl::goto`] / [`vw_htcl::hover`]. + #[tokio::test] + async fn hover_from_inside_if_body() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("m.htcl"); + let src = "## Target proc doc.\n\ + proc target { x } { }\n\ + proc caller { } {\n \ + if {1} {\n \ + target -x 1\n \ + }\n\ + }\n"; + std::fs::write(&path, src).unwrap(); + let backend = HtclBackend::new(); + let uri = Url::from_file_path(&path).unwrap(); + backend.set_text_sync(uri.clone(), src.into()).await; + let hover = backend + .hover( + &uri, + Position { + line: 4, + character: 4, + }, + ) + .await; + assert!( + hover.is_some(), + "hover from inside `if {{…}}` body returned None" + ); + } + + /// Reproduces the exact user scenario against the on-disk + /// `~/src/htcl/amd/` tree. Only runs when that path exists, so + /// the test is a no-op in CI / fresh checkouts. + #[tokio::test] + async fn goto_finds_sibling_workspace_dep_real_htcl_tree() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + eprintln!("skipping — {} not present", cpm5_module.display()); + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + + // Find the line + column of `vivado_cmd::set_property` — + // avoids hard-coding a line number that will drift as the + // wrapper regenerates. + let mut target_line = None; + for (i, line) in text.lines().enumerate() { + if let Some(col) = line.find("vivado_cmd::set_property") { + // Cursor on the `set_property` word, past the + // `vivado_cmd::` prefix (12 chars). + target_line = Some((i as u32, (col + 12) as u32)); + break; + } + } + let Some((line, character)) = target_line else { + panic!("no `vivado_cmd::set_property` in cpm5/module.htcl"); + }; + let locs = backend + .goto_definition(&cpm5_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto-def against real htcl tree returned no location \ + for cpm5/module.htcl:{line}:{character}" + ); + let hit = &locs[0]; + let path = hit.uri.to_file_path().unwrap(); + assert!( + path.to_string_lossy().contains("vivado-cmd"), + "expected to land in the vivado-cmd tree, got {:?}", + hit + ); + } + + /// Regression against the on-disk cpm5 tree for goto and hover + /// on `vivado_cmd::create_bd_cell` — the sole cell-creation + /// call at the top of `create_cpm5`. (Previously covered + /// `vivado_cmd::create_ip` too, but the IP generator's + /// `-bd 0` path is now rejected up front with an `error`, so + /// the generated wrapper only calls `create_bd_cell`.) + #[tokio::test] + async fn goto_and_hover_on_create_bd_cell_and_create_ip_in_cpm5() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + for needle in &["vivado_cmd::create_bd_cell"] { + let (line, character) = text + .lines() + .enumerate() + .find_map(|(i, l)| { + l.find(needle).map(|c| (i as u32, (c + 12) as u32)) + }) + .unwrap_or_else(|| panic!("no {needle} in cpm5/module.htcl")); + let locs = backend + .goto_definition(&cpm5_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto-def on {needle} at {line}:{character} returned nothing" + ); + let hover = + backend.hover(&cpm5_uri, Position { line, character }).await; + assert!( + hover.is_some(), + "hover on {needle} at {line}:{character} returned None" + ); + } + } + + /// Companion to [`goto_finds_sibling_workspace_dep_real_htcl_tree`] + /// for hover — same file, same cursor position, same expected + /// outcome: the imported proc's signature resolves and hover + /// returns something rather than `None`. + #[tokio::test] + async fn hover_finds_imported_proc_real_htcl_tree() { + let cpm5_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/cpm5/module.htcl"); + if !cpm5_module.exists() { + return; + } + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + let text = std::fs::read_to_string(&cpm5_module).unwrap(); + backend.set_text_sync(cpm5_uri.clone(), text.clone()).await; + let target = text.lines().enumerate().find_map(|(i, line)| { + line.find("vivado_cmd::set_property") + .map(|col| (i as u32, (col + 12) as u32)) + }); + let Some((line, character)) = target else { + panic!("no `vivado_cmd::set_property` in cpm5/module.htcl"); + }; + let hover = + backend.hover(&cpm5_uri, Position { line, character }).await; + assert!( + hover.is_some(), + "hover against real htcl tree returned None \ + for cpm5/module.htcl:{line}:{character}" + ); + } + + /// Hover + goto on a namespaced newtype (`dcmac::MacPortProps`) + /// used as a return-type annotation resolve to the type + /// declaration. Guards the analyzer's type-annotation path + /// against regressions and validates end-to-end with a real + /// generated wrapper. + #[tokio::test] + async fn hover_goto_on_namespaced_newtype_return_type() { + let dcmac_module = + std::path::PathBuf::from("/home/ry/src/htcl/amd/dcmac/module.htcl"); + if !dcmac_module.exists() { + return; + } + let backend = HtclBackend::new(); + let dcmac_uri = Url::from_file_path(&dcmac_module).unwrap(); + let text = std::fs::read_to_string(&dcmac_module).unwrap(); + backend.set_text_sync(dcmac_uri.clone(), text.clone()).await; + // Find a real type-annotation site (arg-type slot on + // `MacPortProps::from` etc.) — not the `namespace eval + // dcmac::MacPortProps {}` word, which passes the string as + // a namespace name rather than a type annotation. + let target = text.lines().enumerate().find_map(|(i, line)| { + line.find(": dcmac::MacPortProps") + .map(|col| (i as u32, (col + 9) as u32)) + }); + let Some((line, character)) = target else { + panic!("no `: dcmac::MacPortProps` in dcmac/module.htcl"); + }; + let hover = backend + .hover(&dcmac_uri, Position { line, character }) + .await; + assert!( + hover.is_some(), + "hover on `dcmac::MacPortProps` at line {line}:{character} \ + returned None — type-annotation path not wired" + ); + let locs = backend + .goto_definition(&dcmac_uri, Position { line, character }) + .await; + assert!( + !locs.is_empty(), + "goto on `dcmac::MacPortProps` at line {line}:{character} \ + returned no locations" + ); + } + + /// Sibling-workspace fallback with a NESTED src chain — mirrors + /// the real vivado-cmd layout where `module.htcl` re-sources + /// per-command files under `cmd/`. `set_property` doesn't live + /// in the module.htcl entry directly; it's reached through + /// `src "cmd/set_property.htcl"` inside the dep module. This + /// caught the actual reproduction case where a shallower test + /// (proc in the dep's module.htcl) passed but goto-def against + /// the real vivado-cmd tree still returned nothing. + #[tokio::test] + async fn goto_finds_sibling_workspace_dep_via_nested_src() { + let dir = tempfile::tempdir().unwrap(); + let amd = dir.path().join("amd"); + let cpm5 = amd.join("cpm5"); + let vivado_cmd = amd.join("vivado-cmd"); + let vivado_cmd_cmd = vivado_cmd.join("cmd"); + std::fs::create_dir_all(&cpm5).unwrap(); + std::fs::create_dir_all(&vivado_cmd_cmd).unwrap(); + std::fs::write( + cpm5.join("vw.toml"), + "[workspace]\nname=\"cpm5\"\nversion=\"0.1.0\"\n\n[dependencies]\n", + ) + .unwrap(); + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n\n\ + [dependencies]\n", + ) + .unwrap(); + // vivado-cmd/module.htcl re-sources set_property.htcl — + // matching the real layout. + std::fs::write( + vivado_cmd.join("module.htcl"), + "src \"cmd/set_property.htcl\"\n", + ) + .unwrap(); + // vivado-cmd/cmd/set_property.htcl defines the proc. + let set_property_path = vivado_cmd_cmd.join("set_property.htcl"); + std::fs::write( + &set_property_path, + "namespace eval vivado_cmd {\n \ + proc set_property { args } { }\n}\n", + ) + .unwrap(); + let cpm5_module = cpm5.join("module.htcl"); + std::fs::write( + &cpm5_module, + "src @vivado-cmd\nvivado_cmd::set_property -dict {} -objects x\n", + ) + .unwrap(); + + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + backend + .set_text_sync( + cpm5_uri.clone(), + std::fs::read_to_string(&cpm5_module).unwrap(), + ) + .await; + // Cursor on `set_property` — the call is + // `vivado_cmd::set_property ...` (col 0). `vivado_cmd::` + // is 12 chars; `set_property` starts at col 12. + let locs = backend + .goto_definition( + &cpm5_uri, + Position { + line: 1, + character: 12, + }, + ) + .await; + assert!(!locs.is_empty(), "goto-def returned no location"); + let set_property_uri = Url::from_file_path(&set_property_path).unwrap(); + assert_eq!( + locs[0].uri, set_property_uri, + "expected jump to {set_property_uri}, got {:?}", + locs[0] + ); + } + + /// Sibling-workspace fallback: when a file's own workspace + /// doesn't declare a `@dep/…` import but a sibling directory + /// under a shared parent DOES have its own `vw.toml` with a + /// matching basename, the resolver should still find it. + /// + /// Layout (mirrors `~/src/htcl/amd/{cpm5,vivado-cmd}` as the + /// user's actual reproduction): + /// + /// /amd/cpm5/vw.toml # empty deps + /// /amd/cpm5/module.htcl # calls vivado_cmd::foo + /// /amd/vivado-cmd/vw.toml + /// /amd/vivado-cmd/module.htcl # namespace eval vivado_cmd { proc foo … } + /// + /// Regression for "goto-def returns 'No definition found' once + /// I'm in a vw-tracked dependency." + #[tokio::test] + async fn goto_finds_sibling_workspace_dep() { + let dir = tempfile::tempdir().unwrap(); + let amd = dir.path().join("amd"); + let cpm5 = amd.join("cpm5"); + let vivado_cmd = amd.join("vivado-cmd"); + std::fs::create_dir_all(&cpm5).unwrap(); + std::fs::create_dir_all(&vivado_cmd).unwrap(); + std::fs::write( + cpm5.join("vw.toml"), + "[workspace]\nname=\"cpm5\"\nversion=\"0.1.0\"\n\n[dependencies]\n", + ) + .unwrap(); + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n\n\ + [dependencies]\n", + ) + .unwrap(); + // The vivado-cmd module: define namespace `vivado_cmd` with + // a `foo` proc so a call to `vivado_cmd::foo` from cpm5 has + // somewhere to land. + let vivado_module = vivado_cmd.join("module.htcl"); + std::fs::write( + &vivado_module, + "namespace eval vivado_cmd {\n proc foo { x } { }\n}\n", + ) + .unwrap(); + let cpm5_module = cpm5.join("module.htcl"); + std::fs::write(&cpm5_module, "src @vivado-cmd\nvivado_cmd::foo -x 1\n") + .unwrap(); + + let backend = HtclBackend::new(); + let cpm5_uri = Url::from_file_path(&cpm5_module).unwrap(); + backend + .set_text_sync( + cpm5_uri.clone(), + std::fs::read_to_string(&cpm5_module).unwrap(), + ) + .await; + + // Cursor on `foo` — line 1, at the start of the call word + // (`vivado_cmd::foo` starts at column 0, `foo` starts after + // `vivado_cmd::` which is 12 chars). + let locs = backend + .goto_definition( + &cpm5_uri, + Position { + line: 1, + character: 12, + }, + ) + .await; + assert!(!locs.is_empty(), "goto-def returned no location"); + let vivado_uri = Url::from_file_path(&vivado_module).unwrap(); + assert_eq!(locs[0].uri, vivado_uri, "landed on wrong file"); + } + + #[tokio::test] + async fn completion_in_command_position_lists_imported_procs() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Append a partial proc name at end of file so cursor lands in + // command position. + let new_text = "src lib\ngreet -who world\ngre\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + let items = backend + .completion( + &main_uri, + Position { + line: 2, + character: 3, + }, + ) + .await; + let labels: Vec<_> = items.iter().map(|i| i.label.as_str()).collect(); + assert!(labels.contains(&"greet"), "labels = {labels:?}"); + } + + #[tokio::test] + async fn hover_on_imported_call_shows_signature() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Hover on `greet` on line 1. + let hover = backend + .hover( + &main_uri, + Position { + line: 1, + character: 0, + }, + ) + .await + .expect("hover"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!(), + }; + assert!(body.contains("proc greet"), "{body}"); + assert!(body.contains("Greet someone."), "{body}"); + assert!(body.contains("-who"), "{body}"); + } + + #[tokio::test] + async fn diagnostics_accept_calls_to_imported_procs() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // No errors when the call matches the imported signature. + let diags = backend.diagnostics(&main_uri).await; + let errs: Vec<_> = diags + .iter() + .filter(|d| d.severity == Some(DiagnosticSeverity::ERROR)) + .collect(); + assert!(errs.is_empty(), "{errs:?}"); + } + + #[tokio::test] + async fn hover_works_on_call_inside_command_substitution() { + // Mirrors the user's cips.htcl shape: + // src lib + // set cell [greet -who x] + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + let new_text = "src lib\nset cell [greet -who x]\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + // Cursor on `greet` inside the `[ … ]` on line 1. + let hover = backend + .hover( + &main_uri, + Position { + line: 1, + character: 11, + }, + ) + .await + .expect("hover should resolve calls inside `[…]`"); + let body = match hover.contents { + HoverContents::Markup(m) => m.value, + _ => panic!(), + }; + assert!(body.contains("proc greet"), "{body}"); + } + + #[tokio::test] + async fn signature_help_works_on_call_inside_command_substitution() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + // Cursor right after `greet ` inside `[ … ]`. + let new_text = "src lib\nset cell [greet ]\n"; + backend + .set_text_sync(main_uri.clone(), new_text.into()) + .await; + let help = backend + .signature_help( + &main_uri, + Position { + line: 1, + character: 16, + }, + ) + .await + .expect("signature help inside `[…]`"); + assert!( + help.signatures[0].label.starts_with("greet"), + "{:?}", + help.signatures[0].label + ); + } + + #[tokio::test] + async fn diagnostics_still_flag_wrong_flag_on_imported_call() { + let (_dir, backend, main_uri, _lib_uri) = + temp_workspace_with_import().await; + backend + .set_text_sync( + main_uri.clone(), + "src lib\ngreet -whoz world\n".into(), + ) + .await; + let diags = backend.diagnostics(&main_uri).await; + assert!( + diags + .iter() + .any(|d| d.message.contains("undefined argument -whoz")), + "{diags:?}" + ); + } + + #[tokio::test] + async fn workspace_diagnostics_surface_errors_in_imported_files() { + // Break the imported lib (return with a value in an + // unannotated proc — one of the new checks) and open the + // main file that `src`s it. The main file itself is + // error-free. workspace_diagnostics must report the lib's + // diagnostic against the LIB's URI so the editor's + // workspace picker points to the right file. + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("broken.htcl"); + std::fs::write(&lib_path, "proc broken {} { return 42 }\n").unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src broken\n"; + std::fs::write(&main_path, main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + // Set the editor's workspace root to the temp dir so the + // filter accepts the lib file (which lives inside it). + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + // Main entry gets an entry (possibly empty), so the editor + // can clear stale state. + assert!(ws.contains_key(&main_uri), "main uri missing: {ws:?}"); + let lib_diags = ws.get(&lib_uri).unwrap_or_else(|| { + panic!("no diagnostics routed to {lib_uri}: {ws:?}") + }); + assert!( + lib_diags + .iter() + .any(|d| d.message.contains("no declared return type")), + "expected the return-in-unannotated-proc error in lib: {lib_diags:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_clear_when_import_is_fixed() { + // When the user fixes an error in an imported file, the + // next workspace_diagnostics call must include an + // entry for that file — with an EMPTY diagnostic list. + // That empty payload is what the editor overwrites its + // cached "had errors" state with; without it, the + // stale errors linger in `space-D` even after the fix. + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("lib.htcl"); + std::fs::write(&lib_path, "proc broken {} { return 42 }\n").unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\n"; + std::fs::write(&main_path, main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + // Sanity: broken lib produces a workspace diagnostic. + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + assert!( + !ws.get(&lib_uri).map(|v| v.is_empty()).unwrap_or(true), + "expected non-empty lib diagnostics before fix: {ws:?}", + ); + // Fix the lib on disk. Since main.htcl is what's open, + // resetting main's text re-triggers the workspace build + // and reloads lib from disk. + std::fs::write(&lib_path, "proc fixed {} { puts hi }\n").unwrap(); + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + // The lib URI must appear with an empty list so the + // editor clears its cached errors. + let lib_after = ws.get(&lib_uri).unwrap_or_else(|| { + panic!("lib uri missing from post-fix workspace diags: {ws:?}") + }); + assert!( + lib_after.is_empty(), + "expected empty lib diagnostics after fix, got {lib_after:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_clear_when_open_import_is_fixed() { + // Regression for the "stuck diagnostic in Helix" bug: + // both `main.htcl` and `lib.htcl` are open. `main` srcs + // `lib`. Fixing `lib.htcl` should IMMEDIATELY clear its + // diagnostics — even though `main`'s analysis (which + // still holds a cross-file diag pointing at `lib`) hasn't + // been reindexed yet. The aggregator must trust `lib`'s + // own opened analysis over any stale cross-file entries + // targeting it from other files. + let dir = tempfile::tempdir().unwrap(); + let lib_path = dir.path().join("lib.htcl"); + let bad_lib = "proc broken {} { return 42 }\n"; + std::fs::write(&lib_path, bad_lib).unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\n"; + std::fs::write(&main_path, main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + // Open both. `main`'s analysis will contain a cross-file + // diagnostic for `lib`; `lib`'s own analysis will contain + // its own local diagnostic. + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + backend.set_text_sync(lib_uri.clone(), bad_lib.into()).await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + assert!( + !ws.get(&lib_uri).map(|v| v.is_empty()).unwrap_or(true), + "expected non-empty lib diagnostics before fix: {ws:?}", + ); + // Fix `lib` via the OPEN buffer — but DO NOT touch `main`. + // `main`'s analysis is now stale; its cross-file entry + // still points at `lib:`. + backend + .set_text_sync(lib_uri.clone(), "proc fixed {} unit {}\n".into()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + let lib_after = ws.get(&lib_uri).unwrap_or_else(|| { + panic!("lib uri missing from post-fix workspace diags: {ws:?}") + }); + assert!( + lib_after.is_empty(), + "expected empty lib diagnostics after fixing the open buffer, got \ + {lib_after:?} (main.htcl's stale analysis re-injected them)", + ); + } + + /// Wait until a preloaded/virtual-open URI has a committed + /// analysis. Test-only helper — `wait_for_reindex` only fires + /// on the NEXT commit, so it hangs when the preload's indexer + /// has already committed by the time the test observer + /// subscribes. + #[cfg(test)] + async fn wait_until_analysis_present(backend: &HtclBackend, uri: &Url) { + for _ in 0..200 { + if backend.analysis_for(uri).await.is_some() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + panic!("timed out waiting for analysis of {uri}"); + } + + /// Poll `analysis_for(uri)` until the returned analysis + /// satisfies `predicate`, or timeout. Test-only helper — the + /// fan-out reindex fires a NEW commit under a NEW generation, + /// so callers waiting for downstream refresh can't just re-use + /// `wait_for_reindex`; they need a predicate that recognizes + /// "the analysis I want has arrived." + #[cfg(test)] + async fn wait_until_analysis_matches( + backend: &HtclBackend, + uri: &Url, + predicate: impl Fn(&DocAnalysis) -> bool, + ) { + for _ in 0..200 { + if let Some(a) = backend.analysis_for(uri).await { + if predicate(&a) { + return; + } + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + panic!("timed out waiting for matching analysis of {uri}"); + } + + #[tokio::test] + async fn edit_to_imported_file_ripples_to_open_importer() { + // Regression: user edits `clock.htcl` (adds a doc comment + // to `configure_clocks`) but hovers in `module.htcl` still + // show the pre-edit signature. Root cause was no fan-out + // reindex — module's analysis committed BEFORE the clock + // edit and had no mechanism to re-fire on upstream changes. + // + // Test shape: dir with `lib.htcl` (defines a proc) and + // `main.htcl` (`src lib`). Open both. Update lib on disk + // AND via `set_text` to add a doc comment. Assert main's + // analysis picks it up WITHOUT re-touching main. + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + "[workspace]\nname = \"t\"\n", + ) + .unwrap(); + let lib_path = dir.path().join("lib.htcl"); + let lib_v1 = "proc greet {} unit { puts hi }\n"; + std::fs::write(&lib_path, lib_v1).unwrap(); + let lib_uri = Url::from_file_path(&lib_path).unwrap(); + let main_path = dir.path().join("main.htcl"); + let main_src = "src lib\ngreet\n"; + std::fs::write(&main_path, main_src).unwrap(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + + let backend = HtclBackend::new(); + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.into()) + .await; + backend.set_text_sync(lib_uri.clone(), lib_v1.into()).await; + // Sanity: main's initial view contains lib_v1's proc but + // NO doc comment yet. + let a = backend.analysis_for(&main_uri).await.unwrap(); + assert!(a.view.view_source.contains("proc greet")); + assert!( + !a.view.view_source.contains("## greeting"), + "pre-edit view unexpectedly has doc comment: {}", + a.view.view_source, + ); + + // User edits lib.htcl: add a doc comment. Update disk + // (equivalent of `did_save`) AND the open buffer. + let lib_v2 = "## greeting proc\nproc greet {} unit { puts hi }\n"; + std::fs::write(&lib_path, lib_v2).unwrap(); + backend.set_text_sync(lib_uri.clone(), lib_v2.into()).await; + + // main.htcl is NOT touched here — the fan-out is what + // should ripple the change through. + wait_until_analysis_matches(&backend, &main_uri, |a| { + a.view.view_source.contains("## greeting") + }) + .await; + } + + #[tokio::test] + async fn workspace_diagnostics_preload_covers_unopened_entry_points() { + // The workspace has a `design.htcl` (entry point `vw check` + // would discover) with warnings. The editor has NOT opened + // it. `workspace_diagnostics` should STILL surface those + // warnings because `set_workspace_roots` preloads the same + // entry-point set. Without this, Helix's space-D picker + // would show nothing for warnings in files the user + // hasn't visited. + let dir = tempfile::tempdir().unwrap(); + // Minimal vw.toml to make this a valid workspace root + // (workspace-discovery walks up looking for it). + std::fs::write( + dir.path().join("vw.toml"), + "[workspace]\nname = \"t\"\n", + ) + .unwrap(); + // design.htcl carries a stub proc with a `@default(0)` + // arg; the redundant-default warning fires on the call + // site below. + let design_src = "\ +proc use_it { @default(0) count } unit { puts $count } +use_it -count 0 +"; + let design_path = dir.path().join("design.htcl"); + std::fs::write(&design_path, design_src).unwrap(); + let design_uri = Url::from_file_path(&design_path).unwrap(); + // Open a DIFFERENT file — `other.htcl` — that does NOT + // src design.htcl. Without preload, design.htcl wouldn't + // appear in the docs map at all. + let other_path = dir.path().join("other.htcl"); + std::fs::write(&other_path, "puts hi\n").unwrap(); + let other_uri = Url::from_file_path(&other_path).unwrap(); + let backend = HtclBackend::new(); + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(other_uri.clone(), "puts hi\n".into()) + .await; + // Wait for the preload's design.htcl indexer to commit. + // `wait_for_reindex` would hang if the preload already + // committed (it waits for the NEXT commit); the poll + // helper handles the already-committed case correctly. + wait_until_analysis_present(&backend, &design_uri).await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + let design_diags = ws.get(&design_uri).unwrap_or_else(|| { + panic!( + "design.htcl missing from workspace diags — preload didn't \ + register it. Full map: {ws:?}" + ) + }); + assert!( + design_diags.iter().any(|d| d.message.contains("redundant")), + "expected redundant-default warning from preloaded design.htcl, \ + got {design_diags:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_awaits_pending_preload_indexers() { + // The race the previous test doesn't cover: the user + // opens a file BEFORE preload indexers commit. If + // `workspace_diagnostics` doesn't wait for preloads, the + // fan-out that publishes for space-D reads a partial + // snapshot — preloaded URIs whose indexer is still + // running contribute NOTHING, so their diagnostics are + // silently dropped from the picker. + // + // We simulate the race by NOT awaiting the preload + // between `set_workspace_roots` and the first + // `workspace_diagnostics` call. Preload for `warn.htcl` + // hasn't committed at that instant; the assertion is + // that `workspace_diagnostics` still returns the warning + // (having awaited the commit internally). + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + "[workspace]\nname = \"t\"\n", + ) + .unwrap(); + let warn_src = "\ +proc use_it { @default(0) count } unit { puts $count } +use_it -count 0 +"; + let warn_path = dir.path().join("design.htcl"); + std::fs::write(&warn_path, warn_src).unwrap(); + let warn_uri = Url::from_file_path(&warn_path).unwrap(); + let backend = HtclBackend::new(); + // set_workspace_roots kicks off the preload but returns + // BEFORE any preload indexer commits. + backend + .set_workspace_roots(vec![dir.path().to_path_buf()]) + .await; + // Straight to workspace_diagnostics — no + // wait_until_analysis_present. This is the racey path. + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + let diags = ws.get(&warn_uri).unwrap_or_else(|| { + panic!("preloaded design.htcl missing from ws diags: {ws:?}") + }); + assert!( + diags.iter().any(|d| d.message.contains("redundant")), + "expected redundant-default warning, got {diags:?}", + ); + } + + #[tokio::test] + async fn workspace_diagnostics_skip_out_of_workspace_deps() { + // The workspace is `main_dir`; the imported `dep.htcl` + // lives OUTSIDE it. Errors in the dep should NOT show up + // in workspace diagnostics — that's just noise for a file + // the user isn't editing from this workspace. + let dep_dir = tempfile::tempdir().unwrap(); + let dep_path = dep_dir.path().join("dep.htcl"); + std::fs::write(&dep_path, "proc broken {} { return 42 }\n").unwrap(); + let main_dir = tempfile::tempdir().unwrap(); + let main_path = main_dir.path().join("main.htcl"); + // Use an absolute `src` pointing at the dep tempfile. + let dep_str = dep_path.to_string_lossy().into_owned(); + // Strip the .htcl since `src` re-adds it. + let dep_no_ext = dep_str.trim_end_matches(".htcl"); + let main_src = format!("src {dep_no_ext}\n"); + std::fs::write(&main_path, &main_src).unwrap(); + let backend = HtclBackend::new(); + let main_uri = Url::from_file_path(&main_path).unwrap(); + let dep_uri = Url::from_file_path(&dep_path).unwrap(); + backend + .set_workspace_roots(vec![main_dir.path().to_path_buf()]) + .await; + backend + .set_text_sync(main_uri.clone(), main_src.clone()) + .await; + let ws: std::collections::HashMap> = + backend.workspace_diagnostics().await.into_iter().collect(); + assert!( + !ws.contains_key(&dep_uri), + "dep diagnostics should be filtered out: {ws:?}", + ); + } + + #[tokio::test] + async fn save_skips_debounce_and_commits_immediately() { + // Simulates the "small edit + Ctrl-s" flow: the user made + // a tiny change (so set_text just fired a 250ms-debounced + // indexer that hasn't started yet), then saved. `save` + // must abort the pending debounced task and commit its + // OWN indexer without waiting. + // + // We verify by racing `save` against a bounded timeout: + // if `save` still went through the debounce, this would + // time out because the sleep would still be pending. + let backend = HtclBackend::new(); + let uri = Url::parse("file:///tmp/save-test.htcl").unwrap(); + // `set_text` puts a debounced indexer in flight — it + // won't commit for 250ms even though the analysis itself + // is fast for this trivial input. + backend + .set_text(uri.clone(), "proc f {} unit { puts hi }\n".into()) + .await; + // `save` bumps the generation and spawns a fresh + // zero-debounce indexer, which should commit essentially + // right away. If we're wrong and save honors the + // debounce, the `changed()` await would block ~250ms; + // set a 100ms timeout to catch that regression. + let saved = backend.save(&uri); + let waited = tokio::time::timeout( + std::time::Duration::from_millis(500), + async { + saved.await; + backend.wait_for_reindex(&uri).await; + }, + ) + .await; + assert!(waited.is_ok(), "save didn't commit within timeout"); + // And the committed analysis must actually exist. + let a = backend.analysis_for(&uri).await; + assert!(a.is_some(), "save didn't leave a committed analysis"); + } +} diff --git a/vw-analyzer/src/lib.rs b/vw-analyzer/src/lib.rs new file mode 100644 index 0000000..20848ec --- /dev/null +++ b/vw-analyzer/src/lib.rs @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw analyzer` — multi-language LSP for the vw HDL workflow. +//! +//! The server is built around a [`LanguageBackend`] abstraction even +//! while only [`HtclBackend`] is wired up. This keeps the architectural +//! slot for VHDL (initially a `vhdl_ls` proxy, later a direct Oxide +//! VHDL frontend integration) open from day one — see the project +//! plan's "LSP design" section. + +mod backend; +mod htcl_backend; +mod server; +mod src_complete; +mod vhdl_backend; +mod workspace; + +pub use backend::{LanguageBackend, SymbolInfo}; +pub use htcl_backend::HtclBackend; +pub use server::Analyzer; +pub use vhdl_backend::VhdlBackend; + +use tokio::io::{ + AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, + BufReader, +}; +use tower_lsp::{LspService, Server}; + +/// Run the LSP server on stdio. Returns when the editor disconnects. +/// +/// Both the standalone `vw-analyzer` binary and the `vw analyzer` +/// subcommand call this so the editor sees identical behavior either +/// way. +pub async fn run_stdio() { + let (service, socket) = LspService::new(Analyzer::new); + let stdout = tokio::io::stdout(); + + // Splice stdin through `forward_until_exit` instead of handing it + // to tower-lsp directly. tower-lsp 0.20's `serve` loop is a + // `join!` whose read side only completes on stdin EOF; its `exit` + // notification handling merely flips server state and closes the + // client sink — it does NOT stop reading stdin. Editors such as + // Helix send `exit` and keep the stdin pipe OPEN, expecting the + // server to self-terminate, so without help the process lingers + // forever after every `:lsp-restart`. And because Helix only + // clears a server's diagnostics once that server's stdout closes + // (its transport synthesizes an `exit` on `StreamClosed` to drive + // the cleanup), a never-exiting instance keeps its diagnostics + // live and each restart stacks another copy on top — the doubling. + // + // The forwarder relays every message verbatim, then drops its + // write half right after relaying `exit`. tower-lsp reads `exit`, + // then reads EOF, `serve` returns, and the process exits cleanly — + // closing stdout, which is the signal the editor needs. + let (server_stdin, feed) = tokio::io::duplex(1 << 16); + let pump = forward_until_exit(tokio::io::stdin(), feed); + let server = Server::new(server_stdin, stdout, socket).serve(service); + + // Race the server against the forwarder. The forwarder returns as + // soon as it relays `exit` (or real stdin hits EOF) — i.e. the + // moment the session is over — whereas `serve` itself does NOT + // reliably return on `exit`: tower-lsp 0.20 processes the `exit` + // notification but its `join!(read_input, …)` only unwinds on + // stdin EOF, and even the EOF-after-exit case fails to complete + // the join. So we treat the forwarder finishing as the + // authoritative end-of-session and terminate on it. + tokio::select! { + _ = server => {} + _ = pump => {} + } + + // Force the process down rather than falling off `main`: + // `tokio::io::stdin()` reads on a blocking thread parked in + // `read()` while the real stdin pipe stays open (Helix keeps it + // open across `:lsp-restart`), and the runtime's drop waits on + // that thread — a clean return would hang. Exiting closes our + // stdout, which is the signal the editor needs to clear this + // instance's diagnostics (its transport synthesizes the terminal + // `exit` on that EOF). A client that waits for the `shutdown` + // response already received it before sending `exit`, so nothing + // in flight is lost. + std::process::exit(0); +} + +/// Relay LSP `Content-Length`-framed messages from `src` to `dst` +/// byte-for-byte, returning once an `exit` notification has been +/// relayed (or `src` reaches EOF / a malformed frame is hit). The +/// caller drops `dst` when this returns, which is what surfaces EOF to +/// tower-lsp's serve loop and lets the process terminate on `exit` — +/// see `run_stdio` for why that's necessary. Framing-only: it parses +/// just enough of each frame (Content-Length header, then the JSON +/// `method`) to detect `exit`; everything is forwarded unchanged. +async fn forward_until_exit(src: R, mut dst: W) +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + let mut src = BufReader::new(src); + loop { + // Read the header block, capturing raw bytes + Content-Length. + let mut headers: Vec = Vec::new(); + let mut content_len: Option = None; + loop { + let mut line = Vec::new(); + match src.read_until(b'\n', &mut line).await { + Ok(0) => return, // stdin EOF + Ok(_) => {} + Err(_) => return, + } + headers.extend_from_slice(&line); + if let Some(colon) = line.iter().position(|&b| b == b':') { + let (name, rest) = line.split_at(colon); + if name.eq_ignore_ascii_case(b"content-length") { + content_len = std::str::from_utf8(&rest[1..]) + .ok() + .and_then(|s| s.trim().parse::().ok()); + } + } + // A blank line (just CRLF/LF) ends the header block. + let trimmed = line.strip_suffix(b"\n").unwrap_or(line.as_slice()); + let trimmed = trimmed.strip_suffix(b"\r").unwrap_or(trimmed); + if trimmed.is_empty() { + break; + } + } + let Some(len) = content_len else { return }; + + // Read the exact body. + let mut body = vec![0u8; len]; + if src.read_exact(&mut body).await.is_err() { + return; + } + + // Forward the frame verbatim. + if dst.write_all(&headers).await.is_err() + || dst.write_all(&body).await.is_err() + || dst.flush().await.is_err() + { + return; + } + + // Stop once `exit` has been forwarded so tower-lsp hits EOF. + let is_exit = serde_json::from_slice::(&body) + .ok() + .and_then(|v| v.get("method")?.as_str().map(|m| m == "exit")) + .unwrap_or(false); + if is_exit { + return; + } + } +} + +#[cfg(test)] +mod tests { + use super::forward_until_exit; + + fn frame(body: &str) -> Vec { + format!("Content-Length: {}\r\n\r\n{}", body.len(), body).into_bytes() + } + + #[tokio::test] + async fn forwards_frames_then_stops_after_exit() { + let mut input = Vec::new(); + input.extend(frame( + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}"#, + )); + input.extend(frame( + r#"{"jsonrpc":"2.0","method":"initialized","params":{}}"#, + )); + input.extend(frame(r#"{"jsonrpc":"2.0","id":2,"method":"shutdown"}"#)); + input.extend(frame(r#"{"jsonrpc":"2.0","method":"exit"}"#)); + // Anything after `exit` must NOT be forwarded. + input.extend(frame(r#"{"jsonrpc":"2.0","method":"never"}"#)); + + let mut out: Vec = Vec::new(); + forward_until_exit(&input[..], &mut out).await; + + let s = String::from_utf8(out).unwrap(); + assert!(s.contains(r#""method":"initialize""#)); + assert!(s.contains(r#""method":"shutdown""#)); + assert!(s.contains(r#""method":"exit""#)); + assert!( + !s.contains("never"), + "forwarding must stop right after `exit`" + ); + } + + #[tokio::test] + async fn relays_body_bytes_exactly() { + // A body containing the literal text `"method":"exit"` inside a + // string value must NOT trip early termination — only the real + // top-level method matters. + let tricky = r#"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"text":"\"method\":\"exit\""}}"#; + let mut input = frame(tricky); + input.extend(frame(r#"{"jsonrpc":"2.0","method":"exit"}"#)); + + let mut out: Vec = Vec::new(); + forward_until_exit(&input[..], &mut out).await; + + let s = String::from_utf8(out).unwrap(); + assert!(s.contains("didChange"), "real message must be forwarded"); + assert!(s.contains(r#""method":"exit""#)); + } +} diff --git a/vw-analyzer/src/main.rs b/vw-analyzer/src/main.rs new file mode 100644 index 0000000..c349930 --- /dev/null +++ b/vw-analyzer/src/main.rs @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw-analyzer` binary entry point. +//! +//! Spawns the LSP server on stdio. The editor (or `vw analyzer` +//! subcommand) exec's this binary directly. + +#[tokio::main] +async fn main() { + // Silent by default — Helix and most LSP clients flag any stderr + // output from a language server as an error. Opt in with + // `VW_ANALYZER_LOG=info` (or `debug`/`trace`) for development. + // ANSI off so colors don't show up as escape codes in the + // client's log viewer. + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_ansi(false) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_env("VW_ANALYZER_LOG") + .unwrap_or_else(|_| "vw_analyzer=off".into()), + ) + .init(); + + vw_analyzer::run_stdio().await; +} diff --git a/vw-analyzer/src/server.rs b/vw-analyzer/src/server.rs new file mode 100644 index 0000000..0176f5a --- /dev/null +++ b/vw-analyzer/src/server.rs @@ -0,0 +1,625 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! LSP server entry point. Owns the per-language backends and +//! dispatches `textDocument/*` requests by URI. + +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use tower_lsp::jsonrpc::Result; +use tower_lsp::lsp_types::notification::Progress; +use tower_lsp::lsp_types::request::WorkDoneProgressCreate; +use tower_lsp::lsp_types::*; +use tower_lsp::{Client, LanguageServer}; +use tracing::{debug, info}; + +use crate::backend::LanguageBackend; +use crate::htcl_backend::HtclBackend; + +pub struct Analyzer { + client: Client, + backends: Vec>, + /// Monotonic counter for `$/progress` tokens. Every user-facing + /// slow operation (diagnostics, goto-def, hover, completion) + /// generates a fresh token and reports begin/end so Helix and + /// other LSP clients render a pulsating "indexing" indicator + /// while the request is in flight. Wrapped in Arc so the + /// background diagnostic-publish task fired from did_change + /// can share the counter with the foreground handlers. + progress_seq: Arc, +} + +impl Analyzer { + pub fn new(client: Client) -> Self { + let backends: Vec> = vec![ + Arc::new(HtclBackend::new()), + Arc::new(crate::VhdlBackend::new(client.clone())), + ]; + Self { + client, + backends, + progress_seq: Arc::new(AtomicU64::new(0)), + } + } + + fn backend_for(&self, uri: &Url) -> Option> { + self.backends.iter().find(|b| b.handles(uri)).cloned() + } + + /// Fire a background diagnostics publish for `uri`. Returns + /// immediately; the spawned task awaits the current indexer + /// via the backend's `analysis_for` and publishes when it + /// completes. Used from `did_open`/`did_change` so the LSP's + /// notification queue isn't stalled by the ~1s indexer + /// wall-clock — rapid typing no longer serializes into a + /// queue of stale re-indexes. + /// + /// The progress token creation is fire-and-forget from a + /// separate task, wrapping only the `analysis_for` await — + /// the diagnostics publish is NOT gated on the client's + /// progress-create response. If we wrapped publish itself in + /// `with_progress`, a client that never responds to + /// `window/workDoneProgress/create` would hang the entire + /// pipeline. Progress is UX polish; diagnostics are the + /// contract, so diagnostics win. + fn spawn_publish_diagnostics(&self, uri: Url, version: Option) { + let Some(backend) = self.backend_for(&uri) else { + return; + }; + if backend.pushes_diagnostics() { + // Backend already published diagnostics inside + // `set_text` via its own outbound RPC. Skipping the + // pull path avoids racing with (and empty-clobbering) + // that side-channel publish. See + // `LanguageBackend::pushes_diagnostics` docs. + return; + } + let client = self.client.clone(); + let progress_seq = self.progress_seq.clone(); + let uri_task = uri.clone(); + // Detached task: wait for the next indexer commit while an + // LSP progress spinner is active, then publish the diagnostics + // from THAT fresh analysis. Wrapping the wait in + // `with_progress` is what makes Helix's pulsing "indexing…" + // indicator show up during the rebuild — the previous version + // wrapped an empty `async {}` future so Begin+End fired in + // the same millisecond, effectively no-op. + // + // Reads (completion, hover, goto-def) DO NOT go through this + // task — they use `backend.diagnostics` / `analysis_for` + // directly and are served instantly from the stale-cache. So + // typing latency stays great; only the diagnostics-refresh + + // progress spinner are gated on the actual rebuild. + tokio::spawn(async move { + with_progress( + &client, + &progress_seq, + "Indexing", + uri_task.as_ref(), + backend.wait_for_reindex(&uri_task), + ) + .await; + let diags = backend.diagnostics(&uri_task).await; + debug!( + uri = %uri_task, + count = diags.len(), + "publishing diagnostics" + ); + client + .publish_diagnostics(uri_task.clone(), diags, version) + .await; + // Fan out cross-file diagnostics for EVERY file the + // just-completed analysis touched. Helix's `space-D` + // workspace-diagnostic view reads its cache of pushed + // `publishDiagnostics` — the LSP 3.17 pull path we + // implement in `workspace_diagnostic` isn't wired in + // there yet, so without this fan-out the picker stays + // empty until the user actually opens each broken + // file. We publish empty diagnostics for files with + // no findings too, so a fixed error clears from the + // picker as soon as the change commits. + for (uri, diagnostics) in backend.workspace_diagnostics().await { + if uri == uri_task { + continue; + } + client.publish_diagnostics(uri, diagnostics, None).await; + } + }); + } +} + +/// Free-function `with_progress` that takes just the components +/// needed to negotiate a workDoneProgress token. Sharing the impl +/// this way lets the background diagnostic-publish task fire +/// progress notifications without cloning the whole Analyzer. +async fn with_progress( + client: &Client, + progress_seq: &AtomicU64, + title: &str, + message: &str, + fut: impl Future, +) -> T { + let seq = progress_seq.fetch_add(1, Ordering::Relaxed); + let token = NumberOrString::String(format!("vw-analyzer-{seq}")); + // Server-initiated progress: create the token first. If the + // client refuses, fall through to the future without + // reporting — the request still runs, just without the + // spinner. + let created = client + .send_request::(WorkDoneProgressCreateParams { + token: token.clone(), + }) + .await + .is_ok(); + if created { + let begin = ProgressParams { + token: token.clone(), + value: ProgressParamsValue::WorkDone(WorkDoneProgress::Begin( + WorkDoneProgressBegin { + title: title.to_string(), + cancellable: Some(false), + message: Some(message.to_string()), + percentage: None, + }, + )), + }; + client.send_notification::(begin).await; + } + let result = fut.await; + if created { + let end = ProgressParams { + token, + value: ProgressParamsValue::WorkDone(WorkDoneProgress::End( + WorkDoneProgressEnd { message: None }, + )), + }; + client.send_notification::(end).await; + } + result +} + +#[tower_lsp::async_trait] +impl LanguageServer for Analyzer { + async fn initialize( + &self, + params: InitializeParams, + ) -> Result { + info!("vw-analyzer initializing"); + // Capture the editor's workspace roots so backends can use + // them as fallback dep-lookup dirs when analyzing files + // opened outside the nearest `vw.toml`. Newer LSP clients + // send `workspaceFolders`; older ones use `rootUri` — we + // accept whichever is present. Missing → empty (each + // file's own workspace still resolves in isolation, which + // was the pre-fallback behavior). + let roots = collect_workspace_roots(¶ms); + for backend in &self.backends { + backend.set_workspace_roots(roots.clone()).await; + } + Ok(InitializeResult { + server_info: Some(ServerInfo { + name: "vw-analyzer".into(), + version: Some(env!("CARGO_PKG_VERSION").into()), + }), + capabilities: ServerCapabilities { + // FULL sync — the client sends the whole buffer + // on every change. Do NOT switch this to + // `TextDocumentSyncCapability::Options { ... }` + // to opt into `didSave`: Helix's LSP client + // stopped sending `didChange` altogether when we + // tried that (verified 2026-07 — no + // notifications reached the server after the + // switch, and diagnostics froze until reload). + // Keep this `Kind(FULL)` until we find a Helix- + // safe way to also receive save events (e.g. + // dynamic registration via + // `client/registerCapability`). + text_document_sync: Some(TextDocumentSyncCapability::Kind( + TextDocumentSyncKind::FULL, + )), + document_symbol_provider: Some(OneOf::Left(true)), + workspace_symbol_provider: Some(OneOf::Left(true)), + hover_provider: Some(HoverProviderCapability::Simple(true)), + definition_provider: Some(OneOf::Left(true)), + completion_provider: Some(CompletionOptions { + // `-` opens a flag list; a space after a flag pops + // its `@enum(…)` choices (or the next available + // flags when there are no enum constraints), so + // the user doesn't have to start typing blind to + // discover options. + trigger_characters: Some(vec![ + "-".to_string(), + " ".to_string(), + ]), + ..Default::default() + }), + signature_help_provider: Some(SignatureHelpOptions { + trigger_characters: Some(vec![ + " ".to_string(), + "-".to_string(), + ]), + retrigger_characters: Some(vec!["-".to_string()]), + work_done_progress_options: Default::default(), + }), + rename_provider: Some(OneOf::Left(true)), + references_provider: Some(OneOf::Left(true)), + // LSP 3.17 pull-based diagnostics — Helix uses this + // for `space-D`'s workspace-wide diagnostic picker. + // `workspace_diagnostics: true` also opts us into + // the `workspace/diagnostic` request; without it the + // editor only knows about diagnostics we've + // proactively pushed for open files. + diagnostic_provider: Some( + DiagnosticServerCapabilities::Options(DiagnosticOptions { + identifier: Some("vw-htcl".into()), + inter_file_dependencies: true, + workspace_diagnostics: true, + work_done_progress_options: Default::default(), + }), + ), + ..Default::default() + }, + }) + } + + async fn initialized(&self, _: InitializedParams) { + info!("vw-analyzer initialized"); + // Dynamically register `workspace/didChangeWatchedFiles` + // for the paths any backend cares about. We don't declare + // static capability at `initialize` time because dynamic + // registration lets each backend pick its own patterns — + // today the VHDL backend needs `vw.toml`, `vw.lock`, and + // `ip/**/*.htcl` to reflect `vw update` and IP-config + // edits back into the wrapped `vhdl_ls::VHDLServer`. + // + // Registration failures (client that doesn't advertise + // dynamic registration, or refuses this specific one) are + // logged and swallowed — the LSP still runs, just without + // the reactive config-reload path. + let watchers = vec![ + FileSystemWatcher { + glob_pattern: GlobPattern::String("**/vw.toml".into()), + kind: None, + }, + FileSystemWatcher { + glob_pattern: GlobPattern::String("**/vw.lock".into()), + kind: None, + }, + FileSystemWatcher { + glob_pattern: GlobPattern::String("**/ip/**/*.htcl".into()), + kind: None, + }, + ]; + let registration = Registration { + id: "vw-analyzer-watched-files".into(), + method: "workspace/didChangeWatchedFiles".into(), + register_options: serde_json::to_value( + DidChangeWatchedFilesRegistrationOptions { watchers }, + ) + .ok(), + }; + if let Err(e) = + self.client.register_capability(vec![registration]).await + { + info!( + "vw-analyzer: dynamic watched-files registration \ + failed ({e}); config reactivity disabled" + ); + } + } + + async fn did_change_watched_files( + &self, + params: DidChangeWatchedFilesParams, + ) { + for backend in &self.backends { + backend.did_change_watched_files(¶ms).await; + } + } + + async fn did_change_workspace_folders( + &self, + params: DidChangeWorkspaceFoldersParams, + ) { + // Rebuild the roots list from scratch on any change. We + // don't retain state between initialize and here, so an + // added-only event still tells us the FULL updated set — + // both `added` and `removed` are already applied by the + // client before this notification per LSP spec. + let added: Vec = params + .event + .added + .iter() + .filter_map(|f| f.uri.to_file_path().ok()) + .collect(); + for backend in &self.backends { + backend.set_workspace_roots(added.clone()).await; + } + } + + async fn shutdown(&self) -> Result<()> { + info!("vw-analyzer shutting down"); + Ok(()) + } + + async fn did_open(&self, params: DidOpenTextDocumentParams) { + let uri = params.text_document.uri.clone(); + let version = Some(params.text_document.version); + debug!(%uri, "did_open"); + if let Some(backend) = self.backend_for(&uri) { + backend + .set_text(uri.clone(), params.text_document.text) + .await; + } + // Fire the diagnostic publish as a background task so the + // ~1s indexer wall-clock doesn't stall tower-lsp's + // notification queue. Rapid typing (each keystroke fires + // did_change → set_text → publish) previously serialized + // into a queue of stale re-indexes; now every did_change + // returns in microseconds and the LATEST index's + // diagnostics arrive whenever it wins the abort race. + self.spawn_publish_diagnostics(uri, version); + } + + async fn did_change(&self, params: DidChangeTextDocumentParams) { + let uri = params.text_document.uri.clone(); + let version = Some(params.text_document.version); + let Some(backend) = self.backend_for(&uri) else { + return; + }; + // FULL sync: each change is the entire new text. + if let Some(change) = params.content_changes.into_iter().last() { + backend.set_text(uri.clone(), change.text).await; + } + // Background publish (see `did_open`). + self.spawn_publish_diagnostics(uri, version); + } + + async fn did_close(&self, params: DidCloseTextDocumentParams) { + let uri = params.text_document.uri; + if let Some(backend) = self.backend_for(&uri) { + backend.close(&uri).await; + } + } + + async fn did_save(&self, params: DidSaveTextDocumentParams) { + let uri = params.text_document.uri.clone(); + debug!(%uri, "did_save"); + if let Some(backend) = self.backend_for(&uri) { + // Zero-debounce reindex — the whole point of + // handling save is that `Ctrl-s` should force a + // fresh check now, not 250ms from now. + backend.save(&uri).await; + } + // Same publish path as did_change so the fresh index's + // diagnostics land in the editor. + self.spawn_publish_diagnostics(uri, None); + } + + async fn document_symbol( + &self, + params: DocumentSymbolParams, + ) -> Result> { + let uri = params.text_document.uri; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let symbols = backend.document_symbols(&uri).await; + if symbols.is_empty() { + Ok(None) + } else { + Ok(Some(DocumentSymbolResponse::Nested(symbols))) + } + } + + async fn symbol( + &self, + params: WorkspaceSymbolParams, + ) -> Result>> { + let query = params.query; + // Walk every registered backend (today just one) and merge the + // matches — keeps the same dispatch shape as `backend_for` so + // adding a second language later doesn't need a refactor. + let mut symbols = Vec::new(); + for backend in &self.backends { + symbols.extend(backend.workspace_symbols(&query).await); + } + if symbols.is_empty() { + Ok(None) + } else { + Ok(Some(symbols)) + } + } + + async fn hover(&self, params: HoverParams) -> Result> { + // Reads from the cached DocAnalysis — no per-request + // progress wrapping needed since the answer lands in + // microseconds after indexing has completed. + let uri = params.text_document_position_params.text_document.uri; + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.hover(&uri, position).await) + } + + async fn goto_definition( + &self, + params: GotoDefinitionParams, + ) -> Result> { + // Cached — see the note on `hover`. + let uri = params + .text_document_position_params + .text_document + .uri + .clone(); + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let locs = backend.goto_definition(&uri, position).await; + if locs.is_empty() { + Ok(None) + } else { + Ok(Some(GotoDefinitionResponse::Array(locs))) + } + } + + async fn completion( + &self, + params: CompletionParams, + ) -> Result> { + // Cached — see the note on `hover`. + let uri = params.text_document_position.text_document.uri; + let position = params.text_document_position.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let items = backend.completion(&uri, position).await; + if items.is_empty() { + Ok(None) + } else { + Ok(Some(CompletionResponse::Array(items))) + } + } + + async fn signature_help( + &self, + params: SignatureHelpParams, + ) -> Result> { + let uri = params + .text_document_position_params + .text_document + .uri + .clone(); + let position = params.text_document_position_params.position; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.signature_help(&uri, position).await) + } + + async fn rename( + &self, + params: RenameParams, + ) -> Result> { + let uri = params.text_document_position.text_document.uri.clone(); + let position = params.text_document_position.position; + let new_name = params.new_name; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + Ok(backend.rename(&uri, position, &new_name).await) + } + + async fn references( + &self, + params: ReferenceParams, + ) -> Result>> { + let uri = params.text_document_position.text_document.uri.clone(); + let position = params.text_document_position.position; + let include_declaration = params.context.include_declaration; + let Some(backend) = self.backend_for(&uri) else { + return Ok(None); + }; + let locs = backend + .references(&uri, position, include_declaration) + .await; + if locs.is_empty() { + Ok(None) + } else { + Ok(Some(locs)) + } + } + + async fn diagnostic( + &self, + params: DocumentDiagnosticParams, + ) -> Result { + // Pull-based single-file diagnostics. Same payload the + // push path serves via `publishDiagnostics` — the editor + // may request it explicitly (Helix does when the buffer + // opens, before any push has fired) as a + // no-guess-when-they-arrive alternative. + let uri = params.text_document.uri.clone(); + let items = match self.backend_for(&uri) { + Some(backend) => backend.diagnostics(&uri).await, + None => Vec::new(), + }; + Ok(DocumentDiagnosticReportResult::Report( + DocumentDiagnosticReport::Full( + RelatedFullDocumentDiagnosticReport { + related_documents: None, + full_document_diagnostic_report: + FullDocumentDiagnosticReport { + result_id: None, + items, + }, + }, + ), + )) + } + + async fn workspace_diagnostic( + &self, + _params: WorkspaceDiagnosticParams, + ) -> Result { + // Collect from every backend. Each returns a set of + // (uri, diagnostics) tuples pulled from its open docs' + // workspace-view analyses — files transitively `src`d by + // an open document surface their errors here even if the + // user hasn't opened them, which is what makes Helix's + // `space-D` picker useful for whole-workspace triage. + let mut items = Vec::new(); + for backend in &self.backends { + for (uri, diagnostics) in backend.workspace_diagnostics().await { + items.push(WorkspaceDocumentDiagnosticReport::Full( + WorkspaceFullDocumentDiagnosticReport { + uri, + version: None, + full_document_diagnostic_report: + FullDocumentDiagnosticReport { + result_id: None, + items: diagnostics, + }, + }, + )); + } + } + Ok(WorkspaceDiagnosticReportResult::Report( + WorkspaceDiagnosticReport { items }, + )) + } +} + +/// Extract workspace roots from an `initialize` request as +/// filesystem paths. Prefers `workspaceFolders` (LSP 3.6+, sent +/// by every modern client) and falls back to `rootUri` for older +/// clients. Both may be absent — e.g. when the editor opens a +/// file with no folder context — in which case we return an +/// empty vec and each file's own `vw.toml` is the only source +/// of dep names, matching the pre-fallback behavior. +fn collect_workspace_roots( + params: &InitializeParams, +) -> Vec { + if let Some(folders) = params.workspace_folders.as_ref() { + if !folders.is_empty() { + return folders + .iter() + .filter_map(|f| f.uri.to_file_path().ok()) + .collect(); + } + } + // `root_uri` is deprecated but still what a lot of clients + // (including bare-bones LSP integrations) send. + #[allow(deprecated)] + if let Some(uri) = params.root_uri.as_ref() { + if let Ok(p) = uri.to_file_path() { + return vec![p]; + } + } + Vec::new() +} diff --git a/vw-analyzer/src/src_complete.rs b/vw-analyzer/src/src_complete.rs new file mode 100644 index 0000000..09059dc --- /dev/null +++ b/vw-analyzer/src/src_complete.rs @@ -0,0 +1,367 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Filesystem-aware completion for `src` import paths. +//! +//! The `vw-htcl` crate stays free of filesystem concerns, so this +//! lives in the analyzer alongside the workspace resolver. When the +//! cursor sits in the path-position of a `src` command, we +//! enumerate the directory implied by the partial path and offer: +//! +//! - every `.htcl` file at that level, labelled by basename (no +//! extension), and +//! - every subdirectory at that level that transitively contains at +//! least one `.htcl` file, labelled with a trailing `/`. +//! +//! Three flavors of partial are recognized, matching +//! [`vw_htcl::src_path::classify`]: +//! +//! - `@/...` — resolve against the workspace dependency's cached +//! root. +//! - `/abs/...` — filesystem-absolute. +//! - anything else — relative to the importing file's directory. +//! +//! When the partial is just `@` or `@` (no `/` yet), suggest +//! dependency names from the workspace resolver instead. + +use std::path::{Path, PathBuf}; + +use tower_lsp::lsp_types::{ + CompletionItem, CompletionItemKind, CompletionTextEdit, InsertTextFormat, + Position, Range, TextEdit, +}; +use vw_htcl::cmdline::CmdLine; +use vw_htcl::src_path::{classify, PathKind}; +use vw_htcl::{LineCol, LineIndex, Resolver, Span}; + +/// True when the cursor sits in the path-position of a `src` command +/// (i.e. the first complete word is `src` and we're typing the path). +pub fn is_src_path_context(line: &CmdLine<'_>) -> bool { + line.words.first().copied() == Some("src") && line.words.len() == 1 +} + +/// Generate path completions for `line.partial`, treating it as the +/// `` of `src `. +/// +/// `entry_file` is the open file's path on disk; it anchors relative +/// imports and lets us walk up to the workspace's `vw.toml` for dep +/// resolution. `line_index` maps source offsets to LSP positions. +pub fn src_path_completions( + entry_file: &Path, + line: &CmdLine<'_>, + line_index: &LineIndex, + resolver: &Resolver, +) -> Vec { + let partial = line.partial; + + // `@` with no `/` yet → dep-name completion. Replace the + // whole partial with `@/` so the next completion fires on + // the contents. + if let Some(prefix_after_at) = partial.strip_prefix('@') { + if !prefix_after_at.contains('/') { + return dep_name_completions( + resolver, + prefix_after_at, + line.partial_span, + line_index, + ); + } + } + + // Otherwise: resolve the directory the partial points into, then + // enumerate it. + let Some((dir, segment_start)) = resolve_dir(entry_file, resolver, partial) + else { + return Vec::new(); + }; + let segment = &partial[segment_start..]; + let replace = Span::new( + line.partial_span.start + segment_start as u32, + line.partial_span.end, + ); + enumerate_entries(&dir, segment, replace, line_index) +} + +/// Resolve the *directory* part of `partial` to an on-disk path, plus +/// the byte offset into `partial` where the trailing (still-being- +/// typed) segment begins. Returns `None` when the partial points at a +/// dep that doesn't exist or a path that can't be classified. +fn resolve_dir( + entry_file: &Path, + resolver: &Resolver, + partial: &str, +) -> Option<(PathBuf, usize)> { + let kind = classify(partial).kind; + let (base, body) = match &kind { + PathKind::Relative => { + let dir = entry_file.parent()?.to_path_buf(); + (dir, partial) + } + PathKind::Absolute => { + (PathBuf::from("/"), partial.trim_start_matches('/')) + } + PathKind::Named { name, subpath } => { + let root = resolver.dep_root(name)?.to_path_buf(); + (root, subpath.as_str()) + } + }; + // Split `body` at its last `/`: everything before is the + // sub-directory walk; everything after is the segment being typed + // (used for the replace range and ignored for enumeration). + let (subdir, trailing_segment) = match body.rfind('/') { + Some(i) => (&body[..i], &body[i + 1..]), + None => ("", body), + }; + let mut dir = base; + if !subdir.is_empty() { + dir.push(subdir); + } + let segment_start = partial.len() - trailing_segment.len(); + Some((dir, segment_start)) +} + +fn enumerate_entries( + dir: &Path, + segment: &str, + replace: Span, + line_index: &LineIndex, +) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let _ = segment; // LSP client filters by prefix; we list everything. + + let mut out: Vec<(String, CompletionItemKind)> = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name.starts_with('.') { + continue; + } + let ft = entry.file_type().ok(); + if ft.is_some_and(|t| t.is_dir()) { + if dir_has_htcl(&path) { + out.push((format!("{name}/"), CompletionItemKind::FOLDER)); + } + } else if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + let stem = + path.file_stem().and_then(|s| s.to_str()).unwrap_or(name); + // `module.htcl` is the dep's default entry point, already + // reachable as bare `@` — listing it here as + // `@/module` would just be a noisier alias. + if stem == vw_htcl::src_path::DEFAULT_MODULE { + continue; + } + out.push((stem.to_string(), CompletionItemKind::FILE)); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + let range = lsp_range(replace, line_index); + out.into_iter() + .map(|(label, kind)| build_item(label, kind, range)) + .collect() +} + +fn dep_name_completions( + resolver: &Resolver, + _prefix: &str, + partial_span: Span, + line_index: &LineIndex, +) -> Vec { + let mut deps: Vec<(&str, &Path)> = resolver.deps().collect(); + deps.sort_by_key(|(n, _)| *n); + let range = lsp_range(partial_span, line_index); + // Bare `@` is a complete import on its own (resolves to the + // dep's `module.htcl`), so don't append a trailing `/` — that + // would leave behind invalid syntax for a user who just wanted + // the default module. Users who want to drill in still type `/` + // themselves, which retriggers completion against the dep root. + deps.into_iter() + .map(|(name, _)| { + build_item(format!("@{name}"), CompletionItemKind::MODULE, range) + }) + .collect() +} + +/// True if `dir` contains, or transitively contains, any `.htcl` file. +/// Short-circuits on the first hit. +fn dir_has_htcl(dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(ft) = entry.file_type() else { continue }; + if ft.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + return true; + } + } else if ft.is_dir() { + // Skip dot-dirs to keep `.git`, `.svn`, etc. out of the + // walk. + if path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with('.')) + { + continue; + } + if dir_has_htcl(&path) { + return true; + } + } + } + false +} + +fn build_item( + label: String, + kind: CompletionItemKind, + range: Range, +) -> CompletionItem { + let new_text = label.clone(); + CompletionItem { + label, + kind: Some(kind), + insert_text_format: Some(InsertTextFormat::PLAIN_TEXT), + text_edit: Some(CompletionTextEdit::Edit(TextEdit { range, new_text })), + ..Default::default() + } +} + +fn lsp_range(span: Span, line_index: &LineIndex) -> Range { + let (start, end) = line_index.range(span); + Range { + start: lc_to_pos(start), + end: lc_to_pos(end), + } +} + +fn lc_to_pos(lc: LineCol) -> Position { + Position { + line: lc.line, + character: lc.character, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use vw_htcl::cmdline; + + fn workspace_fixture() -> (tempfile::TempDir, PathBuf, Resolver) { + // amd-htcl/ + // module.htcl ← default entry, HIDDEN from list + // cmd.htcl + // ip.htcl + // cmd/foo.htcl + // scripts/ ← no .htcl, should NOT appear + // ip/bd/cell.htcl ← nested, ip/ should appear + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("amd-htcl"); + fs::create_dir_all(dep.join("cmd")).unwrap(); + fs::create_dir_all(dep.join("scripts")).unwrap(); + fs::create_dir_all(dep.join("ip/bd")).unwrap(); + fs::write(dep.join("module.htcl"), "# entry").unwrap(); + fs::write(dep.join("cmd.htcl"), "# stub").unwrap(); + fs::write(dep.join("ip.htcl"), "# stub").unwrap(); + fs::write(dep.join("cmd/foo.htcl"), "# stub").unwrap(); + fs::write(dep.join("scripts/notes.txt"), "not htcl").unwrap(); + fs::write(dep.join("ip/bd/cell.htcl"), "# stub").unwrap(); + // entry file + let entry = dir.path().join("prime.htcl"); + fs::write(&entry, "src @amd-htcl/cmd\n").unwrap(); + let resolver = Resolver::new().with_dep("amd-htcl", dep); + // hold dir handle so files persist for the test + (dir, entry, resolver) + } + + fn labels_for(src: &str, entry: &Path, resolver: &Resolver) -> Vec { + let line = cmdline::analyze(src, src.len() as u32); + let idx = LineIndex::new(src); + let items = src_path_completions(entry, &line, &idx, resolver); + let mut labels: Vec = + items.into_iter().map(|c| c.label).collect(); + labels.sort(); + labels + } + + #[test] + fn lists_dep_root_after_trailing_slash() { + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/", &entry, &resolver); + // .htcl files: cmd, ip. dirs with .htcl: cmd/, ip/. + // scripts/ is omitted (no .htcl inside). + assert_eq!(labels, vec!["cmd", "cmd/", "ip", "ip/"]); + } + + #[test] + fn lists_dep_subdirectory() { + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/ip/", &entry, &resolver); + // ip/ has bd/ (containing cell.htcl) — bd/ should show; no other entries. + assert_eq!(labels, vec!["bd/"]); + } + + #[test] + fn partial_segment_replaces_only_the_segment() { + // User has typed `src @amd-htcl/c` — replace should cover just + // the `c`, not the whole `@amd-htcl/c`. + let src = "src @amd-htcl/c"; + let (_dir, entry, resolver) = workspace_fixture(); + let line = cmdline::analyze(src, src.len() as u32); + let idx = LineIndex::new(src); + let items = src_path_completions(&entry, &line, &idx, &resolver); + let labels: Vec = + items.iter().map(|c| c.label.clone()).collect(); + // Both `cmd` and `cmd/` start with `c`. + assert!(labels.contains(&"cmd".to_string()), "{labels:?}"); + // The text-edit range should cover only the `c` (single char on line 0). + let edit = match items[0].text_edit.as_ref() { + Some(CompletionTextEdit::Edit(e)) => e, + _ => panic!("expected text edit"), + }; + assert_eq!(edit.range.start.character, 14, "{:?}", edit.range); + assert_eq!(edit.range.end.character, 15); + } + + #[test] + fn dep_name_completion_when_no_slash_yet() { + // Bare `@` is a complete import on its own, so the + // completion shouldn't append `/` — selecting `@amd-htcl` + // alone should leave valid syntax that resolves to + // `/module.htcl`. + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @", &entry, &resolver); + assert_eq!(labels, vec!["@amd-htcl"]); + } + + #[test] + fn dep_root_listing_hides_module_htcl() { + // `module.htcl` is the default entry — already importable as + // bare `@amd-htcl`, so it should not show up as `module` in + // the per-dep file listing. + let (_dir, entry, resolver) = workspace_fixture(); + let labels = labels_for("src @amd-htcl/", &entry, &resolver); + assert!(!labels.contains(&"module".to_string()), "{labels:?}"); + // Sanity: the non-default modules still show. + assert!(labels.contains(&"cmd".to_string())); + assert!(labels.contains(&"ip".to_string())); + } + + #[test] + fn relative_completion_uses_entry_directory() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("ip")).unwrap(); + fs::write(dir.path().join("ip/cips.htcl"), "# stub").unwrap(); + let entry = dir.path().join("prime.htcl"); + fs::write(&entry, "src ip/\n").unwrap(); + let resolver = Resolver::new(); + let labels = labels_for("src ip/", &entry, &resolver); + assert_eq!(labels, vec!["cips"]); + } +} diff --git a/vw-analyzer/src/vhdl_backend.rs b/vw-analyzer/src/vhdl_backend.rs new file mode 100644 index 0000000..91d02fa --- /dev/null +++ b/vw-analyzer/src/vhdl_backend.rs @@ -0,0 +1,591 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! VHDL [`LanguageBackend`] backed by an embedded +//! [`vhdl_ls::VHDLServer`] per workspace. +//! +//! `vhdl_ls::SharedRpcChannel` wraps `Rc` and is +//! therefore `!Send`, so the server can't live in an +//! `Arc` shared across tokio worker threads. +//! We pin each workspace's server to a dedicated `std::thread` and +//! communicate over a synchronous mpsc: the LSP-facing async methods +//! send a typed request + oneshot reply, the worker thread receives +//! it, calls the corresponding `VHDLServer::text_document_*` method, +//! and returns the response. +//! +//! Outbound notifications (`publishDiagnostics`, `logMessage`, etc.) +//! emitted by the wrapped server hit a `TowerLspRpc` bridge that +//! forwards them to the `tower_lsp::Client` on a captured runtime +//! handle. Unknown notification methods are logged and dropped — +//! `vhdl_ls` only emits a small set today (diagnostics + log/show +//! messages) and adding a new one is a rare event. + +use async_trait::async_trait; +use camino::Utf8PathBuf; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::{Arc, Mutex as StdMutex}; +use tokio::sync::{oneshot, Mutex as TokioMutex}; +use tower_lsp::lsp_types::{ + ClientCapabilities, CompletionItem, Diagnostic, + DidChangeTextDocumentParams, DidChangeWatchedFilesParams, + DidCloseTextDocumentParams, DidOpenTextDocumentParams, DocumentSymbol, + DocumentSymbolParams, DocumentSymbolResponse, GotoDefinitionParams, + GotoDefinitionResponse, Hover, HoverParams, InitializeParams, Location, + LogMessageParams, MessageType, PartialResultParams, Position, + PublishDiagnosticsParams, ShowMessageParams, SignatureHelp, + SymbolInformation, TextDocumentIdentifier, TextDocumentItem, + TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier, + WorkDoneProgressParams, WorkspaceEdit, +}; +use tower_lsp::Client; +use tracing::warn; + +use crate::backend::LanguageBackend; + +/// Per-workspace worker owning a `VHDLServer`. Communication happens +/// over a synchronous mpsc; async LSP handlers await responses via +/// `tokio::sync::oneshot`. +struct WorkspaceHandle { + tx: std::sync::mpsc::Sender, + /// Held only for cleanup at drop; joining is best-effort. + _thread: StdMutex>>, +} + +impl Drop for WorkspaceHandle { + fn drop(&mut self) { + let _ = self.tx.send(Message::Shutdown); + if let Some(handle) = self._thread.lock().unwrap().take() { + let _ = handle.join(); + } + } +} + +#[allow(dead_code)] +enum Message { + DidOpen(DidOpenTextDocumentParams), + /// Populated once we wire delta-mode text sync; today the + /// full-buffer replace path routes through `DidOpen` because + /// the wrapped server treats duplicates as no-ops. + DidChange(DidChangeTextDocumentParams), + DidClose(DidCloseTextDocumentParams), + Hover(HoverParams, oneshot::Sender>), + GotoDefinition( + GotoDefinitionParams, + oneshot::Sender>, + ), + DocumentSymbols( + DocumentSymbolParams, + oneshot::Sender>, + ), + /// Pull-based diagnostics for a single file. Since `vhdl_ls` + /// pushes diagnostics asynchronously through + /// `publishDiagnostics`, this returns whatever the cache holds + /// for `uri` at the moment of the request. Callers wanting + /// fresher results should nudge `did_change` first. + Diagnostics(Url, oneshot::Sender>), + /// Ask the worker to swap the wrapped server's config in + /// place. Triggered by `did_change_watched_files` after the + /// backend re-renders from live workspace state. + UpdateConfig(vhdl_lang::Config), + Shutdown, +} + +pub struct VhdlBackend { + client: Client, + workspaces: TokioMutex>>, + /// `Some(handle)` in production; captured at construction so + /// worker threads can hand outbound notifications back to + /// tower_lsp via `handle.spawn`. Tests skip this by leaving it + /// `None` and reading messages out of the RpcMock instead. + runtime: tokio::runtime::Handle, +} + +impl VhdlBackend { + pub fn new(client: Client) -> Self { + Self { + client, + workspaces: TokioMutex::new(HashMap::new()), + runtime: tokio::runtime::Handle::current(), + } + } + + /// Resolve `uri` to a workspace root and (create + bootstrap) + /// the per-workspace worker if it isn't running yet. Returns + /// `None` when the URI doesn't sit under a `vw.toml` — the + /// caller should skip the request rather than fabricate an + /// answer. + async fn ensure_workspace( + &self, + uri: &Url, + ) -> Option> { + let path = uri.to_file_path().ok()?; + let ws = crate::workspace::find_workspace_dir(&path)?; + let mut map = self.workspaces.lock().await; + if let Some(existing) = map.get(&ws) { + return Some(existing.clone()); + } + let cfg = match vw_lib::render_vhdl_lang_config(&ws, None) { + Ok(c) => c, + Err(e) => { + warn!("vhdl_backend: config render failed for {ws}: {e}"); + return None; + } + }; + // Make sure the VHDL standard library is available — fetched + // into the dep cache on first use — and point vhdl_ls at it, so + // a machine without a system rust_hdl install still resolves + // `ieee` / `std`. `None` (fetch failed, offline + uncached) + // falls back to vhdl_ls's built-in search of installed + // locations. + let stdlib = vw_lib::ensure_vhdl_stdlib() + .await + .ok() + .map(|p| p.to_string()); + let handle = spawn_workspace_worker( + ws.clone(), + self.client.clone(), + self.runtime.clone(), + cfg, + stdlib, + ); + map.insert(ws, handle.clone()); + Some(handle) + } +} + +fn spawn_workspace_worker( + root: Utf8PathBuf, + client: Client, + runtime: tokio::runtime::Handle, + initial_config: vhdl_lang::Config, + stdlib_libraries_path: Option, +) -> Arc { + let (tx, rx) = std::sync::mpsc::channel::(); + let thread = std::thread::spawn(move || { + workspace_thread( + root, + client, + runtime, + initial_config, + stdlib_libraries_path, + rx, + ); + }); + Arc::new(WorkspaceHandle { + tx, + _thread: StdMutex::new(Some(thread)), + }) +} + +fn workspace_thread( + root: Utf8PathBuf, + client: Client, + runtime: tokio::runtime::Handle, + initial_config: vhdl_lang::Config, + stdlib_libraries_path: Option, + rx: std::sync::mpsc::Receiver, +) { + let bridge = TowerLspRpc { client, runtime }; + let rpc = vhdl_ls::SharedRpcChannel::new(Rc::new(bridge)); + let mut server = vhdl_ls::VHDLServer::new_with_config( + rpc, + vhdl_ls::VHDLServerSettings { + non_project_file_handling: vhdl_ls::NonProjectFileHandling::Analyze, + // Point vhdl_ls at the stdlib vw fetched into the dep cache + // (`None` → its built-in search of installed locations). + libraries_path: stdlib_libraries_path, + ..Default::default() + }, + initial_config, + ); + // Synthesize the LSP `initialize` handshake — root_uri anchors + // vhdl_ls's workspace-scoped file resolution. Client + // capabilities left at default: minimal is enough since we're + // not proxying advanced features (semantic tokens, semantic + // highlighting) through the outer analyzer today. + #[allow(deprecated)] + let init_params = InitializeParams { + process_id: None, + root_path: None, + root_uri: Url::from_directory_path(root.as_std_path()).ok(), + initialization_options: None, + capabilities: ClientCapabilities::default(), + trace: None, + workspace_folders: None, + client_info: None, + locale: None, + }; + server.initialize_request(init_params); + server.initialized_notification(); + + while let Ok(msg) = rx.recv() { + match msg { + Message::DidOpen(p) => { + server.text_document_did_open_notification(&p); + } + Message::DidChange(p) => { + server.text_document_did_change_notification(&p); + } + Message::DidClose(_) => { + // `vhdl_ls::VHDLServer` doesn't expose a + // `did_close` — the file stays part of the + // project until an `update_config` drops it. + // That matches vhdl_ls's stdio behavior. + } + Message::Hover(p, reply) => { + let r = server + .text_document_hover(&p.text_document_position_params); + let _ = reply.send(r); + } + Message::GotoDefinition(p, reply) => { + let r = server + .text_document_definition(&p.text_document_position_params) + .map(GotoDefinitionResponse::Scalar); + let _ = reply.send(r); + } + Message::DocumentSymbols(p, reply) => { + let r = server.document_symbol(&p); + let _ = reply.send(r); + } + Message::UpdateConfig(cfg) => { + server.set_config(cfg); + } + Message::Diagnostics(uri, reply) => { + // The wrapped server has already emitted + // `publishDiagnostics` for this file — the pull- + // based `textDocument/diagnostic` handler in + // vhdl_ls delegates to the same analyze pipeline. + // For now we return `[]` and rely on the push + // path; a follow-up phase can wire the pull-based + // `text_document_diagnostic` if any editor we + // support prefers pull. + let _ = (uri, reply.send(Vec::new())); + } + Message::Shutdown => break, + } + } +} + +/// `vhdl_ls::RpcChannel` implementation that forwards notifications +/// and requests to the outer tower_lsp `Client`. +/// +/// Only the notifications `vhdl_ls` actually emits today are wired +/// through — new ones surface as warnings so we notice. +struct TowerLspRpc { + client: Client, + runtime: tokio::runtime::Handle, +} + +impl vhdl_ls::RpcChannel for TowerLspRpc { + fn send_notification(&self, method: String, params: serde_json::Value) { + let client = self.client.clone(); + self.runtime.spawn(async move { + match method.as_str() { + "textDocument/publishDiagnostics" => { + match serde_json::from_value::( + params, + ) { + Ok(p) => { + client + .publish_diagnostics( + p.uri, + p.diagnostics, + p.version, + ) + .await + } + Err(e) => { + warn!("vhdl_backend: bad publishDiagnostics: {e}") + } + } + } + "window/logMessage" => { + match serde_json::from_value::(params) { + Ok(p) => client.log_message(p.typ, p.message).await, + Err(e) => { + warn!("vhdl_backend: bad logMessage: {e}") + } + } + } + "window/showMessage" => { + match serde_json::from_value::(params) { + Ok(p) => client.show_message(p.typ, p.message).await, + Err(e) => { + warn!("vhdl_backend: bad showMessage: {e}") + } + } + } + other => warn!( + "vhdl_backend: dropping unhandled notification {other}" + ), + } + }); + } + + fn send_request(&self, method: String, _params: serde_json::Value) { + // `vhdl_ls` sends server→client requests rarely (e.g. + // `client/registerCapability`). tower_lsp doesn't expose a + // generic method+Value send, so drop with a warning until + // one of these actually becomes load-bearing. + warn!( + "vhdl_backend: dropping server→client request {method} — \ + not yet bridged" + ); + } +} + +fn text_document_position( + uri: Url, + position: Position, +) -> TextDocumentPositionParams { + TextDocumentPositionParams { + text_document: TextDocumentIdentifier { uri }, + position, + } +} + +#[async_trait] +impl LanguageBackend for VhdlBackend { + fn language_id(&self) -> &str { + "vhdl" + } + + fn handles(&self, uri: &Url) -> bool { + let path = uri.path(); + path.ends_with(".vhd") || path.ends_with(".vhdl") + } + + fn pushes_diagnostics(&self) -> bool { + // vhdl_ls calls `publish_diagnostics` synchronously in + // `text_document_did_{open,change}_notification`; the + // outbound `TowerLspRpc` bridges those through the + // tower_lsp `Client`. The outer server's pull path would + // race and (with our current empty `diagnostics()` stub) + // clobber those real diagnostics with an empty vec — so + // opt out of it entirely. + true + } + + async fn set_text(&self, uri: Url, text: String) { + let Some(ws) = self.ensure_workspace(&uri).await else { + return; + }; + // vhdl_ls doesn't distinguish "first open" from + // "subsequent change" at the API level — sending + // `did_open` every set_text call is safe (the server + // ignores duplicates) and simpler than tracking per-URI + // state ourselves. The alternative — sending `did_change` + // with a full-buffer replace — would produce an + // equivalent effect but leaves us on the hook for + // synthesizing valid version numbers. + let params = DidOpenTextDocumentParams { + text_document: TextDocumentItem { + uri, + language_id: "vhdl".into(), + version: 0, + text, + }, + }; + let _ = ws.tx.send(Message::DidOpen(params)); + } + + async fn did_change_watched_files( + &self, + params: &DidChangeWatchedFilesParams, + ) { + // Which workspaces are affected? Walk each event's URI up + // to its nearest `vw.toml` and dedup — one config re-render + // per workspace, not per file. Deleting a `vw.toml` shows + // up as `Deleted` events whose containing dir *may* still + // be a workspace (parent `vw.toml`) or *may* not — either + // way, re-render is safe: it either produces a valid + // config for the surviving workspace or a `render_...` + // error we log and ignore. + let mut affected: std::collections::HashSet = + std::collections::HashSet::new(); + for change in ¶ms.changes { + let Ok(path) = change.uri.to_file_path() else { + continue; + }; + if let Some(ws) = crate::workspace::find_workspace_dir(&path) { + affected.insert(ws); + } + } + if affected.is_empty() { + return; + } + let map = self.workspaces.lock().await; + for ws in affected { + let Some(handle) = map.get(&ws) else { + // Workspace has no live server (no `.vhd` opened + // yet). Nothing to reload — next `.vhd` open will + // build a fresh server from the current state. + continue; + }; + match vw_lib::render_vhdl_lang_config(&ws, None) { + Ok(cfg) => { + let _ = handle.tx.send(Message::UpdateConfig(cfg)); + } + Err(e) => { + warn!( + "vhdl_backend: config re-render failed for \ + {ws}: {e}" + ); + } + } + } + } + + async fn close(&self, uri: &Url) { + let Some(ws) = self.ensure_workspace(uri).await else { + return; + }; + let params = DidCloseTextDocumentParams { + text_document: TextDocumentIdentifier { uri: uri.clone() }, + }; + let _ = ws.tx.send(Message::DidClose(params)); + } + + async fn diagnostics(&self, uri: &Url) -> Vec { + let Some(ws) = self.ensure_workspace(uri).await else { + return Vec::new(); + }; + let (tx, rx) = oneshot::channel(); + if ws.tx.send(Message::Diagnostics(uri.clone(), tx)).is_err() { + return Vec::new(); + } + rx.await.unwrap_or_default() + } + + async fn document_symbols(&self, uri: &Url) -> Vec { + let Some(ws) = self.ensure_workspace(uri).await else { + return Vec::new(); + }; + let params = DocumentSymbolParams { + text_document: TextDocumentIdentifier { uri: uri.clone() }, + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + }; + let (tx, rx) = oneshot::channel(); + if ws.tx.send(Message::DocumentSymbols(params, tx)).is_err() { + return Vec::new(); + } + match rx.await.unwrap_or(None) { + Some(DocumentSymbolResponse::Nested(v)) => v, + // vhdl_ls always returns Nested; the Flat branch is + // only reached if the wrapper starts negotiating with + // the client. Convert defensively. + Some(DocumentSymbolResponse::Flat(_)) => Vec::new(), + None => Vec::new(), + } + } + + async fn workspace_symbols(&self, _query: &str) -> Vec { + // Not yet wired — `VHDLServer::workspace_symbol` needs the + // query threaded through; add in a follow-up. + Vec::new() + } + + async fn hover(&self, uri: &Url, position: Position) -> Option { + let ws = self.ensure_workspace(uri).await?; + let params = HoverParams { + text_document_position_params: text_document_position( + uri.clone(), + position, + ), + work_done_progress_params: WorkDoneProgressParams::default(), + }; + let (tx, rx) = oneshot::channel(); + if ws.tx.send(Message::Hover(params, tx)).is_err() { + return None; + } + rx.await.ok().flatten() + } + + async fn goto_definition( + &self, + uri: &Url, + position: Position, + ) -> Vec { + let Some(ws) = self.ensure_workspace(uri).await else { + return Vec::new(); + }; + let params = GotoDefinitionParams { + text_document_position_params: text_document_position( + uri.clone(), + position, + ), + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + }; + let (tx, rx) = oneshot::channel(); + if ws.tx.send(Message::GotoDefinition(params, tx)).is_err() { + return Vec::new(); + } + match rx.await.unwrap_or(None) { + Some(GotoDefinitionResponse::Scalar(loc)) => vec![loc], + Some(GotoDefinitionResponse::Array(v)) => v, + Some(GotoDefinitionResponse::Link(_)) | None => Vec::new(), + } + } + + async fn completion( + &self, + _uri: &Url, + _position: Position, + ) -> Vec { + // Wire in Phase 3b — `VHDLServer::request_completion` takes + // `CompletionParams` and we need to bridge the returned + // `CompletionList`. + Vec::new() + } + + async fn signature_help( + &self, + _uri: &Url, + _position: Position, + ) -> Option { + // vhdl_ls's signature help is limited; wire in a follow-up. + None + } + + async fn rename( + &self, + _uri: &Url, + _position: Position, + _new_name: &str, + ) -> Option { + None + } + + async fn references( + &self, + _uri: &Url, + _position: Position, + _include_declaration: bool, + ) -> Vec { + Vec::new() + } +} + +/// Suppress a false-positive dead-code warning: `VersionedTextDocumentIdentifier` +/// is currently only reachable if we ever fill out did_change params +/// with version numbers; keep the import to signal the surface. +#[allow(dead_code)] +fn _placeholder(v: VersionedTextDocumentIdentifier) { + let _ = v; +} + +/// Marker type: outbound `window/showMessage`s coming in as +/// `MessageType::WARNING` etc. are what we surface at the top of +/// `TowerLspRpc::send_notification`. Kept here so the `use` stays +/// referenced across builds. +#[allow(dead_code)] +const _MSG_TYPES: &[MessageType] = &[ + MessageType::ERROR, + MessageType::WARNING, + MessageType::INFO, + MessageType::LOG, +]; diff --git a/vw-analyzer/src/workspace.rs b/vw-analyzer/src/workspace.rs new file mode 100644 index 0000000..5e02c92 --- /dev/null +++ b/vw-analyzer/src/workspace.rs @@ -0,0 +1,374 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Workspace-aware helpers for the analyzer. +//! +//! The bare LSP backend deals with one file at a time. Cross-file +//! features — goto-definition into an imported module, completion of +//! procs defined in `@dep/foo`, validating a call against a signature +//! that lives elsewhere — need a view that spans the importing file +//! plus everything it pulled in via `src`. +//! +//! This module computes that view on demand. It's deliberately +//! re-computed per query rather than cached: htcl files are tiny next +//! to a Vivado IP wrapper, the LSP's edits-per-second is modest, and a +//! cache would have to deal with invalidation when an `@dep/...` +//! file on disk changes. A targeted cache is a sensible follow-up once +//! the access pattern is settled. + +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use camino::{Utf8Path, Utf8PathBuf}; +use tower_lsp::lsp_types::Url; + +use vw_htcl::{parse, CommandKind, Resolver, SrcImport, Stmt}; + +/// A flattened source view used for cross-file analysis. +/// +/// `view_source` is the local file's text *first* (so the cursor's +/// byte offset in the open document is the same offset in the view — +/// hover/goto/etc. don't need offset translation for the local file), +/// followed by every transitively imported file's text concatenated. +/// Each appended region is recorded in [`imports`](Self::imports) so +/// spans landing there can be mapped back to the file they came from. +pub struct WorkspaceView { + pub view_source: String, + /// Byte length of the *local* file's contribution. Spans whose + /// `start < local_len` belong to the open file; everything past + /// that lives in some imported file. + pub local_len: u32, + pub imports: Vec, + /// Names of every dep the file's resolver knows about + /// (workspace `vw.toml` + editor extra_roots + sibling + /// scan). Passed to the validator's undefined-src-module + /// check so `src @` where `` isn't in the set + /// gets a spanned Error diagnostic. Empty when no workspace + /// context resolved. + pub dep_names: std::collections::HashSet, +} + +pub struct ImportRegion { + /// Inclusive start offset in `view_source`. + pub start: u32, + /// Exclusive end offset in `view_source`. + pub end: u32, + pub file_uri: Url, +} + +impl WorkspaceView { + /// If `offset` lies inside an imported file's region, return the + /// import region plus the file-local offset of that span; `None` + /// means the offset is in the open file itself. + pub fn locate(&self, offset: u32) -> Option<(&ImportRegion, u32)> { + if offset < self.local_len { + return None; + } + self.imports + .iter() + .find(|r| offset >= r.start && offset < r.end) + .map(|r| (r, offset - r.start)) + } +} + +/// Build a workspace view by reading every file the entry transitively +/// `src`s. Returns a view with `imports` empty when the entry can't be +/// resolved to a filesystem path or has no imports — the analyzer can +/// still use it; it just won't see anything cross-file. +/// +/// `extra_roots` supplies fallback workspace roots (typically the +/// editor's `rootUri` / `workspaceFolders`) so files opened outside +/// the enclosing `vw.toml` — e.g. via goto-def into a dep cache +/// dir — still resolve `@name/…` imports through the outer +/// workspace's dep graph. Pass `&[]` when the caller doesn't have +/// that context. +pub fn build_view( + file_uri: &Url, + local_text: &str, + extra_roots: &[PathBuf], +) -> WorkspaceView { + let mut view = WorkspaceView { + view_source: local_text.to_string(), + local_len: local_text.len() as u32, + imports: Vec::new(), + dep_names: std::collections::HashSet::new(), + }; + + let Ok(file_path) = file_uri.to_file_path() else { + return view; + }; + let parent = file_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let resolver = build_resolver_with(&file_path, extra_roots); + // Snapshot dep names now — the resolver may get moved into + // collect_imports below; the diagnostics pass needs the + // set as a plain HashSet. + view.dep_names = + resolver.deps().map(|(name, _)| name.to_string()).collect(); + + let mut loaded: HashSet = HashSet::new(); + if let Ok(canonical) = file_path.canonicalize() { + loaded.insert(canonical); + } + let mut queue: Vec<(PathBuf, String)> = Vec::new(); + collect_imports(local_text, &parent, &resolver, &mut loaded, &mut queue); + + while let Some((path, text)) = queue.pop() { + view.view_source.push('\n'); + // Record `start` *after* the separator so a span's local + // offset within the imported file is `span.start - start` + // with no off-by-one for the inserted newline. + let start = view.view_source.len() as u32; + view.view_source.push_str(&text); + let end = view.view_source.len() as u32; + if let Ok(import_uri) = Url::from_file_path(&path) { + view.imports.push(ImportRegion { + start, + end, + file_uri: import_uri, + }); + } + // Recurse into this file's own imports. + let import_parent = path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + collect_imports( + &text, + &import_parent, + &resolver, + &mut loaded, + &mut queue, + ); + } + + view +} + +/// Build a [`Resolver`] for the workspace that owns `entry_file`. +/// Convenience wrapper — see [`build_resolver_with`] for the full +/// variant that also honors editor-supplied fallback workspace +/// roots (LSP `rootUri` / `workspaceFolders`). +pub fn build_resolver(entry_file: &Path) -> Resolver { + build_resolver_with(entry_file, &[]) +} + +/// Build a [`Resolver`] for `entry_file`, merging dep declarations +/// from every source that could plausibly resolve a `@name/…` +/// import when the file's own workspace doesn't declare it: +/// +/// 1. The file's own workspace — walk up to the nearest `vw.toml` +/// and expand its dep graph via +/// [`vw_lib::transitive_dep_cache_paths`]. Highest precedence. +/// 2. Each path in `extra_roots` — treated as a workspace directory +/// and expanded the same way. Used to plumb the LSP's `rootUri` +/// (or `workspaceFolders`) so a file opened via goto-def *out* +/// of the editor's root workspace still inherits its dep names. +/// 3. Sibling-workspace layout scan — for every ancestor directory +/// of `entry_file`, treat each direct subdirectory that itself +/// contains a `vw.toml` as an implicit dep whose name is the +/// subdirectory basename. This lets a +/// `~/src/htcl/amd/cpm5/module.htcl` still see `@vivado-cmd` +/// at `~/src/htcl/amd/vivado-cmd/` even when neither its own +/// workspace nor the editor's root declares the dep — a monorepo +/// layout that's typical of a `foo-htcl` collection of siblings. +/// +/// First-seen wins on name collisions in the order above, so the +/// file's own workspace's choice never gets overridden. +/// True when `entry_file` sits inside the workspace's `test/` +/// directory subtree. Used to decide whether the analyzer's +/// resolver includes `[test-dependencies]` for LSP goto-def / +/// hover / diagnostics inside test files. +fn is_test_file(entry_file: &Path, workspace_dir: &Utf8Path) -> bool { + let ws_std = workspace_dir.as_std_path(); + let Ok(rel) = entry_file.strip_prefix(ws_std) else { + return false; + }; + rel.components() + .next() + .is_some_and(|c| std::path::Component::Normal("test".as_ref()) == c) +} + +pub fn build_resolver_with( + entry_file: &Path, + extra_roots: &[PathBuf], +) -> Resolver { + let mut merged: std::collections::HashMap = + std::collections::HashMap::new(); + if let Some(workspace_dir) = find_workspace_dir(entry_file) { + // A file under `/test/**` is a test file — pull in + // `[test-dependencies]` too so `src @` resolves + // in the analyzer just like it does in `vw test`. Matches + // the CLI's `check_htcl_with_mode(_, include_test)` + // behavior. + let include_test = is_test_file(entry_file, &workspace_dir); + // Transitive: a library that does `src @other-lib/...` + // shouldn't force every consumer to redeclare `other-lib` + // in their own `vw.toml`. The walker pulls in each dep's + // own deps so the resolver sees the whole graph + // (Cargo-style first-seen-wins on name conflicts). + if let Ok(paths) = vw_lib::transitive_dep_cache_paths_with_test( + &workspace_dir, + include_test, + ) { + for (name, path) in paths { + merged.entry(name).or_insert(path); + } + } + // Cargo-parity self-reference: a workspace named `foo` + // resolves `src @foo/bar` to `/bar.htcl`. Uses + // `entry(...).or_insert(...)` so a legitimately-declared + // external `foo` (rare but possible) still wins. + if let Ok(cfg) = vw_lib::load_workspace_config(&workspace_dir) { + merged + .entry(cfg.workspace.name) + .or_insert_with(|| workspace_dir.as_std_path().to_path_buf()); + } + } + for root in extra_roots { + let Ok(root_utf8) = Utf8PathBuf::from_path_buf(root.clone()) else { + continue; + }; + if !root_utf8.join("vw.toml").exists() { + continue; + } + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(&root_utf8) { + for (name, path) in paths { + merged.entry(name).or_insert(path); + } + } + } + collect_sibling_workspaces(entry_file, &mut merged); + let mut resolver = Resolver::new(); + for (name, path) in merged { + resolver = resolver.with_dep(name, path); + } + resolver +} + +/// Walk up from `entry_file`, and at each ancestor directory add +/// every subdirectory that contains its own `vw.toml` as an +/// implicit dep — keyed by the subdirectory basename. +/// +/// This mirrors a monorepo layout that's common for htcl workspaces: +/// `~/src/htcl/amd/{cips,cpm5,clk-wizard,vivado-cmd}/`. From any +/// one of those, the others are visible as siblings even when +/// no `vw.toml` explicitly declares them. Without this heuristic, +/// jumping into a dep-module file from an editor whose LSP has +/// restarted rooted at that dep's own `vw.toml` (helix's +/// `roots = ["vw.toml"]` behavior) would strand the analyzer with +/// no way to resolve `@sibling-name/…` imports. +/// +/// Stops at the filesystem root or after a handful of ancestors — +/// scanning `~` or `/` for candidate workspaces would be both slow +/// and semantically wrong. +fn collect_sibling_workspaces( + entry_file: &Path, + merged: &mut std::collections::HashMap, +) { + // Cap the walk so we don't scan every level up to `/`. Six + // ancestors covers the typical `~/src////` + // + a few extra tolerance for deeper nestings. + const MAX_ANCESTORS: usize = 6; + let mut cursor = match entry_file.parent() { + Some(p) => p.to_path_buf(), + None => return, + }; + for _ in 0..MAX_ANCESTORS { + let read_dir = match std::fs::read_dir(&cursor) { + Ok(r) => r, + Err(_) => break, + }; + for entry in read_dir.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if !ft.is_dir() { + continue; + } + let sub = entry.path(); + if !sub.join("vw.toml").is_file() { + continue; + } + let Some(name) = sub.file_name().and_then(|n| n.to_str()) else { + continue; + }; + merged.entry(name.to_string()).or_insert(sub); + } + cursor = match cursor.parent() { + Some(p) => p.to_path_buf(), + None => break, + }; + } +} + +/// Walk up from `start`'s parent directory looking for a `vw.toml`. +pub(crate) fn find_workspace_dir(start: &Path) -> Option { + let mut cur = start.parent()?.to_path_buf(); + loop { + if cur.join("vw.toml").exists() { + return Utf8PathBuf::from_path_buf(cur).ok(); + } + cur = cur.parent()?.to_path_buf(); + } +} + +/// Parse `text` and queue each new (not yet seen) `src` resolution as +/// `(canonical_path, file_text)` for the caller to incorporate. +fn collect_imports( + text: &str, + parent_dir: &Path, + resolver: &Resolver, + loaded: &mut HashSet, + queue: &mut Vec<(PathBuf, String)>, +) { + let parsed = parse(text); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(SrcImport { + path: Some(raw), .. + }) = &cmd.kind + else { + continue; + }; + let Ok(resolved) = resolver.resolve(parent_dir, raw) else { + continue; + }; + // Resolver already canonicalizes when possible; defensive + // dedup either way. + if !loaded.insert(resolved.clone()) { + continue; + } + let Ok(content) = fs::read_to_string(&resolved) else { + continue; + }; + queue.push((resolved, content)); + } +} + +/// Public helper: resolve the import at `raw` from `entry_file`'s +/// directory. Used by goto-on-import-path so the analyzer can return +/// a Location pointing at the imported file. +/// +/// `extra_roots` — see [`build_view`] for the same rationale — lets +/// callers plumb through the editor's workspace roots so a +/// `src @dep/file.htcl` in a file outside the enclosing workspace +/// still resolves. +pub fn resolve_import( + entry_file: &Path, + raw: &str, + extra_roots: &[PathBuf], +) -> Option { + let parent = entry_file.parent()?; + build_resolver_with(entry_file, extra_roots) + .resolve(parent, raw) + .ok() +} + +/// Allow `&Utf8Path` callers to canonicalize through us. +#[allow(dead_code)] +pub fn workspace_root(entry_file: &Utf8Path) -> Option { + find_workspace_dir(Path::new(entry_file.as_str())) +} diff --git a/vw-api-client/Cargo.toml b/vw-api-client/Cargo.toml new file mode 100644 index 0000000..98da264 --- /dev/null +++ b/vw-api-client/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "vw-api-client" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Progenitor generated clients for the VW service APIs" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[dependencies] +vw-api-types-versions = { path = "../vw-api-types/versions" } +base64 = "0.22" +progenitor = "0.14.0" +progenitor-client = "0.14.0" +rand = "0.8" +reqwest = { version = "0.13", features = ["json", "stream"] } +serde.workspace = true +uuid.workspace = true +thiserror.workspace = true diff --git a/vw-api-client/src/lib.rs b/vw-api-client/src/lib.rs new file mode 100644 index 0000000..4a3c37f --- /dev/null +++ b/vw-api-client/src/lib.rs @@ -0,0 +1,170 @@ +//! Progenitor generated clients for the vw service APIs. +//! +//! The user and admin APIs are served by two separate dropshot servers on two +//! separate ports, so each gets its own client in its own module. +//! +//! The OpenAPI documents these are generated from live in `openapi/` at the +//! root of the repository and belong to `vw-openapi-manager`: run +//! `cargo xtask openapi generate` after changing an endpoint. Each is read through +//! the `-latest.json` symlink the manager maintains, so this crate does not +//! have to be edited when the API version changes. `cargo xtask openapi check` +//! (also run as a test) fails if a document is out of date, which keeps these +//! clients from drifting from the service. +//! +//! Both APIs identify the caller by a Github access token in the authorization +//! header of every request. Rather than make each call site remember that, +//! [`user_client`] and [`admin_client`] build clients with the header already +//! attached. The token is optional: a service run with `--no-auth` answers +//! without one. + +use reqwest::header::{ + HeaderMap, HeaderValue, InvalidHeaderValue, AUTHORIZATION, +}; + +/// Client for the vw user API. +pub mod user { + // Sync types are reused rather than regenerated, for the same reason as in + // `agent` below: a client and a relay passing structurally identical but + // incompatible spellings of the same manifest would need a conversion at + // every hop. + progenitor::generate_api!( + spec = "../openapi/vw-user-api/vw-user-api-latest.json", + replace = { + Artifact = vw_api_types_versions::latest::Artifact, + CleanResult = vw_api_types_versions::latest::CleanResult, + CommitResult = vw_api_types_versions::latest::CommitResult, + Digest = vw_api_types_versions::latest::Digest, + FileEntry = vw_api_types_versions::latest::FileEntry, + SyncPlan = vw_api_types_versions::latest::SyncPlan, + TargetKind = vw_api_types_versions::latest::TargetKind, + TreeManifest = vw_api_types_versions::latest::TreeManifest, + }, + ); +} + +/// Client for the agent that runs on a build instance. +/// +/// Used by `vw-svc` to relay source, not by anything on a developer's machine +/// — the agents are only reachable from inside the rack. +pub mod agent { + // The shared types are reused rather than regenerated. Progenitor would + // otherwise mint its own `TreeManifest` and `Digest`, structurally + // identical to the ones in `vw-api-types` and incompatible with them, and + // every relayed request would have to be copied field by field between two + // spellings of the same thing. + progenitor::generate_api!( + spec = "../openapi/vw-sync-api/vw-sync-api-latest.json", + replace = { + CleanResult = vw_api_types_versions::latest::CleanResult, + CommitResult = vw_api_types_versions::latest::CommitResult, + Credentials = vw_api_types_versions::latest::Credentials, + S3Credentials = vw_api_types_versions::latest::S3Credentials, + Digest = vw_api_types_versions::latest::Digest, + FileEntry = vw_api_types_versions::latest::FileEntry, + SyncPlan = vw_api_types_versions::latest::SyncPlan, + TargetKind = vw_api_types_versions::latest::TargetKind, + TreeManifest = vw_api_types_versions::latest::TreeManifest, + }, + ); +} + +/// Client for the vw admin API. +pub mod admin { + progenitor::generate_api!( + "../openapi/vw-admin-api/vw-admin-api-latest.json" + ); +} + +/// Error conditions for constructing a client. +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("the access token cannot be sent in an http header: {0}")] + InvalidToken(#[from] InvalidHeaderValue), + #[error("building the http client failed: {0}")] + HttpClient(#[from] reqwest::Error), +} + +/// How to reach a vw service. +pub struct ClientConfig<'a> { + /// Base URL of the service, e.g. `https://vw.example.com:2727`. + pub base_url: &'a str, + + /// The Github access token to identify the caller with, if there is one. + /// + /// Optional because a service run with `--no-auth` answers without one. + /// Against a service that does require authorization, calls made without a + /// token come back `401 Unauthorized`. + pub token: Option<&'a str>, + + /// Accept whatever TLS certificate the service presents, without verifying + /// it against a trust anchor or checking that it names the host. + /// + /// This exists for services fronted by a self-signed certificate, which is + /// the usual case for a development deployment. It removes the guarantee + /// that you are talking to the service you think you are, and the access + /// token is sent to whatever answers, so do not use it against anything + /// you care about. + pub insecure: bool, +} + +/// A user API client for the service described by `config`. +pub fn user_client(config: &ClientConfig<'_>) -> Result { + Ok(user::Client::new_with_client( + config.base_url, + http_client(config)?, + )) +} + +/// An admin API client for the service described by `config`. +/// +/// The token's Github username must have been passed to the service in its +/// `--admin-users` argument for these endpoints to answer. +pub fn admin_client(config: &ClientConfig<'_>) -> Result { + Ok(admin::Client::new_with_client( + config.base_url, + http_client(config)?, + )) +} + +/// A client for the agent at `base_url`. +/// +/// No credentials: the agents sit on the rack's internal network behind +/// `vw-svc`, which has already decided whether the caller owns the environment +/// by the time anything reaches here. +pub fn agent_client(base_url: &str) -> Result { + Ok(agent::Client::new_with_client( + base_url, + http_client(&ClientConfig { + base_url, + token: None, + insecure: false, + })?, + )) +} + +/// An http client that presents the configured token, if there is one, on +/// every request. +fn http_client(config: &ClientConfig<'_>) -> Result { + let mut headers = HeaderMap::new(); + if let Some(token) = config.token { + let mut authorization = + HeaderValue::from_str(&format!("Bearer {token}"))?; + // Keep the token out of anything that debug-formats the request + // headers. + authorization.set_sensitive(true); + headers.insert(AUTHORIZATION, authorization); + } + + Ok(reqwest::Client::builder() + .default_headers(headers) + .danger_accept_invalid_certs(config.insecure) + // A vivado session is a websocket, and a websocket is an HTTP/1.1 + // upgrade — a mechanism HTTP/2 does not have. Over TLS, ALPN otherwise + // negotiates HTTP/2 and the upgrade is refused before it starts, which + // is a confusing way to find out. Nothing here benefits from HTTP/2: + // the requests are small and already sent concurrently over separate + // connections. The oxide SDK forces the same thing for the same + // reason. + .http1_only() + .build()?) +} diff --git a/vw-api-types/Cargo.toml b/vw-api-types/Cargo.toml new file mode 100644 index 0000000..c6b5718 --- /dev/null +++ b/vw-api-types/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vw-api-types" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "VW service API" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[dependencies] +vw-api-types-versions = { path = "./versions" } +schemars.workspace = true +serde.workspace = true diff --git a/vw-api-types/src/lib.rs b/vw-api-types/src/lib.rs new file mode 100644 index 0000000..4933f7c --- /dev/null +++ b/vw-api-types/src/lib.rs @@ -0,0 +1 @@ +pub use vw_api_types_versions::latest::*; diff --git a/vw-api-types/versions/Cargo.toml b/vw-api-types/versions/Cargo.toml new file mode 100644 index 0000000..2af334d --- /dev/null +++ b/vw-api-types/versions/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vw-api-types-versions" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "VW service API" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[dependencies] +schemars.workspace = true +serde.workspace = true +uuid.workspace = true +oxide.workspace = true +clap.workspace = true +iddqd.workspace = true diff --git a/vw-api-types/versions/src/latest.rs b/vw-api-types/versions/src/latest.rs new file mode 100644 index 0000000..79c7887 --- /dev/null +++ b/vw-api-types/versions/src/latest.rs @@ -0,0 +1,28 @@ +pub use crate::v1::Artifact; +pub use crate::v1::ArtifactPathParam; +pub use crate::v1::ArtifactsCleared; +pub use crate::v1::BenchQuery; +pub use crate::v1::CleanResult; +pub use crate::v1::CommitResult; +pub use crate::v1::Credentials; +pub use crate::v1::Digest; +pub use crate::v1::DriverBuildQuery; +pub use crate::v1::Environment; +pub use crate::v1::EnvironmentCreate; +pub use crate::v1::EnvironmentImages; +pub use crate::v1::EnvironmentPathParam; +pub use crate::v1::FileEntry; +pub use crate::v1::GeneratedFileQuery; +pub use crate::v1::ImageRef; +pub use crate::v1::ObjectStoreQuery; +pub use crate::v1::OxideInstance; +pub use crate::v1::S3Credentials; +pub use crate::v1::SshKeyPair; +pub use crate::v1::SyncPlan; +pub use crate::v1::TargetBlobPathParam; +pub use crate::v1::TargetKind; +pub use crate::v1::TargetPathParam; +pub use crate::v1::TreeManifest; +pub use crate::v1::UserEnvironment; +pub use crate::v1::UserEnvironmentPathParam; +pub use crate::v1::VivadoSessionQuery; diff --git a/vw-api-types/versions/src/lib.rs b/vw-api-types/versions/src/lib.rs new file mode 100644 index 0000000..99bd79d --- /dev/null +++ b/vw-api-types/versions/src/lib.rs @@ -0,0 +1,2 @@ +pub mod latest; +pub mod v1; diff --git a/vw-api-types/versions/src/v1.rs b/vw-api-types/versions/src/v1.rs new file mode 100644 index 0000000..62fbb79 --- /dev/null +++ b/vw-api-types/versions/src/v1.rs @@ -0,0 +1,522 @@ +use iddqd::{bi_upcast, BiHashItem}; +use oxide::types::InstanceState; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// An environment is a collection of instances that work together +/// to build, analyze and test vw designs. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct Environment { + /// The name of this environment. + pub name: String, + /// The images this environment's instances boot from, chosen when the + /// environment was created. + /// + /// Absent when the service has no Oxide backend configured, in which case + /// the environment is a bare record that will never be provisioned. + pub images: Option, + /// The Oxide instance id for the vivado instance. + pub vivado_instance: Option, + /// The Oxide instance id for the helios instance. + pub helios_instance: Option, + /// The Oxide instance id for the artifact instance. + pub artifact_instance: Option, +} + +/// The images each of an environment's instances boots from. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentImages { + /// Image the vivado instance boots from. + pub vivado: ImageRef, + /// Image the helios instance boots from. + pub helios: ImageRef, + /// Image the artifact instance boots from. + pub artifact: ImageRef, +} + +/// An Oxide image an environment's instances are built from. +/// +/// Pinned by id, so publishing a newer image does not silently change what an +/// existing environment boots. The name is carried along for display. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct ImageRef { + pub id: Uuid, + pub name: String, +} + +/// Information about an Oxide instance that underpins a VW instance. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct OxideInstance { + /// The Oxide instance id, once the control plane has assigned one. + /// + /// Absent in the window between asking for an instance and hearing back + /// about it, which is long enough to be worth showing: an environment + /// mid-creation reports `creating` with no id rather than looking like + /// nothing has happened. + pub id: Option, + pub state: InstanceState, + /// The address to reach this instance on from outside the rack, once it + /// has one. + /// + /// Absent until the instance exists and the control plane has attached an + /// external address to it. This is what a developer's ssh goes to. + pub external_ip: Option, + /// The instance's address on the rack's own network. + /// + /// What `vw-svc` sends source to, rather than the external address: the + /// internal path is a regional fabric rather than the public internet, and + /// the difference is most of the bandwidth. + pub internal_ip: Option, +} + +/// Which half of an environment a request is about. +/// +/// Only the two that take source. The artifact instance holds build output and +/// is reached as an object store, so there is nothing to synchronize to it. +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum TargetKind { + Vivado, + Helios, +} + +impl std::fmt::Display for TargetKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Vivado => f.write_str("vivado"), + Self::Helios => f.write_str("helios"), + } + } +} + +/// Which environment and half a synchronization request is for. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct TargetPathParam { + /// The name of the environment. + pub name: String, + /// Which half of it. + pub kind: TargetKind, +} + +/// Which piece of content is being delivered, and where. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct TargetBlobPathParam { + pub name: String, + pub kind: TargetKind, + /// The digest of the content in the body, verified on arrival. + pub digest: Digest, +} + +/// Body of a request to create an environment. +/// +/// Every field is optional; an image left unset is resolved to the newest +/// image the service can see whose name matches that instance kind's +/// convention. An image named here must already exist, or the request is +/// rejected. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentCreate { + /// Name of the image the vivado instance should boot from. + #[serde(default)] + pub vivado_image: Option, + /// Name of the image the helios instance should boot from. + #[serde(default)] + pub helios_image: Option, + /// Name of the image the artifact instance should boot from. + #[serde(default)] + pub artifact_image: Option, +} + +/// The ssh keypair that opens an environment's instances. +/// +/// Generated by the service when the environment is created and handed out +/// only to the environment's owner. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct SshKeyPair { + /// The private key, in OpenSSH format, ready to pass to `ssh -i`. + pub private_key: String, + /// The matching public key, as it appears in an `authorized_keys` file. + pub public_key: String, +} + +/// Path parameters used to identify an environment in the user API. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentPathParam { + /// The name of this environment to create. + pub name: String, +} + +/// An environment together with the owner's username. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct UserEnvironment { + /// User the environment belongs to + pub user: String, + /// Environment info + pub environment: Environment, +} + +/// Path parameters used to identify an environment in the admin API. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct UserEnvironmentPathParam { + /// User the environment belongs to + pub user: String, + /// The name of this environment to create. + pub name: String, +} + +impl BiHashItem for UserEnvironment { + type K1<'a> = &'a str; + type K2<'a> = &'a str; + + fn key1(&self) -> Self::K1<'_> { + self.user.as_str() + } + + fn key2(&self) -> Self::K2<'_> { + self.environment.name.as_str() + } + + bi_upcast!(); +} + +// ----------------------------------------------------------------------------- +// Source synchronization +// ----------------------------------------------------------------------------- +// +// A target's source tree is carried as a set of content-addressed files. The +// sender describes the tree it wants to exist, the receiver answers with the +// content it does not already hold, the sender uploads only that, and a commit +// makes the receiver's filesystem match. +// +// Whole files rather than diffs, because build sources are small and content +// addressing already collapses the unchanged ones — the same shape Bazel and +// Buck2 arrived at. Complete manifests rather than changesets, because the +// same environment may be synchronized from a different machine tomorrow, and +// a changeset computed against one machine's state is meaningless against +// another's. + +/// A BLAKE3 digest of a file's contents, lowercase hex. +/// +/// BLAKE3 because the sender re-hashes changed files on every save, and it is +/// several times faster than SHA-256 for that. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(transparent)] +pub struct Digest(pub String); + +impl std::fmt::Display for Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl Digest { + /// Whether this looks like a digest at all. + /// + /// Worth checking before a digest reaches the filesystem: it names a file + /// in the content store, so anything outside `[0-9a-f]{64}` is either a + /// bug or an attempt to escape the store's directory. + pub fn is_well_formed(&self) -> bool { + self.0.len() == 64 + && self + .0 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + } +} + +/// One file in a synchronized tree. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FileEntry { + /// Where the file goes, relative to the tree root, `/`-separated. + pub path: String, + /// The digest of its contents. + pub digest: Digest, + /// Whether the execute bit is set. The only mode bit that survives the + /// trip, because it is the only one a build cares about. + pub executable: bool, +} + +/// The complete desired state of a target's source tree. +/// +/// Complete, not a changeset: a path absent from this is a path that should +/// not exist, which is the only way deletions and renames can be expressed. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct TreeManifest { + pub entries: Vec, +} + +/// What the receiver still needs before a manifest can be applied. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct SyncPlan { + /// Digests the receiver holds nowhere — neither in its content store nor + /// anywhere in the tree it already has. + /// + /// Content already present under a different path is not listed: a rename + /// or a move costs nothing, because the receiver copies it locally. + pub missing: Vec, +} + +/// What applying a manifest did. +#[derive( + Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema, +)] +pub struct CommitResult { + pub created: usize, + pub updated: usize, + pub deleted: usize, + pub unchanged: usize, +} + +/// What a build needs in order to fetch its dependencies. +/// +/// Sources are synchronized from a developer's machine, but the things a build +/// pulls from Github are not: they are fetched by the instance itself, which +/// therefore needs credentials for them. These are the caller's own — the same +/// token that authorized the request that carries them — so an instance can +/// reach exactly what its owner can reach and nothing more. +/// +/// Nothing keeps a copy. `vw-svc` reads these off the request it is already +/// authorizing and passes them straight through. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +pub struct Credentials { + /// The Github login the token belongs to. + pub user: String, + /// A Github access token. + pub token: String, +} + +/// Written by hand so that debug formatting a request, a struct that contains +/// one of these, or an error that quotes one cannot put a live credential in a +/// log. The derived version would print the token. +impl std::fmt::Debug for Credentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credentials") + .field("user", &self.user) + .field("token", &"") + .finish() + } +} + +/// How a client wants the vivado worker for a session set up. +/// +/// Only what the instance cannot work out for itself. The tree, the workspace +/// configuration and the dependency cache are all on its side already, so the +/// two flags that select what to build are the whole of it — everything else +/// is read from `vw.toml` where the sources are. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct VivadoSessionQuery { + /// `--part`, for a workspace that declares parts at the top level. + #[serde(default)] + pub part: Option, + /// `--variant`, for a workspace that declares variants. + #[serde(default)] + pub variant: Option, + /// Attach the Tcl call stack to INFO messages, not only to warnings and + /// errors. + #[serde(default)] + pub info_with_stack: bool, + /// Forward vivado's unclassified chatter rather than discarding it. + #[serde(default)] + pub verbose: bool, +} + +/// What removing an instance's build output came to. +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + JsonSchema, +)] +pub struct CleanResult { + /// Whether there was any build output to remove. + pub existed: bool, + /// How much space it was taking, measured before it went. + pub bytes: u64, +} + +/// Which testbenches to run, and how. +/// +/// Everything here is a choice the developer made on the command line. +/// Discovery itself is not: it reads the tree, and the tree is on the +/// instance. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct BenchQuery { + /// Substring match against a testbench's entity name. Absent runs all. + #[serde(default)] + pub filter: Option, + /// The VHDL standard, as `nvc` spells it. + #[serde(default)] + pub standard: Option, + /// How many benches run at once. Absent lets the instance decide from its + /// own processor count, which is the number that matters — it is the + /// machine doing the work. + #[serde(default)] + pub concurrency: Option, + /// Directory names to skip while looking for benches, comma separated. + /// + /// One string rather than a repeated parameter because a query parameter + /// has to be scalar, and because that is already how `--ignore` is + /// written on the command line. + #[serde(default)] + pub ignore: Option, +} + +impl BenchQuery { + /// The directory names to skip, as a list. + pub fn ignored(&self) -> Vec { + self.ignore + .iter() + .flat_map(|joined| joined.split(',')) + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect() + } +} + +/// How to reach an environment's object store, and the key that opens it. +/// +/// Generated on the artifact instance and handed out from there, so the admin +/// credential that minted it never leaves the machine that holds the store. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +pub struct S3Credentials { + /// The base URL of the S3 API, e.g. `http://172.30.0.7:3900`. + /// + /// Composed from [`port`](Self::port) and an address by whoever knows + /// which address the instance is reachable on. Each side supplies what it + /// knows: the instance cannot tell which of its addresses another machine + /// can use, and nobody else should be guessing which port it chose. + pub endpoint: String, + /// The port the store serves S3 on, as the instance running it configured + /// it. + pub port: u16, + /// The region the store answers to. Garage's own default is `garage`, and + /// signing fails against the wrong one. + pub region: String, + /// The bucket this environment's artifacts go in. + pub bucket: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +/// Written by hand so a secret cannot reach a log through a debug format. +impl std::fmt::Debug for S3Credentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("S3Credentials") + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"") + .finish() + } +} + +/// Which of an environment's buckets is being asked about. +/// +/// One store holds a bucket per kind of instance that produces artifacts, and +/// one key opens all of them — what differs between two callers is only which +/// bucket is theirs to write to. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct ObjectStoreQuery { + /// The instance kind whose bucket is wanted. Absent means vivado, which is + /// the one that produces images today. + #[serde(default)] + pub kind: Option, +} + +/// One artifact an environment's build produced. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct Artifact { + /// Which instance built it, and so which bucket it is in. + pub kind: TargetKind, + /// The file's name, which is its key in the bucket. + pub name: String, + /// Its size in bytes. + pub size: u64, + /// When the store last accepted it, as the store reports it. + pub modified: Option, +} + +/// Which artifact is being fetched. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct ArtifactPathParam { + pub name: String, + pub kind: TargetKind, + /// The artifact's file name. + pub artifact: String, +} + +/// Which generated file is wanted. +/// +/// A path rather than a path segment because these are nested several levels +/// deep — vivado buries a stub inside its project — and a path with slashes in +/// it is not something a route can carry. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct GeneratedFileQuery { + pub path: String, +} + +/// How to build the driver. +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +pub struct DriverBuildQuery { + /// Build with optimizations. + #[serde(default)] + pub release: bool, + /// Anything else to put on cargo's command line, separated by spaces. + /// + /// One string because a query parameter has to be scalar. Split on + /// whitespace at the far end, so a value containing a space cannot be + /// expressed — no flag the driver build needs has one, and the + /// alternative is reimplementing half a shell's quoting rules. + #[serde(default)] + pub args: Option, +} + +impl DriverBuildQuery { + /// The extra arguments, as cargo will receive them. + pub fn arguments(&self) -> Vec { + self.args + .iter() + .flat_map(|joined| joined.split_whitespace()) + .map(str::to_owned) + .collect() + } +} + +/// What clearing an environment's artifacts came to. +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + JsonSchema, +)] +pub struct ArtifactsCleared { + /// How many objects were removed. + pub removed: usize, + /// How much space they were taking. + pub bytes: u64, +} diff --git a/vw-api/Cargo.toml b/vw-api/Cargo.toml new file mode 100644 index 0000000..d1810d4 --- /dev/null +++ b/vw-api/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vw-api" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "VW service API" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[dependencies] +vw-api-types-versions = {path = "../vw-api-types/versions" } +dropshot.workspace = true +schemars.workspace = true +dropshot-api-manager = "0.7.2" +dropshot-api-manager-types = "0.7.2" +serde.workspace = true +uuid.workspace = true diff --git a/vw-api/src/lib.rs b/vw-api/src/lib.rs new file mode 100644 index 0000000..7b79107 --- /dev/null +++ b/vw-api/src/lib.rs @@ -0,0 +1,328 @@ +use dropshot::{ + api_description, FreeformBody, HttpError, HttpResponseCreated, + HttpResponseDeleted, HttpResponseOk, HttpResponseUpdatedNoContent, Path, + Query, RequestContext, ResultsPage, TypedBody, UntypedBody, + WebsocketChannelResult, WebsocketConnection, +}; +use dropshot_api_manager_types::api_versions; +use vw_api_types_versions::latest; + +api_versions!([ + // WHEN CHANGING THE API (part 1 of 2): + // + // +- Pick a new semver and define it in the list below. The list MUST + // | remain sorted, which generally means that your version should go at + // | the very top. + // | + // | Duplicate this line, uncomment the *second* copy, update that copy for + // | your new API version, and leave the first copy commented out as an + // | example for the next person. + // v + // (next_int, IDENT), + (1, INITIAL), +]); + +// WHEN CHANGING THE API (part 2 of 2): +// +// The call to `api_versions!` above defines constants of type +// `semver::Version` that you can use in your Dropshot API definition to specify +// the version when a particular endpoint was added or removed. For example, if +// you used: +// +// (1, INITIAL) +// +// Then you could use `VERSION_INITIAL` as the version in which endpoints were +// added or removed. + +/// User API. For all endpoints, the caller is identified by a Github access +/// token in the authorization header of the request. +#[api_description] +pub trait VwUserApi { + type Context; + + // + // Environment CRUD + // + + /// Return a list of all environments for the calling user. + #[endpoint { + method = GET, + path = "/environments", + }] + async fn get_environments( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Create an environment with the specified name. + /// + /// The images the environment's instances boot from are chosen here and + /// pinned for the life of the environment. Any image named in the body + /// must already exist. + /// + /// Returns the ssh keypair generated for the new environment, so a caller + /// can save it without a second round trip. The same pair is available + /// afterwards from `get_environment_keys`. + #[endpoint { + method = PUT, + path = "/environment/{name}" + }] + async fn create_environment( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + /// Get an environment with the specified name. + #[endpoint { + method = GET, + path = "/environment/{name}" + }] + async fn get_environment( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete an environment with the specified name. + #[endpoint { + method = DELETE, + path = "/environment/{name}" + }] + async fn delete_environment( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + // + // Source synchronization + // + // Relayed to the instance that serves the named half of the environment, + // over the rack's internal network. The client never reaches an instance + // directly, so this is the only route source takes. + // + + /// Report what source content an environment's instance still needs. + #[endpoint { + method = POST, + path = "/environment/{name}/target/{kind}/sync/plan", + }] + async fn sync_plan( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + /// Deliver one piece of source content. + #[endpoint { + method = PUT, + path = "/environment/{name}/target/{kind}/sync/blob/{digest}", + }] + async fn sync_blob( + rqctx: RequestContext, + path_params: Path, + body: UntypedBody, + ) -> Result; + + /// Make the instance's source tree match the manifest. + #[endpoint { + method = POST, + path = "/environment/{name}/target/{kind}/sync/commit", + }] + async fn sync_commit( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + /// Discard an environment's source tree, so the next sync sends all of it. + /// + /// An ordinary sync does not need this: the instance is told the whole + /// desired state and replaces whatever differs from it. This is for when + /// what the instance says it has is itself in question — with the tree and + /// the delivered content both gone there is nothing left to be wrong + /// about, and the sync that follows sends every file. + /// + /// Build output on the instance is not touched. + #[endpoint { + method = DELETE, + path = "/environment/{name}/target/{kind}/sync", + }] + async fn sync_clear( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove everything a build wrote on one of an environment's instances. + /// + /// `target/` is the one directory synchronization will never touch, in + /// either direction, so it outlives every push and has to be removed on + /// purpose. Source on the instance is left alone. + #[endpoint { + method = DELETE, + path = "/environment/{name}/target/{kind}/build-output", + }] + async fn clean_build_output( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Build the driver on an environment's helios instance. + /// + /// Relayed frame for frame. The driver's target is native there and its + /// pinned toolchain is installed there, which is the whole reason the + /// build does not happen on a developer's machine. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{name}/driver/build", + }] + async fn driver_build( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// Run an environment's testbenches on its vivado instance. + /// + /// Relayed frame for frame. What comes back is the same stream of events a + /// local run produces, so the display on a developer's terminal is driven + /// by exactly what would have driven it here. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{name}/bench/session", + }] + async fn bench_session( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// Drive a vivado worker on an environment's vivado instance. + /// + /// Relayed frame for frame to the instance, which spawns the worker when + /// this opens and tears it down when it closes. A build is a conversation + /// that runs for a long time and produces output throughout, so it is a + /// websocket rather than a request and a reply — the developer sees each + /// message as vivado emits it, exactly as they would running it locally. + /// + /// The source being built is whatever the last synchronization put on the + /// instance. Nothing is shipped over this socket. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{name}/vivado/session", + }] + async fn vivado_session( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// The VHDL vivado generated for this environment's IP. + /// + /// A developer's static analysis needs these to resolve the design, and + /// they only exist where vivado ran. Relayed from the vivado instance. + #[endpoint { + method = POST, + path = "/environment/{name}/generated", + }] + async fn generated_manifest( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// One generated file's contents. + #[endpoint { + method = GET, + path = "/environment/{name}/generated/file", + }] + async fn generated_file( + rqctx: RequestContext, + path_params: Path, + query: Query, + ) -> Result, HttpError>; + + /// List the artifacts an environment's builds have produced. + /// + /// Read from the environment's own object store, which lives on its + /// artifact instance. + #[endpoint { + method = GET, + path = "/environment/{name}/artifacts", + }] + async fn get_artifacts( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + + /// Remove every artifact an environment has stored. + /// + /// Irreversible: the object store keeps no versions, so what goes is gone. + /// The instances themselves are untouched — a build's output is still on + /// the machine that made it until that machine is cleaned or replaced. + #[endpoint { + method = DELETE, + path = "/environment/{name}/artifacts", + }] + async fn clear_artifacts( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Download one artifact. + /// + /// Streamed through this service rather than handed out as a link to the + /// store. The store sits on the rack's internal network, and its instance's + /// external address is often only reachable over a VPN — needing one to + /// collect a build's output would make this useless from anywhere else. + /// The body is passed through as it arrives, so an image of any size costs + /// this service no more memory than a small one. + #[endpoint { + method = GET, + path = "/environment/{name}/artifacts/{kind}/{artifact}", + }] + async fn get_artifact( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Fetch the ssh keypair that opens an environment's instances. + /// + /// The private key is only ever handed to the environment's owner. + #[endpoint { + method = GET, + path = "/environment/{name}/keys" + }] + async fn get_environment_keys( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; +} + +/// Administrator API. For all endpoints, the caller is identified by a Github +/// access token in the authorization header of the request. The caller's Github +/// username must provided to the server at startup in the --admin_users +/// arguments for authorization to be granted. +#[api_description] +pub trait VwAdminApi { + type Context; + + /// Return a list of all environments. + #[endpoint { + method = GET, + path = "/environments", + }] + async fn get_environments( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Delete an environment with the specified name for the specified user. + #[endpoint { + method = DELETE, + path = "/environment/{user}/{name}" + }] + async fn delete_environment( + rqctx: RequestContext, + path_params: Path, + ) -> Result; +} diff --git a/vw-bench/Cargo.toml b/vw-bench/Cargo.toml new file mode 100644 index 0000000..c567aae --- /dev/null +++ b/vw-bench/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vw-bench" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Running a workspace's testbenches, wherever they run" + +[dependencies] +camino.workspace = true +serde.workspace = true +tokio.workspace = true +thiserror.workspace = true +vw-lib = { path = "../vw-lib" } diff --git a/vw-bench/src/lib.rs b/vw-bench/src/lib.rs new file mode 100644 index 0000000..c2f5f52 --- /dev/null +++ b/vw-bench/src/lib.rs @@ -0,0 +1,211 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Running a workspace's testbenches. +//! +//! One orchestrator, used from both sides. A developer running `vw bench` on +//! their own machine and an agent running it on an instance discover the same +//! benches, prepare the workspace the same way and fan out the same number at +//! a time — because it is the same code, not because two copies were kept in +//! step. +//! +//! What differs between the two is only how a single bench is launched and +//! where the progress goes, so those are the two things passed in. Everything +//! else is a property of the workspace, and the workspace is wherever this +//! happens to be running. +//! +//! **Why a subprocess per bench.** `nvc` inherits stdio, so simulation output +//! goes wherever the process's output goes. Running several in one process +//! would interleave their output beyond repair and lose the ability to show +//! only the failing one's. A child per bench also gives each an isolated +//! build directory and keeps one bench's crash from taking the batch with it — +//! the same reasoning cargo-nextest arrives at. + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Instant; + +use camino::Utf8Path; +use serde::{Deserialize, Serialize}; +use tokio::sync::Semaphore; + +/// What to run. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Request { + /// Substring match against a testbench's entity name. `None` runs all. + pub filter: Option, + /// The VHDL standard, as `nvc` spells it. + pub standard: String, + /// How many run at once. Zero means one — a limit of none would be a + /// machine with every bench on it at the same time. + pub concurrency: usize, + /// Directory names to skip while looking for benches. + pub ignore: Vec, +} + +/// What happened, as it happens. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum Event { + /// The full set, before any of it runs. Sent first so a display can size + /// itself to the work rather than growing as results arrive. + Discovered { names: Vec }, + /// One bench has started. + Started { name: String }, + /// One bench is done. + Finished { + name: String, + passed: bool, + seconds: f64, + /// Everything the bench wrote, kept for the ones that failed. A + /// passing bench's output is nobody's business. + output: String, + }, + /// Something worth saying that is not a result. + Note { message: String }, +} + +/// How to launch one bench. +/// +/// The two callers use different binaries — a developer's `vw` and an +/// instance's agent — so the command is supplied rather than assumed. +pub type Launch = + Arc tokio::process::Command + Send + Sync>; + +/// What a whole run came to. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Summary { + pub passed: usize, + pub failed: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum BenchError { + #[error("not in a vw workspace (no vw.toml in the parent chain)")] + NoWorkspace, + #[error("looking for testbenches under {0}")] + Discover(camino::Utf8PathBuf, #[source] vw_lib::VwError), + #[error("generating anodizer structs")] + Anodize(#[source] vw_lib::VwError), + #[error("generating bench scaffolds")] + Scaffold(#[source] vw_lib::VwError), +} + +/// The benches a request selects, in the order they will run. +pub fn discover( + workspace: &Utf8Path, + request: &Request, +) -> Result, BenchError> { + let bench_dir = workspace.join("bench"); + if !bench_dir.exists() { + return Ok(Vec::new()); + } + + let ignore: HashSet = request.ignore.iter().cloned().collect(); + let benches = vw_lib::list_testbenches(&bench_dir, &ignore, true) + .map_err(|e| BenchError::Discover(bench_dir.clone(), e))?; + + let mut names: Vec = benches + .into_iter() + .map(|t| t.name) + .filter(|n| n.to_lowercase().ends_with("_tb")) + .filter(|n| { + request + .filter + .as_ref() + .is_none_or(|f| n.contains(f.as_str())) + }) + .collect(); + names.sort(); + names.dedup(); + + Ok(names) +} + +/// Put the workspace in a state where every bench can build. +/// +/// Done once, before the fan-out, rather than per bench: both steps write +/// generated files into the workspace, and several children doing that at +/// once would race each other over the same paths. +pub async fn prepare( + workspace: &Utf8Path, + standard: vw_lib::VhdlStandard, +) -> Result<(), BenchError> { + vw_lib::ensure_anodized(workspace, standard, None) + .await + .map_err(BenchError::Anodize)?; + + // A missing generated `bench//Cargo.toml` — after a `git clean`, + // say — stops cargo loading the bench workspace manifest at all, which + // fails every bench's rust build rather than only the cosim ones. + vw_lib::ensure_bench_scaffolds(workspace).map_err(BenchError::Scaffold) +} + +/// Run `names`, at most `concurrency` at a time, reporting as it goes. +pub async fn run( + workspace: &Utf8Path, + names: Vec, + concurrency: usize, + launch: Launch, + report: impl Fn(Event) + Send + Sync + 'static, +) -> Summary { + let report = Arc::new(report); + report(Event::Discovered { + names: names.clone(), + }); + + let permits = Arc::new(Semaphore::new(concurrency.max(1))); + let mut running = Vec::new(); + + for name in names { + let permits = Arc::clone(&permits); + let report = Arc::clone(&report); + let launch = Arc::clone(&launch); + let workspace = workspace.to_owned(); + + running.push(tokio::spawn(async move { + let _permit = permits.acquire().await.expect("semaphore closed"); + report(Event::Started { name: name.clone() }); + + let build_dir = format!("{}/{}", vw_lib::BUILD_DIR, name); + let started = Instant::now(); + let finished = launch(&name, &build_dir) + .current_dir(workspace.as_std_path()) + .output() + .await; + let seconds = started.elapsed().as_secs_f64(); + + let (passed, output) = match finished { + Ok(out) => { + let mut text = + String::from_utf8_lossy(&out.stdout).into_owned(); + text.push_str(&String::from_utf8_lossy(&out.stderr)); + (out.status.success(), text) + } + Err(e) => (false, format!("could not start the bench: {e}")), + }; + + report(Event::Finished { + name, + passed, + seconds, + output, + }); + passed + })); + } + + let mut summary = Summary::default(); + for handle in running { + match handle.await { + Ok(true) => summary.passed += 1, + // A child that failed, or a task that panicked carrying it. Both + // are a bench that did not pass, and the batch continues either + // way — one broken bench should not hide the state of the rest. + Ok(false) | Err(_) => summary.failed += 1, + } + } + + summary +} diff --git a/vw-cli/Cargo.toml b/vw-cli/Cargo.toml index c27a1aa..2a70a1b 100644 --- a/vw-cli/Cargo.toml +++ b/vw-cli/Cargo.toml @@ -14,7 +14,43 @@ path = "src/main.rs" [dependencies] vw-lib = { path = "../vw-lib" } -clap = { version = "4.0", features = ["derive"] } +vw-htcl = { path = "../vw-htcl" } +vw-eda = { path = "../vw-eda" } +vw-vivado = { path = "../vw-vivado" } +vw-analyzer = { path = "../vw-analyzer" } +vw-ip = { path = "../vw-ip" } +vw-htcl-cmd = { path = "../vw-htcl-cmd" } +vw-repl = { path = "../vw-repl" } +vw-api-client = { path = "../vw-api-client" } +tokio-tungstenite.workspace = true +reqwest = { version = "0.13", features = ["json"] } +vw-remote = { path = "../vw-remote" } +vw-bench = { path = "../vw-bench" } +vw-sync = { path = "../vw-sync" } +vw-api-types-versions = { path = "../vw-api-types/versions" } +clap.workspace = true +thiserror.workspace = true colored = "2.0" tokio.workspace = true camino.workspace = true +dirs.workspace = true +tracing-subscriber.workspace = true +petgraph.workspace = true +futures.workspace = true +indicatif.workspace = true +serde_json.workspace = true +ratatui.workspace = true +crossterm.workspace = true +# Demangle Rust (v0/legacy) and C++ symbols in captured crash backtraces so +# `vw bench` failure blocks are readable. +rustc-demangle = "0.1" +cpp_demangle = "0.4" +# `vw run --bunyan`: structured bunyan-format log records (RFC3339 timestamps +# via chrono, `hostname` field via gethostname) for the looker CI viewer. +chrono = "0.4" +gethostname = "1.1" +# `vw cloud sync --watch`: filesystem change notification. +notify = "6" + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-cli/src/bench_runner.rs b/vw-cli/src/bench_runner.rs new file mode 100644 index 0000000..895a85c --- /dev/null +++ b/vw-cli/src/bench_runner.rs @@ -0,0 +1,355 @@ +//! `vw bench` on this machine, and the display both machines feed. +//! +//! The orchestration — what to run, in what order, how many at once — lives in +//! `vw-bench`, because an instance running the same benches has to make the +//! same decisions. What lives here is the part that belongs to a terminal: +//! turning the run's events into the nextest-style panel, and rendering a +//! failure well enough to act on. + +use std::sync::Arc; +use std::time::Instant; + +use camino::Utf8Path; +use colored::*; + +use crate::test_ui::{print_result_line, NextestPanel}; + +#[derive(Clone)] +pub struct BenchResult { + name: String, + /// Combined stdout+stderr of the subprocess. Only failures are kept, so + /// there is no passing bench's output to decide what to do with. + output: String, +} + +/// Run every matching testbench in parallel, here. +/// +/// `filter` is a substring match against the testbench entity name +/// (nextest-style); `None` runs all. `concurrency` caps how many run at once. +pub async fn run_benches( + cwd: &Utf8Path, + filter: Option<&str>, + list: bool, + concurrency: usize, + vhdl_std: vw_lib::VhdlStandard, + ignore: &[String], +) -> Result<(), Box> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + + let request = vw_bench::Request { + filter: filter.map(str::to_owned), + standard: vhdl_std.to_string(), + concurrency, + ignore: ignore.to_vec(), + }; + let names = vw_bench::discover(&ws, &request)?; + + if list { + for n in &names { + println!("{}", n.cyan()); + } + return Ok(()); + } + if names.is_empty() { + report_nothing_found(&ws, filter); + return Ok(()); + } + + if let Err(e) = vw_bench::prepare(&ws, vhdl_std).await { + eprintln!("{} {e}", "error:".bright_red()); + std::process::exit(1); + } + + // One `vw bench --build-dir …` per bench. The child is this same + // binary: it already knows how to run exactly one bench into an isolated + // directory, which is what the internal `--build-dir` mode is for. + let exe = std::env::current_exe()?; + let standard = vhdl_std.to_string(); + let launch: vw_bench::Launch = + Arc::new(move |name: &str, build_dir: &str| { + let mut command = tokio::process::Command::new(&exe); + command.args([ + "bench", + name, + "--build-dir", + build_dir, + "--std", + &standard, + ]); + command + }); + + let overall = Instant::now(); + let panel = Arc::new(NextestPanel::new(names.len() as u64, "testbenches")); + let failures = Arc::new(std::sync::Mutex::new(Vec::new())); + + let summary = { + let panel = Arc::clone(&panel); + let failures = Arc::clone(&failures); + let rows = + Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); + vw_bench::run(&ws, names, concurrency, launch, move |event| { + drive_panel(&panel, &rows, &failures, event) + }) + .await + }; + + panel.clear(); + + let failures = failures.lock().expect("failures").clone(); + if !failures.is_empty() { + println!("\n{}\n", "failures:".red().bold()); + for f in &failures { + print_bench_failure(f); + } + } + print_result_line(panel.passed(), panel.failed(), overall.elapsed()); + if summary.failed > 0 { + std::process::exit(1); + } + Ok(()) +} + +/// Turn one run event into whatever the terminal should show for it. +/// +/// Shared by the local runner and the remote one, so a bench finishing looks +/// the same whichever machine it finished on. +pub fn drive_panel( + panel: &NextestPanel, + rows: &std::sync::Mutex< + std::collections::HashMap, + >, + failures: &std::sync::Mutex>, + event: vw_bench::Event, +) { + match event { + vw_bench::Event::Discovered { .. } => {} + vw_bench::Event::Started { name } => { + let row = panel.start(&name); + rows.lock().expect("rows").insert(name, row); + } + vw_bench::Event::Finished { + name, + passed, + seconds, + output, + } => { + let row = rows.lock().expect("rows").remove(&name); + if let Some(row) = row { + panel.finish(&row, &name, passed, seconds); + } + if !passed { + failures + .lock() + .expect("failures") + .push(BenchResult { name, output }); + } + } + vw_bench::Event::Note { message } => { + eprintln!("{} {message}", "info:".cyan()); + } + } +} + +fn report_nothing_found(ws: &Utf8Path, filter: Option<&str>) { + let bench_dir = ws.join("bench"); + if !bench_dir.exists() { + eprintln!("no bench directory found under {}", ws.as_str().dimmed()); + return; + } + match filter { + Some(f) => eprintln!("no testbenches matched {}", f.dimmed()), + None => eprintln!( + "no testbenches found under {}", + bench_dir.as_str().dimmed() + ), + } +} + +/// Failure block for one bench, in the same visual frame `vw test` uses: +/// the key diagnostic lines surfaced up front, then a (demangled) tail of +/// the captured output for context. +fn print_bench_failure(f: &BenchResult) { + let bar = "─".repeat(64); + println!("{}", bar.red()); + println!(" {} {}", "✗".red().bold(), f.name.bold()); + + let lines: Vec<&str> = f.output.trim_end().lines().collect(); + + // The real reason (`** Fatal: …`, `panicked at …`, `error: …`) often + // sits far above the tail — pull those lines out and show them first. + let key: Vec<&str> = lines + .iter() + .copied() + .filter(|l| is_key_error_line(l)) + .collect(); + if !key.is_empty() { + println!("\n{}", "ERROR:".red().bold()); + for l in &key { + println!(" {}", demangle_line(l).red()); + } + } + + if !lines.is_empty() { + println!("\n{}", "OUTPUT (tail):".bright_black().bold()); + let start = lines.len().saturating_sub(40); + for l in &lines[start..] { + println!(" {}", demangle_line(l)); + } + } + println!("{}\n", bar.red()); +} + +/// Lines worth pulling out of a long crash dump. +fn is_key_error_line(line: &str) -> bool { + let l = line.trim_start(); + l.starts_with("** Fatal:") + || l.starts_with("** Error:") + || l.starts_with("error:") + || l.contains("panicked at") + || l.contains("Caught signal") + || l.starts_with("Assertion") + || l.contains("TEST FAILED") +} + +/// Demangle any Rust (`_R…`) or C++ (`_Z…`) symbols on a line so nvc +/// backtraces are legible. Lines with no mangled tokens pass through +/// untouched. +fn demangle_line(line: &str) -> String { + if !line.contains("_R") && !line.contains("_Z") { + return line.to_string(); + } + line.split(' ') + .map(|tok| { + if tok.starts_with("_R") || tok.starts_with("_Z") { + demangle_symbol(tok) + } else { + tok.to_string() + } + }) + .collect::>() + .join(" ") +} + +fn demangle_symbol(sym: &str) -> String { + if sym.starts_with("_R") { + // Rust v0 — unambiguous. `{:#}` drops the disambiguator hash. + if let Ok(d) = rustc_demangle::try_demangle(sym) { + return format!("{d:#}"); + } + } else if sym.starts_with("_Z") { + // C++ (Itanium) — with a fallback to legacy Rust `_ZN…` mangling. + if let Ok(s) = cpp_demangle::Symbol::new(sym) { + if let Ok(d) = s.demangle(&cpp_demangle::DemangleOptions::default()) + { + return d; + } + } + if let Ok(d) = rustc_demangle::try_demangle(sym) { + return format!("{d:#}"); + } + } + sym.to_string() +} + +/// Run the workspace's testbenches on an environment's instance. +/// +/// The display is the local one, driven by the same events — a bench finishing +/// looks the same whichever machine it finished on, because the thing that +/// decided how it looks never moved. +pub async fn run_benches_remotely( + session: &crate::cloud::Session, + environment: &str, + filter: Option<&str>, + concurrency: Option, + vhdl_std: vw_lib::VhdlStandard, + ignore: &[String], +) -> Result<(), Box> { + use futures::StreamExt; + + let ignored = (!ignore.is_empty()).then(|| ignore.join(",")); + let upgraded = session + .client + .bench_session( + environment, + concurrency.map(|n| n as u32), + filter, + ignored.as_deref(), + Some(&vhdl_std.to_string()), + ) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let mut socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + upgraded, + tokio_tungstenite::tungstenite::protocol::Role::Client, + None, + ) + .await; + + let overall = Instant::now(); + let failures = Arc::new(std::sync::Mutex::new(Vec::new())); + let rows = + Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); + // Sized when the instance says what it found, not before: only it can + // see the tree to count. + let mut panel: Option> = None; + let mut summary = vw_bench::Summary::default(); + + while let Some(message) = socket.next().await { + let text = match message? { + tokio_tungstenite::tungstenite::Message::Text(text) => text, + tokio_tungstenite::tungstenite::Message::Close(_) => break, + _ => continue, + }; + + match serde_json::from_str::(&text)? { + vw_remote::BenchEvent::Progress { event } => { + if let vw_bench::Event::Discovered { names } = &event { + if names.is_empty() { + eprintln!("no testbenches matched"); + return Ok(()); + } + panel = Some(Arc::new(NextestPanel::new( + names.len() as u64, + "testbenches", + ))); + } + if let Some(panel) = panel.as_ref() { + drive_panel(panel, &rows, &failures, event); + } + } + vw_remote::BenchEvent::Done { passed, failed } => { + summary = vw_bench::Summary { passed, failed }; + break; + } + vw_remote::BenchEvent::Fatal { message } => { + if let Some(panel) = panel.as_ref() { + panel.clear(); + } + return Err(message.into()); + } + } + } + + let Some(panel) = panel else { + // The instance never got as far as saying what it found. + return Err("the instance ended the run without reporting".into()); + }; + panel.clear(); + + let failures = failures.lock().expect("failures").clone(); + if !failures.is_empty() { + println!("\n{}\n", "failures:".red().bold()); + for f in &failures { + print_bench_failure(f); + } + } + print_result_line(panel.passed(), panel.failed(), overall.elapsed()); + if summary.failed > 0 { + std::process::exit(1); + } + Ok(()) +} diff --git a/vw-cli/src/cloud.rs b/vw-cli/src/cloud.rs new file mode 100644 index 0000000..e41c4b2 --- /dev/null +++ b/vw-cli/src/cloud.rs @@ -0,0 +1,1289 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw cloud` — manage remote build environments hosted by a vw service. +//! +//! An environment is a set of cloud instances (vivado, helios, artifact) that +//! a workspace builds on. These commands are thin wrappers over the vw service +//! user API, reached through the progenitor generated client in +//! `vw-api-client`. +//! +//! The caller is identified by a Github access token read from `~/.netrc` — +//! the same credential `vw update` uses to fetch private dependencies, so +//! there is nothing extra to configure. Having no token is not by itself an +//! error: a service run with `--no-auth` answers without one, so the request +//! goes out unauthenticated and the missing credential is only reported if +//! the service turns it away. + +use camino::{Utf8Path, Utf8PathBuf}; +use clap::{Args, Subcommand}; +use colored::*; +use indicatif::ProgressBar; +use std::time::{Duration, Instant}; +use vw_api_client::user::{types, Client}; + +/// Where the service lives if the caller does not say otherwise. Matches +/// `vw-svc`'s own default user API port. +const DEFAULT_SERVICE_URL: &str = "http://localhost:2727"; + +/// How often `--wait` asks what an environment's instances are doing. +/// +/// Instances take minutes, so this is far more often than anything changes. +/// It is tuned for how quickly the answer arrives once it does, and the cost +/// is a handful of requests against a service that is doing nothing else for +/// this caller. +const WAIT_POLL: Duration = Duration::from_secs(2); + +/// How long `--wait` waits before giving up. +/// +/// Long enough that a rack under load is never mistaken for a broken one, and +/// short enough that a script does not hang for an afternoon. Reaching it is +/// not a statement that the environment failed — only that it did not finish +/// while somebody was watching, and the states it stopped at are reported. +const WAIT_LIMIT: Duration = Duration::from_secs(900); + +/// Hosts to look for a Github access token under, in preference order. +const CREDENTIAL_HOSTS: [&str; 2] = ["github.com", "api.github.com"]; + +/// The instances an environment is made of, in display order, each with the +/// account to log in as. +/// +/// The account is a property of the image the instance boots: the vivado and +/// artifact images are Ubuntu, the helios one is not. +const INSTANCES: [(&str, &str); 3] = [ + ("vivado", "ubuntu"), + ("helios", "root"), + ("artifact", "ubuntu"), +]; + +/// The status the service answers with when it wants a Github token and did +/// not get an acceptable one. +const UNAUTHORIZED: u16 = 401; + +#[derive(Args)] +pub struct CloudArgs { + #[arg( + long, + global = true, + env = "VW_SVC_URL", + default_value = DEFAULT_SERVICE_URL, + help = "Base URL of the vw service" + )] + url: String, + + #[arg( + long, + global = true, + help = "Accept the service's TLS certificate without verifying it. \ + For development services fronted by a self-signed \ + certificate; this gives up any guarantee about who is on the \ + other end, and your access token is sent to whatever answers." + )] + insecure: bool, + + #[command(subcommand)] + command: CloudCommand, +} + +#[derive(Subcommand)] +pub enum CloudCommand { + #[command(about = "List your remote build environments")] + List, + #[command(about = "Create a remote build environment")] + Create { + #[arg(help = "Environment name")] + name: String, + #[arg( + long, + value_name = "IMAGE", + help = "Image the vivado instance boots from. Defaults to the \ + newest the service can see." + )] + vivado_image: Option, + #[arg( + long, + value_name = "IMAGE", + help = "Image the helios instance boots from. Defaults to the \ + newest the service can see." + )] + helios_image: Option, + #[arg( + long, + value_name = "IMAGE", + help = "Image the artifact instance boots from. Defaults to the \ + newest the service can see." + )] + artifact_image: Option, + #[arg( + long, + value_name = "DIR", + help = "Directory to write the environment's ssh key into. \ + Replaces any key already there. [default: ~/.ssh]" + )] + key_dir: Option, + #[arg( + long, + help = "Do not return until every instance is running. Their \ + agents come up a few seconds after that." + )] + wait: bool, + }, + #[command(about = "Show a remote build environment")] + Get { + #[arg(help = "Environment name")] + name: String, + }, + #[command(about = "Delete a remote build environment")] + Delete { + #[arg(help = "Environment name")] + name: String, + }, + #[command(about = "Push the workspace to an environment's instances")] + Sync { + #[arg(help = "Environment name")] + name: String, + #[arg( + long, + help = "Discard the instance's source tree first, so every file \ + is sent again" + )] + force: bool, + #[arg(long, help = "Keep syncing as files change, until interrupted")] + watch: bool, + #[arg( + long, + value_name = "MS", + default_value_t = 150, + help = "How long to wait for changes to settle before syncing" + )] + debounce: u64, + }, + #[command(about = "List or download an environment's build artifacts")] + Artifacts { + #[arg(help = "Environment name")] + name: String, + #[arg( + long, + value_name = "FILE", + help = "Download this artifact instead of listing. Repeat, or \ + use --all, for several." + )] + get: Vec, + #[arg( + long, + conflicts_with = "get", + help = "Download every artifact instead of listing" + )] + all: bool, + #[arg( + long, + conflicts_with_all = ["get", "all"], + help = "Remove every stored artifact. The object store keeps no \ + versions, so this cannot be undone." + )] + clear: bool, + #[arg( + long, + value_name = "DIR", + help = "Directory to write downloads into [default: .]" + )] + out: Option, + }, + #[command( + about = "Download the ssh key that opens an environment's instances" + )] + Keys { + #[arg(help = "Environment name")] + name: String, + #[arg( + long, + value_name = "DIR", + help = "Directory to write the key into. Replaces any key \ + already there. [default: ~/.ssh]" + )] + dir: Option, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum CloudError { + #[error("no vw workspace here; run this from one, or from a directory inside it")] + NoWorkspace, + #[error( + "no cloud environments exist for you. Create one with `vw cloud \ + create `, or pass --local to build on this machine" + )] + NoEnvironments, + #[error( + "you have several cloud environments ({}); say which with --env, or \ + pass --local to build on this machine", + .0.join(", ") + )] + AmbiguousEnvironment(Vec), + #[error("no artifact called '{0}'; run `vw cloud artifacts ` to see what there is")] + NoSuchArtifact(String), + #[error("'{0}' is not a name an artifact may be written under")] + UnsafeArtifactName(String), + #[error("no driver here; {0} does not exist")] + NoDriver(Utf8PathBuf), + #[error("scanning {0}")] + Scan(camino::Utf8PathBuf, #[source] vw_sync::ScanError), + #[error("reading {0}")] + ReadSource(String, #[source] std::io::Error), + #[error("watching the workspace for changes")] + Watch(#[source] notify::Error), + #[error("reading github credentials: {0}")] + Credentials(#[from] vw_lib::VwError), + #[error( + "this service requires authorization, but no github access token was \ + found in ~/.netrc. Add a machine entry for {} whose password is a \ + github personal access token with access to oxidecomputer/redhawk", + CREDENTIAL_HOSTS[0] + )] + NoCredentials, + #[error("building the api client: {0}")] + Client(#[from] vw_api_client::Error), + #[error("the service returned {status}: {message}")] + Service { status: u16, message: String }, + #[error("talking to the service: {0}")] + Transport(String), + #[error("creating {0}: {1}")] + KeyDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}: {1}")] + KeyWrite(Utf8PathBuf, #[source] std::io::Error), + #[error( + "the {kind} instance of '{environment}' is {state}; it is not coming up" + )] + InstanceUnusable { + environment: String, + kind: String, + state: String, + }, + #[error( + "'{environment}' was still not fully running after {seconds}s ({states}). \ + It was created and may yet come up; check with `vw cloud get {environment}`" + )] + WaitTimedOut { + environment: String, + seconds: u64, + states: String, + }, + #[error("cannot determine the home directory to put the key in")] + NoHomeDirectory, + #[error("home directory {0:?} is not valid utf-8")] + HomeNotUtf8(std::path::PathBuf), +} + +/// A connection to the service, and whether we had a credential to offer it. +pub struct Session { + pub client: Client, + /// Whether a Github token was found and sent. A `401` means very different + /// things depending on this: no token means the caller needs to set one + /// up, a token means Github turned it down. + authenticated: bool, +} + +pub async fn run(args: CloudArgs) -> Result<(), CloudError> { + let session = Session::new(&args.url, args.insecure)?; + + match args.command { + CloudCommand::List => list(&session).await, + CloudCommand::Create { + name, + vivado_image, + helios_image, + artifact_image, + key_dir, + wait, + } => { + create( + &session, + &name, + types::EnvironmentCreate { + vivado_image, + helios_image, + artifact_image, + }, + key_dir.as_deref(), + wait, + ) + .await + } + CloudCommand::Get { name } => get(&session, &name).await, + CloudCommand::Delete { name } => delete(&session, &name).await, + CloudCommand::Artifacts { + name, + get, + all, + clear, + out, + } => artifacts(&session, &name, &get, all, clear, out.as_deref()).await, + CloudCommand::Keys { name, dir } => { + fetch_keys(&session, &name, dir.as_deref()).await + } + CloudCommand::Sync { + name, + force, + watch, + debounce, + } => { + crate::cloud_sync::run( + // `vw cloud sync` is the command that means "everything", so + // it is the one place with no filter. + &session, + &name, + force, + watch, + std::time::Duration::from_millis(debounce), + None, + ) + .await + } + } +} + +impl Session { + fn new(url: &str, insecure: bool) -> Result { + // A missing token is not fatal here. Services run with `--no-auth` + // answer without one, so send what we have and let the service decide. + let token = access_token()?; + Ok(Session { + client: vw_api_client::user_client(&vw_api_client::ClientConfig { + base_url: url, + token: token.as_deref(), + insecure, + })?, + authenticated: token.is_some(), + }) + } + + /// Render a client error in terms of what the service said. + /// + /// The service's own message is the part a person can act on, so pull it + /// out of the response body rather than reporting the client's error enum. + pub fn error( + &self, + error: vw_api_client::user::Error, + ) -> CloudError { + match error { + vw_api_client::user::Error::ErrorResponse(response) => { + let status = response.status().as_u16(); + if status == UNAUTHORIZED && !self.authenticated { + // We never offered a credential, so the service's "no + // token is present" is really a message about this + // machine's setup. + return CloudError::NoCredentials; + } + CloudError::Service { + status, + message: response.into_inner().message, + } + } + other => CloudError::Transport(with_causes(&other)), + } + } +} + +/// Flatten an error and everything underneath it onto one line. +/// +/// The client's own `Display` stops at "error sending request", which hides +/// the part worth reading — a certificate that did not verify, a refused +/// connection. The causes are what tell someone what to do next. +fn with_causes(error: &dyn std::error::Error) -> String { + let mut message = error.to_string(); + let mut cause = error.source(); + while let Some(error) = cause { + // Errors in this chain tend to embed their own cause in their + // `Display`, so only append what has not been said already. + let text = error.to_string(); + if !message.contains(&text) { + message.push_str(&format!(": {text}")); + } + cause = error.source(); + } + message +} + +async fn list(session: &Session) -> Result<(), CloudError> { + // The endpoint takes no pagination parameters, so this one page is every + // environment the caller owns. + let page = session + .client + .get_environments() + .await + .map_err(|e| session.error(e))?; + let environments = page.into_inner().items; + + if environments.is_empty() { + println!( + "No cloud environments. Create one with {}.", + "vw cloud create ".cyan() + ); + return Ok(()); + } + + println!("Environments:"); + for environment in &environments { + println!( + " {} - {}", + environment.name.cyan(), + instance_summary(environment) + ); + } + Ok(()) +} + +async fn create( + session: &Session, + name: &str, + images: types::EnvironmentCreate, + key_dir: Option<&Utf8Path>, + wait: bool, +) -> Result<(), CloudError> { + let keys = session + .client + .create_environment(name, &images) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + println!( + "{} Created cloud environment: {}", + "✓".bright_green(), + name.cyan() + ); + + // The environment exists either way, so a key that cannot be saved is a + // warning and a recovery instruction rather than a failure. Reporting an + // error here would suggest the create had not happened. + match save_keys(name, &keys, key_dir) { + Ok((private, public)) => report_keys(&private, &public), + Err(e) => { + eprintln!("{} {e}", "warning:".yellow()); + eprintln!( + " the environment was created; fetch its key with {}", + format!("vw cloud keys {name}").cyan(), + ); + } + } + + if wait { + wait_for_instances(session, name).await?; + } + + Ok(()) +} + +/// Wait until every one of `name`'s instances is running. +/// +/// Creating an environment records the intent and returns; the instances are +/// the reconciler's business and appear a minute or two later. That is the +/// right shape for a service but the wrong one for a script, which has nothing +/// to do with an environment whose machines do not exist yet. +/// +/// What is waited for is the instance state Oxide reports, which is a weaker +/// promise than the environment being usable: an agent takes a few more +/// seconds to start after the machine it runs on does. It is still the useful +/// boundary, because everything before it is measured in minutes. +async fn wait_for_instances( + session: &Session, + name: &str, +) -> Result<(), CloudError> { + let spinner = ProgressBar::new_spinner(); + spinner.enable_steady_tick(Duration::from_millis(120)); + spinner.set_message(format!("waiting for {}", name.cyan())); + + let deadline = Instant::now() + WAIT_LIMIT; + loop { + let environment = session + .client + .get_environment(name) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + spinner.set_message(instance_summary(&environment)); + + // Reported in the order they are displayed, so the kind named in an + // error is the one whose state the caller just watched go red. + let states: Vec<(&str, Option<&types::InstanceState>)> = INSTANCES + .iter() + .zip(instances(&environment)) + .map(|((kind, _), instance)| { + (*kind, instance.as_ref().map(|i| &i.state)) + }) + .collect(); + + // A machine that has failed or gone away is not on its way to running, + // and waiting out the limit would only delay saying so. + for (kind, state) in &states { + if let Some( + state @ (types::InstanceState::Failed + | types::InstanceState::Destroyed), + ) = state + { + spinner.finish_and_clear(); + return Err(CloudError::InstanceUnusable { + environment: name.to_owned(), + kind: (*kind).to_owned(), + state: state.to_string(), + }); + } + } + + if states + .iter() + .all(|(_, state)| *state == Some(&types::InstanceState::Running)) + { + spinner.finish_and_clear(); + println!( + "{} All instances running: {}", + "✓".bright_green(), + name.cyan() + ); + return Ok(()); + } + + if Instant::now() >= deadline { + spinner.finish_and_clear(); + return Err(CloudError::WaitTimedOut { + environment: name.to_owned(), + seconds: WAIT_LIMIT.as_secs(), + states: instance_summary(&environment), + }); + } + + tokio::time::sleep(WAIT_POLL).await; + } +} + +async fn get(session: &Session, name: &str) -> Result<(), CloudError> { + let environment = session + .client + .get_environment(name) + .await + .map_err(|e| session.error(e))?; + let environment = environment.into_inner(); + + println!("{}", environment.name.cyan()); + if let Some(images) = &environment.images { + println!(" images"); + for ((label, _), image) in INSTANCES.iter().zip([ + &images.vivado, + &images.helios, + &images.artifact, + ]) { + println!(" {label:<8} {}", image.name.bright_black()); + } + } + + println!(" instances"); + for ((label, user), instance) in + INSTANCES.iter().zip(instances(&environment)) + { + match instance { + // An instance the service has asked for but not yet heard back + // about has a state and no address, so there is nothing to show + // but the state. + Some(instance) => println!( + " {label:<8} {:<20} {}", + colored_state(&instance.state), + match instance.external_ip { + Some(ip) => format!("{user}@{ip}"), + None => String::new(), + }, + ), + None => { + println!(" {label:<8} {}", "not provisioned".bright_black()) + } + } + } + + print_login_hints(&environment); + Ok(()) +} + +/// Print a ready-to-run ssh line for every instance that can be reached. +/// +/// The key path is the one `vw cloud create` and `vw cloud keys` write by +/// default; a caller who redirected it elsewhere has to substitute their own. +fn print_login_hints(environment: &types::Environment) { + let reachable: Vec = INSTANCES + .iter() + .zip(instances(environment)) + .filter_map(|((label, user), instance)| { + let ip = instance.as_ref()?.external_ip?; + let key = default_key_dir() + .map(|dir| dir.join(format!("vw-{}.key", environment.name))) + .map(|path| path.to_string()) + .unwrap_or_else(|_| { + format!("~/.ssh/vw-{}.key", environment.name) + }); + Some(format!( + " ssh -i {key} {user}@{ip}{}", + format!(" # {label}").bright_black() + )) + }) + .collect(); + + if reachable.is_empty() { + return; + } + + println!(); + println!("log in with:"); + for line in reachable { + println!("{line}"); + } +} + +async fn delete(session: &Session, name: &str) -> Result<(), CloudError> { + session + .client + .delete_environment(name) + .await + .map_err(|e| session.error(e))?; + println!( + "{} Deleted cloud environment: {}", + "✓".bright_green(), + name.cyan() + ); + Ok(()) +} + +/// The environment's instances in [`INSTANCES`] order. +fn instances( + environment: &types::Environment, +) -> [&Option; 3] { + [ + &environment.vivado_instance, + &environment.helios_instance, + &environment.artifact_instance, + ] +} + +/// A one line rendering of which of an environment's instances are up. +fn instance_summary(environment: &types::Environment) -> String { + INSTANCES + .iter() + .zip(instances(environment)) + .map(|((label, _), instance)| match instance { + Some(instance) => { + format!("{label}: {}", colored_state(&instance.state)) + } + None => format!("{label}: {}", "none".bright_black()), + }) + .collect::>() + .join(" ") +} + +/// Write an environment's ssh key out where ssh can find it. +/// +/// The service generates a keypair per environment and attaches it to every +/// instance, so this is all that stands between `vw cloud create` and being +/// able to log in. +async fn fetch_keys( + session: &Session, + name: &str, + dir: Option<&Utf8Path>, +) -> Result<(), CloudError> { + let keys = session + .client + .get_environment_keys(name) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let (private, public) = save_keys(name, &keys, dir)?; + report_keys(&private, &public); + + Ok(()) +} + +/// Write an environment's keypair into `dir`, returning the paths written. +/// +/// Replaces whatever was there. An environment's keypair is generated once, +/// when it is created, so a file already sitting at one of these paths belongs +/// to an earlier environment of the same name — which cannot still exist, or +/// this one could not have been created. Keeping it would only leave a key to +/// nowhere in the way of the one that works. +fn save_keys( + name: &str, + keys: &types::SshKeyPair, + dir: Option<&Utf8Path>, +) -> Result<(Utf8PathBuf, Utf8PathBuf), CloudError> { + let dir = match dir { + Some(dir) => dir.to_owned(), + None => default_key_dir()?, + }; + let private = dir.join(format!("vw-{name}.key")); + let public = dir.join(format!("vw-{name}.pub")); + + std::fs::create_dir_all(&dir) + .map_err(|e| CloudError::KeyDir(dir.clone(), e))?; + write_key(&private, keys.private_key.as_bytes(), true)?; + write_key(&public, keys.public_key.as_bytes(), false)?; + + Ok((private, public)) +} + +fn report_keys(private: &Utf8Path, public: &Utf8Path) { + println!("{} Wrote {}", "✓".bright_green(), private.as_str().cyan()); + println!("{} Wrote {}", "✓".bright_green(), public.as_str().cyan()); +} + +/// Where keys go when the caller does not say: alongside every other ssh key. +fn default_key_dir() -> Result { + let home = dirs::home_dir().ok_or(CloudError::NoHomeDirectory)?; + let home = + Utf8PathBuf::from_path_buf(home).map_err(CloudError::HomeNotUtf8)?; + Ok(home.join(".ssh")) +} + +/// Write a key file, keeping a private one to the current user. +/// +/// ssh refuses to use a private key that anyone else can read, so getting the +/// mode wrong here would leave a key that looks fine and does not work. +fn write_key( + path: &Utf8Path, + contents: &[u8], + private: bool, +) -> Result<(), CloudError> { + // Removed rather than truncated in place: a key left at 0400 by something + // else cannot be opened for writing even by its owner, and replacing it is + // the whole point. + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(CloudError::KeyWrite(path.to_owned(), e)), + } + + std::fs::write(path, contents) + .map_err(|e| CloudError::KeyWrite(path.to_owned(), e))?; + + #[cfg(unix)] + if private { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| CloudError::KeyWrite(path.to_owned(), e))?; + } + #[cfg(not(unix))] + let _ = private; + + Ok(()) +} + +/// Render an instance state in a colour that says how to feel about it. +/// +/// The states an environment moves through are worth telling apart at a +/// glance: whether it is ready, still on its way, deliberately idle, or +/// broken. `colored` drops the escapes when stdout is not a terminal, so +/// piping this stays plain. +fn colored_state(state: &types::InstanceState) -> ColoredString { + let text = state.to_string(); + match state { + // Up and usable. + types::InstanceState::Running => text.green(), + // On its way somewhere. Nothing to do but wait. + types::InstanceState::Creating + | types::InstanceState::Starting + | types::InstanceState::Stopping + | types::InstanceState::Rebooting + | types::InstanceState::Migrating + | types::InstanceState::Repairing => text.magenta(), + // Idle, and fine. + types::InstanceState::Stopped => text.bright_black(), + // Broken, or gone while the service still expects it to be here. + types::InstanceState::Failed | types::InstanceState::Destroyed => { + text.red() + } + } +} + +/// The Github access token to authenticate with, if this machine has one. +/// +/// A missing `~/.netrc`, or one with no entry for Github, yields `None` rather +/// than an error — the service may not require authorization. A netrc that +/// exists but cannot be read or parsed is still an error, since that is a +/// broken setup the user wants to hear about. +fn access_token() -> Result, CloudError> { + for host in CREDENTIAL_HOSTS { + if let Some(token) = vw_lib::get_access_token_from_netrc(host)? { + return Ok(Some(token)); + } + } + Ok(None) +} + +/// Open a vivado session on an environment's instance. +/// +/// What comes back drives exactly like a local worker, because it implements +/// the same trait and speaks the same protocol. The worker starts when this +/// socket opens and dies when it closes, so a run never inherits anything from +/// the one before it. +pub async fn open_vivado_session( + session: &Session, + environment: &str, + params: vw_remote::SessionParams, +) -> Result, CloudError> { + let upgraded = session + .client + .vivado_session( + environment, + Some(params.info_with_stack), + params.part.as_deref(), + params.variant.as_deref(), + Some(params.verbose), + ) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + upgraded, + tokio_tungstenite::tungstenite::protocol::Role::Client, + None, + ) + .await; + + // No note sink here: where an instance's progress reports belong depends + // on who is asking. A one-shot run writes them to stderr. A full-screen + // REPL must not — anything written straight to the terminal lands in the + // middle of a frame it does not control — so it leaves this unset and the + // reports fall through to its scrollback instead. + Ok(vw_remote::RemoteBackend::new(socket)) +} + +/// Which environment a bare `vw run` should use. +/// +/// Named explicitly, or inferred when there is no ambiguity to resolve. Two +/// environments and no `--env` is a question only the developer can answer, +/// and guessing at it would run a build somewhere they did not intend. +pub async fn pick_environment( + session: &Session, + named: Option<&str>, +) -> Result { + if let Some(name) = named { + return Ok(name.to_owned()); + } + + let environments = session + .client + .get_environments() + .await + .map_err(|e| session.error(e))? + .into_inner() + .items; + + match environments.len() { + 0 => Err(CloudError::NoEnvironments), + 1 => Ok(environments[0].name.clone()), + _ => Err(CloudError::AmbiguousEnvironment( + environments.iter().map(|e| e.name.clone()).collect(), + )), + } +} + +impl Session { + /// A session pointed at whatever service the environment names. + /// + /// For commands that are not `vw cloud` and so have no `--url` of their + /// own. Same variable, same default, so a developer configures the service + /// once and every command finds it. + /// + /// `insecure` comes from the command's own flag; `VW_SVC_INSECURE` says + /// the same thing for a shell that talks to a development service all day + /// and would otherwise pass the flag every time. Either is enough. + pub fn from_env(insecure: bool) -> Result { + let url = std::env::var("VW_SVC_URL") + .unwrap_or_else(|_| DEFAULT_SERVICE_URL.to_owned()); + let insecure = insecure + || std::env::var("VW_SVC_INSECURE") + .is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")); + Session::new(&url, insecure) + } + + /// Whether the failure was the service being out of reach, rather than the + /// service saying no. + /// + /// The difference decides whether a bare `vw run` may quietly fall back to + /// building here: unreachable is a working-from-a-train problem, but a + /// service that answered and refused is telling us something. + pub fn unreachable(error: &CloudError) -> bool { + matches!(error, CloudError::Transport(_)) + } +} + +/// Remove the build output on an environment's instances. +pub async fn clean_build_output( + session: &Session, + environment: &str, +) -> Result<(), CloudError> { + crate::cloud_sync::clean(session, environment).await +} + +/// Push the workspace to an environment before building in it. +/// +/// A build reads what is on the instance, so this is what makes it the same +/// code the developer is looking at. +pub async fn sync_for_build( + session: &Session, + environment: &str, + only: Option, +) -> Result<(), CloudError> { + crate::cloud_sync::run( + session, + environment, + false, + false, + std::time::Duration::from_millis(0), + only, + ) + .await +} + +/// List an environment's artifacts, or fetch some of them. +/// +/// Everything comes through the service rather than from the store directly. +/// The store is on the rack's internal network and its instance's external +/// address is usually only reachable over a VPN — needing one to collect a +/// build's output would make this useless from a train, which is exactly where +/// people want it. +async fn artifacts( + session: &Session, + environment: &str, + get: &[String], + all: bool, + clear: bool, + out: Option<&Utf8Path>, +) -> Result<(), CloudError> { + if clear { + return clear_artifacts(session, environment).await; + } + + let available = session + .client + .get_artifacts(environment) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let wanted: Vec<&vw_api_types_versions::latest::Artifact> = if all { + available.iter().collect() + } else if get.is_empty() { + show(&available); + return Ok(()); + } else { + // Named artifacts have to exist, and saying which one does not is more + // use than a download that quietly produces nothing. + let mut wanted = Vec::new(); + for name in get { + let found = available + .iter() + .find(|artifact| artifact.name == *name) + .ok_or_else(|| CloudError::NoSuchArtifact(name.clone()))?; + wanted.push(found); + } + wanted + }; + + if wanted.is_empty() { + println!("{}", "no artifacts to download".bright_black()); + return Ok(()); + } + + let directory = out.unwrap_or(Utf8Path::new(".")); + std::fs::create_dir_all(directory) + .map_err(|e| CloudError::KeyDir(directory.to_owned(), e))?; + + for artifact in wanted { + download(session, environment, artifact, directory).await?; + } + + Ok(()) +} + +/// Throw away everything an environment has stored. +/// +/// No confirmation, matching the rest of `vw cloud` — `delete` takes three +/// instances down without asking either. What it does report is exactly what +/// went, since that is the only record left of it. +async fn clear_artifacts( + session: &Session, + environment: &str, +) -> Result<(), CloudError> { + let cleared = session + .client + .clear_artifacts(environment) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + if cleared.removed == 0 { + println!("{}", "nothing stored to clear".bright_black()); + return Ok(()); + } + + println!( + "{} removed {} artifact(s), {}", + "\u{2713}".bright_green(), + cleared.removed, + human_bytes(cleared.bytes), + ); + + Ok(()) +} + +/// Show what an environment has built. +/// +/// Grouped by the instance that made it, because a flat alphabetical list +/// interleaves two unrelated builds — a vivado report between two driver +/// binaries tells nobody anything. +fn show(available: &[vw_api_types_versions::latest::Artifact]) { + if available.is_empty() { + println!( + "{}", + "no artifacts yet; run a build that produces one".bright_black(), + ); + return; + } + + let mut sorted: Vec<&vw_api_types_versions::latest::Artifact> = + available.iter().collect(); + sorted.sort_by(|a, b| { + source_order(a.kind) + .cmp(&source_order(b.kind)) + .then_with(|| a.name.cmp(&b.name)) + }); + + for artifact in sorted { + println!( + "{:<10} {:>10} {}", + colored_source(artifact.kind), + human_bytes(artifact.size), + artifact.name, + ); + } +} + +/// Which instance built it, in a colour that is not the other one's. +/// +/// Two builds land in the same listing and they have nothing to do with each +/// other; telling them apart should not require reading. +fn colored_source( + kind: vw_api_types_versions::latest::TargetKind, +) -> colored::ColoredString { + let name = kind.to_string(); + match kind { + vw_api_types_versions::latest::TargetKind::Vivado => name.cyan(), + vw_api_types_versions::latest::TargetKind::Helios => name.magenta(), + } +} + +/// Sort key for the instance that built something. +/// +/// Fixed rather than alphabetical so the order does not change if a kind is +/// ever renamed, and so hardware comes before software, which is the order +/// they happen in. +fn source_order(kind: vw_api_types_versions::latest::TargetKind) -> u8 { + match kind { + vw_api_types_versions::latest::TargetKind::Vivado => 0, + vw_api_types_versions::latest::TargetKind::Helios => 1, + } +} + +/// Fetch one artifact into `directory`. +async fn download( + session: &Session, + environment: &str, + artifact: &vw_api_types_versions::latest::Artifact, + directory: &Utf8Path, +) -> Result<(), CloudError> { + use futures::StreamExt; + + let response = session + .client + .get_artifact(environment, &artifact.kind, &artifact.name) + .await + .map_err(|e| session.error(e))?; + + // An artifact's name carries the stage that produced it — `synth/x.edif` + // and `route/x.edif` are different netlists — so the structure is kept on + // the way down rather than flattened into collisions. + let path = directory.join(safe_name(&artifact.name)?); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CloudError::KeyDir(parent.to_owned(), e))?; + } + + // Written as it arrives rather than collected first: an image runs to + // hundreds of megabytes and there is no reason for it to be in memory on + // the way past. + let mut file = std::fs::File::create(&path) + .map_err(|e| CloudError::KeyWrite(path.clone(), e))?; + + let mut body = response.into_inner_stream(); + let mut written = 0u64; + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|e| CloudError::Transport(e.to_string()))?; + std::io::Write::write_all(&mut file, &chunk) + .map_err(|e| CloudError::KeyWrite(path.clone(), e))?; + written += chunk.len() as u64; + } + + println!( + "{} {} ({})", + "\u{2713}".bright_green(), + path.as_str(), + human_bytes(written), + ); + + Ok(()) +} + +/// An artifact's name, once it has been established that it is only a name. +/// +/// The name becomes a path on the developer's machine, and it arrives from a +/// service. Nothing we run puts anything strange in it, but "nothing we run" +/// is not the same as "nothing", and a download that can write outside the +/// directory it was pointed at is the kind of thing that is obvious only +/// afterwards. +fn safe_name(name: &str) -> Result<&str, CloudError> { + let refused = || CloudError::UnsafeArtifactName(name.to_owned()); + + if name.is_empty() || name.starts_with('/') || name.contains('\\') { + return Err(refused()); + } + for component in name.split('/') { + if component.is_empty() || component == ".." || component == "." { + return Err(refused()); + } + } + + Ok(name) +} + +/// A byte count as a person would say it. +fn human_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut size = bytes as f64; + let mut unit = 0; + while size >= 1024.0 && unit < UNITS.len() - 1 { + size /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + format!("{size:.1} {}", UNITS[unit]) + } +} + +/// Bring the VHDL vivado generated for this environment's IP into the local +/// tree. +/// +/// A static analysis running here has to resolve `entity ip._wrapper` +/// and `entity xil_defaultlib.`, and those only exist where vivado ran. +/// They land at the same paths they have on the instance, so a language server +/// can open them and "go to definition" arrives somewhere real. +/// +/// Only what differs is fetched. A check that changed no IP therefore costs +/// one round trip, which matters because this runs before every check. +pub async fn fetch_generated_ip( + session: &Session, + environment: &str, + workspace: &Utf8Path, +) -> Result { + let manifest = session + .client + .generated_manifest(environment) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let mut written = 0usize; + for entry in &manifest.entries { + let path = workspace.join(safe_name(&entry.path)?); + + // Already here and already right. The common case: IP changes rarely + // and a check runs constantly. + if let Ok(existing) = std::fs::read(&path) { + if vw_sync::digest_bytes(&existing) == entry.digest { + continue; + } + } + + let contents = session + .client + .generated_file(environment, &entry.path) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let bytes = futures::TryStreamExt::try_fold( + contents.into_inner(), + Vec::new(), + |mut collected, chunk| async move { + collected.extend_from_slice(&chunk); + Ok(collected) + }, + ) + .await + .map_err(|e| CloudError::Transport(e.to_string()))?; + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CloudError::KeyDir(parent.to_owned(), e))?; + } + std::fs::write(&path, bytes) + .map_err(|e| CloudError::KeyWrite(path.clone(), e))?; + written += 1; + } + + Ok(written) +} + +#[cfg(test)] +mod test { + use super::*; + + /// The escape `colored` emits for each colour we use. + const GREEN: &str = "\u{1b}[32m"; + const RED: &str = "\u{1b}[31m"; + const MAGENTA: &str = "\u{1b}[35m"; + const GRAY: &str = "\u{1b}[90m"; + + #[test] + fn instance_states_are_coloured_by_meaning() { + // `colored` suppresses escapes off a terminal, which is what the test + // harness looks like. + colored::control::set_override(true); + + let rendered = + |state: types::InstanceState| colored_state(&state).to_string(); + + for (state, expected) in [ + (types::InstanceState::Running, GREEN), + // Everything in motion reads the same, because the answer is + // always "wait". + (types::InstanceState::Creating, MAGENTA), + (types::InstanceState::Starting, MAGENTA), + (types::InstanceState::Stopping, MAGENTA), + (types::InstanceState::Rebooting, MAGENTA), + (types::InstanceState::Migrating, MAGENTA), + (types::InstanceState::Repairing, MAGENTA), + (types::InstanceState::Stopped, GRAY), + (types::InstanceState::Failed, RED), + (types::InstanceState::Destroyed, RED), + ] { + let text = rendered(state); + assert!( + text.starts_with(expected), + "{state} should be coloured {expected:?}, got {text:?}", + ); + // The state name itself still has to be readable. + assert!(text.contains(&state.to_string())); + } + } +} diff --git a/vw-cli/src/cloud_sync.rs b/vw-cli/src/cloud_sync.rs new file mode 100644 index 0000000..b82f6da --- /dev/null +++ b/vw-cli/src/cloud_sync.rs @@ -0,0 +1,562 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Pushing a workspace to the instances that build it. +//! +//! A workspace is split across an environment's two instances: vivado builds +//! the hardware, helios builds whatever drives it, and neither needs the +//! other's sources. Where the line falls is not a question a workspace gets to +//! answer — a vw workspace keeps its driver in `driver`, so that is what goes +//! to helios and everything else goes to vivado. +//! +//! Nothing to declare, nothing to keep in step with the layout, and no way to +//! have a workspace whose configuration disagrees with where its files +//! actually are. +//! +//! Each target is synchronized independently: its own manifest, its own +//! content, its own commit. Nothing is shared between them, so there is no +//! ordering to get right and a failure to reach one instance does not hold up +//! the other. + +use camino::{Utf8Path, Utf8PathBuf}; +use colored::*; + +use futures::{StreamExt, TryStreamExt}; + +use crate::cloud::CloudError; + +/// How many pieces of content are sent at once. +/// +/// Enough to keep the link busy across a slow round trip, few enough not to +/// look like an attack to anything in between. +const UPLOAD_CONCURRENCY: usize = 16; +use vw_api_types_versions::latest as types; + +/// A target's slice of the workspace, ready to be sent. +pub struct Target { + pub kind: types::TargetKind, + /// The directory scanned for this target. + /// + /// A target with its own subtree is rooted there, so paths on the instance + /// are relative to it — `driver`'s `Cargo.toml` lands at `Cargo.toml`, + /// where cargo expects to find it, not at `driver/Cargo.toml`. + pub root: Utf8PathBuf, + /// Paths under the workspace root that belong to another target and must + /// not be sent with this one. + pub excluded: Vec, +} + +/// The directory a vw workspace keeps its driver in. +/// +/// Everything under it is built on helios and nothing else is, which is what +/// makes the split something this can decide rather than something a workspace +/// has to say. +pub const DRIVER: &str = "driver"; + +/// Work out how a workspace divides across an environment's instances. +/// +/// A workspace with no driver gets no helios target, rather than one with an +/// empty tree: an empty manifest is a valid instruction to delete everything, +/// and sending one because there is no `driver` directory would be a poor way +/// to find that out. +pub fn targets(workspace: &Utf8Path) -> Vec { + // Vivado takes the workspace as a whole, less the driver. + let mut targets = vec![Target { + kind: types::TargetKind::Vivado, + root: workspace.to_owned(), + excluded: vec![DRIVER.to_owned()], + }]; + + let driver = workspace.join(DRIVER); + if driver.is_dir() { + targets.push(Target { + kind: types::TargetKind::Helios, + root: driver, + excluded: Vec::new(), + }); + } + + targets +} + +/// Scan a target's tree, dropping anything claimed by another target. +pub fn scan( + target: &Target, +) -> Result { + let manifest = vw_sync::scan(&target.root)?; + + let entries = manifest + .entries + .into_iter() + .filter(|entry| { + !target.excluded.iter().any(|excluded| { + entry.path == *excluded + || entry.path.starts_with(&format!("{excluded}/")) + }) + }) + .collect(); + + Ok(types::TreeManifest { entries }) +} + +impl Target { + /// Read one of this target's files, for delivery. + pub fn read(&self, path: &str) -> std::io::Result> { + std::fs::read(self.root.join(path)) + } + + /// Find the file in `manifest` with this digest. + /// + /// A plan asks for content by digest, and the sender has it by path, so + /// something has to bridge the two. Any path with the right digest will + /// do — that is the whole point of naming content by what it is. + pub fn path_for<'a>( + &self, + manifest: &'a types::TreeManifest, + digest: &types::Digest, + ) -> Option<&'a str> { + manifest + .entries + .iter() + .find(|entry| entry.digest == *digest) + .map(|entry| entry.path.as_str()) + } +} + +/// Push the workspace to an environment, once or continuously. +pub async fn run( + session: &crate::cloud::Session, + environment: &str, + force: bool, + watch: bool, + debounce: std::time::Duration, + only: Option, +) -> Result<(), CloudError> { + let workspace = workspace_root()?; + let mut targets = targets(&workspace); + // A command that only drives one instance should not fail because the + // other one is down. `vw driver build` needs helios and nothing else; a + // vivado instance rebooting is not its problem. + if let Some(only) = only { + targets.retain(|target| target.kind == only); + } + announce(&targets); + + // Only ever the first pass. Forcing is an answer to a doubt about what is + // on the instance, and once the sync below has settled it there is nothing + // left to doubt — re-clearing on every file save would mean re-uploading + // the whole workspace to see a one line edit. + if force { + clear(session, environment, &targets).await?; + } + + sync_once(session, environment, &targets, true).await?; + + if !watch { + return Ok(()); + } + + println!(); + println!( + "{} {}", + "watching".bright_black(), + workspace.as_str().bright_black(), + ); + + let mut changes = watcher(&workspace)?; + loop { + // Wait for something to happen, then let the rest of the burst land + // before scanning. An editor writing a file is several events, and a + // branch switch is thousands; syncing on the first one would mean + // syncing a tree that is still moving. + if changes.recv().await.is_none() { + return Ok(()); + } + while tokio::time::timeout(debounce, changes.recv()).await.is_ok() {} + + if let Err(e) = sync_once(session, environment, &targets, false).await { + // A failed sync is not a reason to stop watching. The next save + // tries again, and an instance that is still coming up will be + // there shortly. + eprintln!("{} {e}", "error:".bright_red()); + } + } +} + +/// Say which instance is getting which directory. +/// +/// Worth the two lines because a target with nothing to do prints nothing, so +/// a workspace with no driver and one whose helios instance is merely up to +/// date look identical from the outside. Said once, before the first pass, +/// because it describes the workspace rather than anything a sync does. +fn announce(targets: &[Target]) { + for target in targets { + println!( + "{} {} {}", + "\u{2192}".bright_black(), + target.kind.to_string().cyan(), + target.root.as_str().bright_black(), + ); + } +} + +/// Throw away what every target's instance has. +/// +/// Leaves each instance as though it had never been synchronized, so the pass +/// that follows finds nothing there and sends the whole tree. This is the +/// entire implementation of forcing: the ordinary path already sends whatever +/// the instance is missing, so making the instance miss everything is all +/// there is to do. +async fn clear( + session: &crate::cloud::Session, + environment: &str, + targets: &[Target], +) -> Result<(), CloudError> { + for target in targets { + let result = session + .client + .sync_clear(environment, &target.kind) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + println!( + "{} {} cleared ({} removed)", + "\u{2717}".bright_black(), + target.kind.to_string().cyan(), + result.deleted, + ); + } + + Ok(()) +} + +/// One pass over every target. +async fn sync_once( + session: &crate::cloud::Session, + environment: &str, + targets: &[Target], + announce: bool, +) -> Result<(), CloudError> { + for target in targets { + let manifest = scan(target) + .map_err(|e| CloudError::Scan(target.root.clone(), e))?; + + let plan = session + .client + .sync_plan(environment, &target.kind, &manifest) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + // Uploaded several at a time. Each one is a whole round trip to the + // service, and source files are small enough that the time is almost + // entirely waiting rather than transferring — sending them one after + // another means the link sits idle for all of it. A first sync of a + // few hundred files is the case that makes this obvious. + // + // Blobs are named by their content and land in a store, so they may + // arrive in any order and are safe to send at once. The commit that + // follows is what imposes an order, and it happens after all of this. + let uploads = plan.missing.iter().map(|digest| { + let path = target + .path_for(&manifest, digest) + .expect("the plan only asks for content the manifest names"); + + async move { + let contents = target + .read(path) + .map_err(|e| CloudError::ReadSource(path.to_owned(), e))?; + + session + .client + .sync_blob( + environment, + &target.kind, + digest.0.as_str(), + contents, + ) + .await + .map_err(|e| session.error(e))?; + + Ok::<(), CloudError>(()) + } + }); + + futures::stream::iter(uploads) + .buffer_unordered(UPLOAD_CONCURRENCY) + .try_collect::>() + .await?; + + let result = session + .client + .sync_commit(environment, &target.kind, &manifest) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + report(target, &manifest, plan.missing.len(), &result, announce); + } + + Ok(()) +} + +/// Say what a target's sync did. +/// +/// A sync that changed nothing says so when `always` is set, and says nothing +/// otherwise. Both are wanted, in different places. Watching a workspace means +/// a sync per keystroke-ish burst, and a line each time that nothing happened +/// would bury the ones where something did. But a sync run once — before a +/// build, or because the developer asked for one — that printed nothing is +/// indistinguishable from a sync that did not run, and "did my sources +/// actually get there" is not a question anyone should have to answer by +/// reading the source. +fn report( + target: &Target, + manifest: &types::TreeManifest, + uploaded: usize, + result: &types::CommitResult, + always: bool, +) { + let changed = result.created + result.updated + result.deleted; + if changed == 0 { + if always { + println!( + "{} {} up to date ({} files)", + "\u{2713}".bright_green(), + target.kind.to_string().cyan(), + manifest.entries.len(), + ); + } + return; + } + + println!( + "{} {} +{} ~{} -{} ({} sent, {} files)", + "\u{2713}".bright_green(), + target.kind.to_string().cyan(), + result.created, + result.updated, + result.deleted, + uploaded, + manifest.entries.len(), + ); +} + +/// The workspace this command was run from. +/// +/// Walks up from the current directory, the way cargo and git do, so it can be +/// run from anywhere inside a workspace rather than only at its root. +fn workspace_root() -> Result { + let cwd = std::env::current_dir().map_err(|_| CloudError::NoWorkspace)?; + let mut dir = + Utf8PathBuf::from_path_buf(cwd).map_err(|_| CloudError::NoWorkspace)?; + + loop { + if dir.join("vw.toml").is_file() { + return Ok(dir); + } + if !dir.pop() { + return Err(CloudError::NoWorkspace); + } + } +} + +/// A stream of "something under `root` changed". +/// +/// The events themselves are discarded. Working out what changed from them is +/// a great deal of bookkeeping that a scan answers directly and correctly, +/// including for the cases events are worst at — a branch switch, a file +/// replaced by a directory, an editor writing through a temporary file. +fn watcher( + root: &Utf8Path, +) -> Result, CloudError> { + use notify::Watcher; + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let mut watcher = notify::recommended_watcher(move |event| { + if let Ok(notify::Event { kind, .. }) = event { + // Access events fire when a build merely reads the tree, which is + // constant and means nothing here. + if !matches!(kind, notify::EventKind::Access(_)) { + let _ = tx.send(()); + } + } + }) + .map_err(CloudError::Watch)?; + + watcher + .watch(root.as_std_path(), notify::RecursiveMode::Recursive) + .map_err(CloudError::Watch)?; + + // The watcher stops when it is dropped, and nothing else owns it. + std::mem::forget(watcher); + + Ok(rx) +} + +/// Remove the build output on every one of an environment's instances. +/// +/// Both halves, because both build: vivado writes under the workspace root and +/// the driver's cargo writes under `driver/`. Cleaning one and leaving the +/// other would be a surprising thing for one command to do. +pub async fn clean( + session: &crate::cloud::Session, + environment: &str, +) -> Result<(), CloudError> { + let workspace = workspace_root()?; + let targets = targets(&workspace); + + for target in &targets { + let result = session + .client + .clean_build_output(environment, &target.kind) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + if result.existed { + println!( + "{} {} removed {}", + "\u{2713}".bright_green(), + target.kind.to_string().cyan(), + human_bytes(result.bytes), + ); + } else { + println!( + "{} {} nothing to remove", + "\u{2713}".bright_green(), + target.kind.to_string().cyan(), + ); + } + } + + Ok(()) +} + +/// A byte count as a person would say it. +/// +/// Build output runs to gigabytes, and "12093847552" is a number nobody reads. +fn human_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut size = bytes as f64; + let mut unit = 0; + while size >= 1024.0 && unit < UNITS.len() - 1 { + size /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + format!("{size:.1} {}", UNITS[unit]) + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn workspace(files: &[&str]) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + for file in files { + let path = root.join(file); + std::fs::create_dir_all(path.parent().unwrap()).expect("parent"); + std::fs::write(&path, "contents").expect("write"); + } + (dir, root) + } + + #[test] + fn a_workspace_with_no_driver_is_all_vivados() { + let (_dir, root) = workspace(&["hdl/top.vhd", "vw.toml"]); + let targets = targets(&root); + + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].kind, types::TargetKind::Vivado); + + let manifest = scan(&targets[0]).expect("scan"); + let paths: Vec<&str> = + manifest.entries.iter().map(|e| e.path.as_str()).collect(); + assert_eq!(paths, ["hdl/top.vhd", "vw.toml"]); + } + + #[test] + fn the_driver_goes_to_helios_and_nowhere_else() { + // The metroid shape: everything is vivado's except `driver`. + let (_dir, root) = workspace(&[ + "hdl/top.vhd", + "vw.toml", + "driver/Cargo.toml", + "driver/src/lib.rs", + ]); + let targets = targets(&root); + + assert_eq!(targets.len(), 2); + + let vivado = scan(&targets[0]).expect("scan vivado"); + let paths: Vec<&str> = + vivado.entries.iter().map(|e| e.path.as_str()).collect(); + assert_eq!( + paths, + ["hdl/top.vhd", "vw.toml"], + "vivado should not be carrying the driver", + ); + + let helios = scan(&targets[1]).expect("scan helios"); + let paths: Vec<&str> = + helios.entries.iter().map(|e| e.path.as_str()).collect(); + // Rooted at `driver`, so cargo finds its manifest where it expects to. + assert_eq!(paths, ["Cargo.toml", "src/lib.rs"]); + } + + #[test] + fn a_name_that_merely_starts_with_driver_is_not_the_driver() { + // `driver` must not swallow `drivers-notes.md`, which shares its + // first six characters and nothing else. + let (_dir, root) = + workspace(&["driver/Cargo.toml", "drivers-notes.md"]); + let targets = targets(&root); + + let vivado = scan(&targets[0]).expect("scan"); + let paths: Vec<&str> = + vivado.entries.iter().map(|e| e.path.as_str()).collect(); + assert_eq!(paths, ["drivers-notes.md"]); + } + + #[test] + fn a_workspace_without_a_driver_syncs_its_hardware_anyway() { + // Plenty of workspaces are hardware only. That should sync what there + // is rather than fail — and certainly not send helios an empty + // manifest, which would mean "delete everything". + let (_dir, root) = workspace(&["hdl/top.vhd"]); + let targets = targets(&root); + + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].kind, types::TargetKind::Vivado); + } + + #[test] + fn build_output_is_not_in_any_target() { + let (_dir, root) = workspace(&[ + "hdl/top.vhd", + "target/synth/top.dcp", + "driver/Cargo.toml", + "driver/target/debug/thing", + ]); + let targets = targets(&root); + + for target in &targets { + let manifest = scan(target).expect("scan"); + assert!( + !manifest + .entries + .iter() + .any(|entry| entry.path.contains("target/")), + "{:?} is carrying build output", + target.kind, + ); + } + } +} diff --git a/vw-cli/src/driver.rs b/vw-cli/src/driver.rs new file mode 100644 index 0000000..72d74f8 --- /dev/null +++ b/vw-cli/src/driver.rs @@ -0,0 +1,141 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw driver build` — building the driver where it runs. +//! +//! The driver targets illumos and pins its own toolchain, so it is built on +//! the helios instance rather than on a developer's machine, which is usually +//! neither. Cargo's output comes back line by line and is printed exactly as +//! cargo wrote it, colour and all — the point is that it looks like a build, +//! because it is one. + +use camino::Utf8Path; +use colored::*; +use futures::StreamExt; + +use crate::cloud::{CloudError, Session}; + +/// Build the driver on an environment's helios instance. +/// +/// Returns whether the build succeeded, so the caller can decide the exit +/// code; a failed build is not an error in the sense of something having gone +/// wrong with vw. +pub async fn build( + session: &Session, + environment: &str, + release: bool, + args: &[String], +) -> Result { + let joined = (!args.is_empty()).then(|| args.join(" ")); + + let upgraded = session + .client + .driver_build(environment, joined.as_deref(), Some(release)) + .await + .map_err(|e| session.error(e))? + .into_inner(); + + let mut socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + upgraded, + tokio_tungstenite::tungstenite::protocol::Role::Client, + None, + ) + .await; + + while let Some(message) = socket.next().await { + let text = + match message.map_err(|e| CloudError::Transport(e.to_string()))? { + tokio_tungstenite::tungstenite::Message::Text(text) => text, + tokio_tungstenite::tungstenite::Message::Close(_) => break, + _ => continue, + }; + + match serde_json::from_str::(&text) + .map_err(|e| CloudError::Transport(e.to_string()))? + { + // A driver is more than one cargo invocation, because userland + // and a kernel module are different targets. Saying which is + // building keeps two sets of cargo output from reading as one. + vw_remote::DriverEvent::Building { unit } => { + println!("{:>12} {unit}", "Building".bright_green().bold()); + } + // Printed rather than rendered: cargo already said it better than + // anything here could. + vw_remote::DriverEvent::Line { text } => println!("{text}"), + // Named so a developer knows what to look for in the store + // without having to guess at target directory layout. + vw_remote::DriverEvent::Produced { artifacts, stored } => { + for artifact in &artifacts { + println!( + "{:>12} {artifact}", + "Produced".bright_green().bold(), + ); + } + if stored > 0 { + println!( + "{:>12} {stored} artifact(s) — `vw cloud artifacts` \ + to fetch them", + "Stored".bright_green().bold(), + ); + } + } + vw_remote::DriverEvent::Done { success, code } => { + if !success { + eprintln!( + "{} the driver build failed{}", + "error:".bright_red(), + match code { + Some(code) => format!(" (exit {code})"), + None => String::from(" (killed)"), + }, + ); + } + return Ok(success); + } + vw_remote::DriverEvent::Fatal { message } => { + return Err(CloudError::Transport(message)); + } + } + } + + // The socket closed without a verdict, which is not a build result. + Err(CloudError::Transport(String::from( + "the instance ended the build without reporting whether it worked", + ))) +} + +/// Build the driver on this machine. +/// +/// What `--local` does: run the same cargo command in the same directory, so +/// somebody with an illumos toolchain to hand is not forced through a network. +pub async fn build_locally( + workspace: &Utf8Path, + release: bool, + args: &[String], +) -> Result { + let driver = workspace.join(vw_cloud_driver_dir()); + if !driver.is_dir() { + return Err(CloudError::NoDriver(driver)); + } + + let mut command = tokio::process::Command::new("cargo"); + command.arg("build"); + if release { + command.arg("--release"); + } + command.args(args); + + let status = command + .current_dir(driver.as_std_path()) + .status() + .await + .map_err(|e| CloudError::Transport(format!("running cargo: {e}")))?; + + Ok(status.success()) +} + +/// The directory a vw workspace keeps its driver in. +fn vw_cloud_driver_dir() -> &'static str { + crate::cloud_sync::DRIVER +} diff --git a/vw-cli/src/htcl_test.rs b/vw-cli/src/htcl_test.rs new file mode 100644 index 0000000..ac150e7 --- /dev/null +++ b/vw-cli/src/htcl_test.rs @@ -0,0 +1,789 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw test` — htcl-level test runner. +//! +//! Discovers `@test`-annotated procs under `/test/**/ +//! *.htcl`, drives them against a Vivado session, and reports +//! results in a nextest-inspired format. +//! +//! Design shape: +//! - Each test file's non-`@test` top-level statements are treated +//! as setup (proc decls, `src` imports). They're shipped to +//! Vivado once per file per session. +//! - Tests without a specific attribute value run in a SHARED +//! Vivado session; `@test(dedicated-eda)` marks tests that need +//! their own Vivado process (spawned per test, capped by +//! `--test-threads`). +//! - Assertion failures throw Tcl errors, which the runner catches +//! as `BackendError::Tcl` and marks the enclosing test FAILED. + +use std::path::PathBuf; +use std::time::Instant; + +use camino::{Utf8Path, Utf8PathBuf}; +use colored::*; + +use vw_eda::EdaBackend; + +use crate::load_htcl_program_for_test; +use crate::test_ui::NextestPanel; +use indicatif::ProgressBar; + +/// Entry point wired from `main.rs::Commands::Test`. +#[allow(clippy::too_many_arguments)] +pub async fn run_htcl_tests( + cwd: &Utf8Path, + filter: Option, + list: bool, + test_threads: usize, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + let test_files = vw_lib::list_htcl_tests(&ws)?; + if test_files.is_empty() { + eprintln!("no tests found in {}", ws.join("test").as_str().dimmed()); + return Ok(()); + } + + // Discover phase — load each test file's program and enumerate + // its `@test` procs. + let mut all_tests: Vec = Vec::new(); + for path in &test_files { + let path_utf8 = Utf8PathBuf::from_path_buf(path.clone()) + .map_err(|p| format!("non-UTF8 path: {p:?}"))?; + let discovered = + discover_tests_in_file(&path_utf8, &ws, filter.as_deref()).await?; + all_tests.extend(discovered); + } + + if list { + for t in &all_tests { + let tag = if t.dedicated { " [dedicated-eda]" } else { "" }; + println!("{}::{}{}", t.display_path, t.name.cyan(), tag.dimmed()); + } + return Ok(()); + } + + if all_tests.is_empty() { + eprintln!( + "no tests matched the filter {}", + filter.as_deref().unwrap_or("").dimmed() + ); + return Ok(()); + } + + let overall_start = Instant::now(); + let mut summary = RunSummary::default(); + + // Shared bucket — one Vivado for all `@test` (without + // `dedicated-eda`) tests. + let (shared, dedicated): (Vec<_>, Vec<_>) = + all_tests.into_iter().partition(|t| !t.dedicated); + + let panel = + NextestPanel::new((shared.len() + dedicated.len()) as u64, "tests"); + + if !shared.is_empty() { + run_shared_bucket( + &ws, + shared, + &mut summary, + &panel, + part, + variant, + log_level, + info_with_stack, + ) + .await?; + } + if !dedicated.is_empty() { + run_dedicated_bucket( + &ws, + dedicated, + test_threads, + &mut summary, + &panel, + part, + variant, + log_level, + info_with_stack, + ) + .await?; + } + + panel.clear(); + print_summary(&summary, overall_start.elapsed()); + if summary.failed > 0 { + std::process::exit(1); + } + Ok(()) +} + +/// One `@test`-annotated proc discovered inside a test file. The +/// `program` and `setup_tcl_lines` fields carry everything the +/// runner needs to ship setup to Vivado before invoking the test. +struct TestCase { + /// Test file path, workspace-relative. + display_path: String, + /// Absolute path — used to key "which file's setup has been + /// shipped in this Vivado session already." + file_path: PathBuf, + /// Proc name as it appears at Tcl call level. + name: String, + /// True if `@test(dedicated-eda)` — this test wants its own + /// Vivado process. + dedicated: bool, + /// `@target(part=)` override — swaps the auto-project's + /// `-part` argument for this specific test. Meaningful only + /// for `dedicated-eda` tests (shared-bucket tests all reuse + /// the workspace-default project). + target_part_override: Option, + /// `@test(dedicated-eda variant=)` override — selects a + /// specific `[[workspace.variants]]` entry for this test. + /// The variant's `part` becomes the auto-project part; its + /// name flows through to `vw::vhdl_design_sources` so + /// variant-exclusive files filter correctly. Same + /// dedicated-eda restriction as `target_part_override`. + /// Meaningful only in variant-mode workspaces. + variant_override: Option, + /// Concrete Tcl lines to ship as setup before invoking the + /// test proc. Includes proc declarations, `src`-imported code, + /// and any top-level setup statements. + setup_tcl: Vec, +} + +async fn discover_tests_in_file( + file: &Utf8Path, + ws: &Utf8Path, + filter: Option<&str>, +) -> Result, Box> { + let program = load_htcl_program_for_test(file).await?; + let source = program.source.clone(); + let parsed = vw_htcl::parse(&source); + // Only reject on parse errors — validator warnings shouldn't + // block tests, matching cargo test's behavior. + if !parsed.errors.is_empty() { + let mut msg = format!("parse errors in {}:\n", file); + for e in &parsed.errors { + msg.push_str(&format!(" {}: {}\n", e.span.start, e.message)); + } + return Err(msg.into()); + } + + let putr_map = vw_htcl::putr::rewrite(&source, &parsed.document); + let signature_table = vw_htcl::signature_table(&parsed.document); + let line_index = vw_htcl::LineIndex::new(&source); + + // Pre-ship: primitive prelude + enum preludes + overload + // dispatchers. Same shape `run_htcl` uses (main.rs:1631-1667) + // and required for wrapped user code to install its procs + // correctly under the shim's `install_proc_body_wrap` machinery. + let mut _ignored: Vec = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + let mut setup_tcl: Vec = Vec::new(); + for p in vw_htcl::emit_primitive_prelude() { + setup_tcl.push(p); + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + setup_tcl.push(prelude); + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + if !dispatcher.trim().is_empty() { + setup_tcl.push(dispatcher); + } + } + + // Partition statements: `@test`-proc decls are collected as + // tests; everything else (including proc decls WITHOUT + // `@test`, `src` imports, and top-level setup) becomes setup. + let mut test_procs: Vec<(String, bool, Option, Option)> = + Vec::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + if let vw_htcl::CommandKind::Proc(proc) = &cmd.kind { + if let Some(test_attr) = proc.attribute("test") { + let Some(name) = proc.name.clone() else { + continue; + }; + let dedicated = test_attr.values.iter().any(|v| { + matches!( + v, + vw_htcl::AttributeValue::Ident { value, .. } + if value == "dedicated-eda" + ) + }); + // Per-test target override, e.g. + // `@test(dedicated-eda target="xcvp1202-vsva2785-3HP-e-S")`. + // Swaps the auto-project's `-part` for this test's + // Vivado session. Only meaningful for dedicated-eda + // tests (the validator warns otherwise); shared tests + // silently fall back to the workspace default. + let target_part_override = + extract_target_from_test_attr(test_attr); + let variant_override = + extract_variant_from_test_attr(test_attr); + test_procs.push(( + name, + dedicated, + target_part_override, + variant_override, + )); + // Test proc decls MUST also be shipped as setup — + // Vivado has to know about the proc before we call + // it. Fall through to the setup-emit below. + } + } + let lowered = vw_htcl::lower_command_with_putr_and_index( + cmd, + &source, + &signature_table, + &putr_map, + &line_index, + ); + // `rewrite_externs` — same as `vw run`. + let stripped = vw_htcl::rewrite_externs(&lowered).text; + if !stripped.trim().is_empty() { + setup_tcl.push(stripped); + } + } + + let display_path = file + .strip_prefix(ws) + .map(|p| p.to_string()) + .unwrap_or_else(|_| file.to_string()); + + let mut out = Vec::new(); + for (name, dedicated, target_part_override, variant_override) in test_procs + { + if let Some(filt) = filter { + if !name.contains(filt) { + continue; + } + } + out.push(TestCase { + display_path: display_path.clone(), + file_path: file.as_std_path().to_path_buf(), + name, + dedicated, + target_part_override, + variant_override, + setup_tcl: setup_tcl.clone(), + }); + } + Ok(out) +} + +/// Pull the `variant=` keyed value out of a `@test(...)` +/// attribute. Same shape as [`extract_target_from_test_attr`] +/// but for the variant-mode workspace path. +fn extract_variant_from_test_attr(attr: &vw_htcl::Attribute) -> Option { + for v in &attr.values { + let vw_htcl::AttributeValue::Keyed { key, value, .. } = v else { + continue; + }; + if key != "variant" { + continue; + } + return match value.as_ref() { + vw_htcl::AttributeValue::String { value, .. } => { + Some(value.clone()) + } + vw_htcl::AttributeValue::Ident { value, .. } => Some(value.clone()), + _ => None, + }; + } + None +} + +/// Pull the `target=` keyed value out of a `@test(...)` +/// attribute. Returns `None` if the attribute has no `target` +/// key or the value isn't a string/ident. +fn extract_target_from_test_attr(attr: &vw_htcl::Attribute) -> Option { + for v in &attr.values { + let vw_htcl::AttributeValue::Keyed { key, value, .. } = v else { + continue; + }; + if key != "target" { + continue; + } + return match value.as_ref() { + vw_htcl::AttributeValue::String { value, .. } => { + Some(value.clone()) + } + vw_htcl::AttributeValue::Ident { value, .. } => Some(value.clone()), + _ => None, + }; + } + None +} + +/// Run all shared-bucket tests in ONE Vivado session. First-file +/// setup lands once, then each proc-invocation is a separate eval. +#[allow(clippy::too_many_arguments)] +async fn run_shared_bucket( + ws: &Utf8Path, + tests: Vec, + summary: &mut RunSummary, + panel: &NextestPanel, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + // Shared bucket uses one Vivado process for every test in it, + // so per-test `@target(part=…)` / `@target(variant=…)` cannot + // apply — those overrides only make sense in dedicated-eda + // tests where each run gets a fresh session. The validator + // flags this at check time; here we silently fall back to the + // workspace default (or the CLI's `--part` / `--variant` + // selector if the user passed one). + let auto_project = workspace_auto_project(ws, part, variant)?; + let active_variant = resolve_active_variant_name(ws, variant)?; + let mut backend = spawn_backend( + ws, + verbose, + info_with_stack, + auto_project, + active_variant, + ) + .await?; + // Track which file's setup we've already shipped. + let mut shipped: std::collections::HashSet = + std::collections::HashSet::new(); + for test in tests { + let label = format!("{}::{}", test.display_path, test.name); + let row = panel.start(&label); + run_one_test(&mut backend, &test, summary, &mut shipped, panel, &row) + .await; + } + // Detach shutdown — see `run_dedicated_bucket` for the + // rationale; here it saves the summary block from a 10s + // Vivado-exit wait tacked onto the tail of `vw test`. + tokio::spawn(async move { + let _ = backend.shutdown().await; + }); + Ok(()) +} + +/// Run `@test(dedicated-eda)` tests, each in its own Vivado +/// process. `test_threads` caps parallelism. +#[allow(clippy::too_many_arguments)] +async fn run_dedicated_bucket( + ws: &Utf8Path, + tests: Vec, + test_threads: usize, + summary: &mut RunSummary, + panel: &NextestPanel, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, +) -> Result<(), Box> { + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + let threads = test_threads.max(1); + // Sequential for MVP — a proper semaphore-based parallel run + // would use tokio::task::JoinSet. Vivado processes each hold + // ~1-2 GB of RAM so caution around parallelism is warranted; + // parallel exec is a follow-up. + let _ = threads; + let ws_default = workspace_auto_project(ws, part, variant)?; + // Precompute variant → part for cheap per-test lookup below. + // `None` when the workspace has no variants block; a missing + // variant name gets a targeted error rather than a silent + // fallback so a typo in `@test(dedicated-eda variant=vpk20)` + // fails loud. + let ws_cfg = vw_lib::load_workspace_config(ws).ok(); + for test in tests { + // Resolution order per-test: + // 1. `variant=` → variant.part, override name from ws_default + // 2. `target=` → the literal part + // 3. no override → workspace default (whatever `part` selected) + let auto_project = if let Some(vname) = &test.variant_override { + let Some(cfg) = &ws_cfg else { + return Err(format!( + "test `{}` requested variant `{}` but the workspace \ + has no `vw.toml` — declare a `[[workspace.variants]]` \ + block first", + test.name, vname, + ) + .into()); + }; + let variant = cfg + .workspace + .select_variant(Some(vname)) + .map_err(|e| e.to_string())? + .ok_or_else(|| { + format!( + "test `{}` requested variant `{}` but the workspace \ + has no `[[workspace.variants]]` block", + test.name, vname, + ) + })?; + Some(vw_vivado::AutoProject { + name: ws_default + .as_ref() + .map(|p| p.name.clone()) + .unwrap_or_else(|| "vw_test".to_string()), + part: variant.part.clone(), + // `vw test` never persists — parallel `@test(dedicated-eda)` + // workers would trip over a shared on-disk project, and + // test scratch has no business dirtying `/target/`. + persist_dir: None, + }) + } else if let Some(p) = &test.target_part_override { + Some(vw_vivado::AutoProject { + name: ws_default + .as_ref() + .map(|p| p.name.clone()) + .unwrap_or_else(|| "vw_test".to_string()), + part: p.clone(), + persist_dir: None, + }) + } else { + ws_default.clone() + }; + // Show the test up front so the runner announces what's + // pending during Vivado boot (~10-30s per dedicated-eda + // test). Overwritten by run_one_test's "running" once + // spawn returns. + let label = format!("{}::{}", test.display_path, test.name); + let row = panel.start(&label); + // Per-test variant precedence: `@test(dedicated-eda variant=…)` + // wins over the CLI `--variant`, which in turn wins over the + // workspace default. Only meaningful when the workspace is + // variant-mode; part-mode workspaces get `None` and the + // session-scoped filter stays inert. + let active_variant = if test.variant_override.is_some() { + test.variant_override.clone() + } else { + resolve_active_variant_name(ws, variant)? + }; + let mut backend = spawn_backend( + ws, + verbose, + info_with_stack, + auto_project, + active_variant, + ) + .await?; + let mut shipped: std::collections::HashSet = + std::collections::HashSet::new(); + run_one_test(&mut backend, &test, summary, &mut shipped, panel, &row) + .await; + // Detach shutdown. Vivado's tear-down is a 10s-bounded + // graceful-exit dance (see VivadoBackend::shutdown), and + // for an in-memory test session nothing about that wait is + // load-bearing — the child process exit / kill happens + // either way. Fire-and-forget so the runner reaches the + // summary block immediately after the last test, and so + // one test's shutdown overlaps the next test's Vivado + // boot instead of serializing. If the process exits before + // the shutdown task finishes, VivadoBackend's Drop still + // SIGKILLs the child — no orphan. + tokio::spawn(async move { + let _ = backend.shutdown().await; + }); + } + Ok(()) +} + +async fn run_one_test( + backend: &mut vw_vivado::VivadoBackend, + test: &TestCase, + summary: &mut RunSummary, + shipped: &mut std::collections::HashSet, + panel: &NextestPanel, + row: &ProgressBar, +) { + let started = Instant::now(); + let label = format!("{}::{}", test.display_path, test.name); + // Ship setup once per file per session. Any error during setup + // is a hard-fail — subsequent tests in the same file would + // observe a broken state, so we bail on the whole file. + if shipped.insert(test.file_path.clone()) { + for line in &test.setup_tcl { + if let Err(e) = backend.eval(line).await { + panel.finish( + row, + &format!("{label} — setup error"), + false, + started.elapsed().as_secs_f64(), + ); + summary.failed += 1; + summary.failures.push(TestFailure { + display: label, + err: e, + }); + return; + } + } + } + // Actually invoke the test proc. + match backend.eval(&test.name).await { + Ok(_) => { + panel.finish(row, &label, true, started.elapsed().as_secs_f64()); + summary.passed += 1; + } + Err(e) => { + panel.finish(row, &label, false, started.elapsed().as_secs_f64()); + summary.failed += 1; + summary.failures.push(TestFailure { + display: label, + err: e, + }); + } + } +} + +// The RUN/PASS/FAIL line machinery lives in `crate::test_ui`, shared +// with the `vw bench` runner so the two look identical. + +async fn spawn_backend( + ws: &Utf8Path, + verbose: bool, + info_with_stack: bool, + auto_project: Option, + active_variant: Option, +) -> Result> { + let rpc_handler = vw_vivado::make_handler_with_variant( + Some(ws.as_std_path().to_path_buf()), + active_variant, + ); + // Raw byte-log per spawn — test runs may spawn multiple Vivados + // in the dedicated bucket; each gets its own timestamped log + // under target/logs/. Silent-on-error: the test runner already + // owns the output surface, and a broken target/ dir shouldn't + // fail the test. + let raw_log = vw_vivado::raw_log_path_for_workspace(ws.as_std_path()).ok(); + let backend = vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await + .map_err(|e| format!("failed to start Vivado worker: {e}"))?; + Ok(backend) +} + +/// Resolve the active variant name for a workspace given the +/// CLI's `--variant` selector. Returns `None` for workspaces with +/// no `[[workspace.variants]]` block (variant-mode is inactive). +/// Errors mirror `workspace_auto_project`'s selector-mismatch +/// checks — a workspace-mode/selector mismatch is caller error. +fn resolve_active_variant_name( + ws: &Utf8Path, + variant: Option<&str>, +) -> Result, Box> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok(None); + }; + if cfg.workspace.variants.is_empty() { + return Ok(None); + } + let selected = cfg + .workspace + .select_variant(variant) + .map_err(|e| e.to_string())?; + Ok(selected.map(|v| v.name.clone())) +} + +/// Derive the auto-project from a workspace's `[[target-parts]]` +/// or `[[workspace.variants]]` block — same rule `vw run` uses. +/// - `part` is the CLI's `--part` selector; ignored when the +/// workspace is variant-mode. +/// - `variant` is the CLI's `--variant` selector; ignored when the +/// workspace is part-mode. +/// +/// Returns: +/// - `Ok(None)` for library workspaces (neither block declared) +/// - `Ok(Some(_))` when a part / variant resolves cleanly +/// - `Err(_)` when the CLI selector is bogus, the multi-entry list +/// has no default marked, or the user passed a selector that +/// doesn't match the workspace's declared mode. +fn workspace_auto_project( + ws: &Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result, Box> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok(None); + }; + if !cfg.workspace.variants.is_empty() { + if part.is_some() { + return Err(format!( + "workspace `{}` declares `[[workspace.variants]]` — use \ + `--variant ` instead of `--part`", + cfg.workspace.name, + ) + .into()); + } + let selected = cfg + .workspace + .select_variant(variant) + .map_err(|e| e.to_string())?; + return Ok(selected.map(|v| vw_vivado::AutoProject { + name: cfg.workspace.name.clone(), + part: v.part.clone(), + // See sibling site above: `vw test` intentionally never + // persists the Vivado project. + persist_dir: None, + })); + } + if variant.is_some() { + return Err(format!( + "workspace `{}` does not declare `[[workspace.variants]]` — \ + remove `--variant` or add a variants block", + cfg.workspace.name, + ) + .into()); + } + let selected = cfg + .workspace + .select_target_part(part) + .map_err(|e| e.to_string())?; + Ok(selected.map(|p| vw_vivado::AutoProject { + name: cfg.workspace.name.clone(), + part: p.to_string(), + persist_dir: None, + })) +} + +#[derive(Default)] +struct RunSummary { + passed: usize, + failed: usize, + failures: Vec, +} + +struct TestFailure { + display: String, + /// The raw backend error kept intact so `print_summary` can + /// render it with the same colored/split treatment `vw run` + /// and the REPL apply — message in bright red, stdout in + /// default, stack in dimmed gray. + err: vw_eda::BackendError, +} + +fn print_summary(summary: &RunSummary, elapsed: std::time::Duration) { + if !summary.failures.is_empty() { + println!("\n{}\n", "failures:".red().bold()); + for f in &summary.failures { + print_failure_block(f); + } + } + crate::test_ui::print_result_line(summary.passed, summary.failed, elapsed); +} + +/// Print one failure block. Layout mirrors nextest's: +/// separator + STDOUT capture + STDERR/error + stack. Colors +/// match `vw run` / REPL — bright red for the failing message, +/// dimmed for the stack, default for captured stdout. +fn print_failure_block(f: &TestFailure) { + let bar = "─".repeat(64); + println!("{}", bar.red()); + println!(" {} {}", "✗".red().bold(), f.display.bold()); + match &f.err { + vw_eda::BackendError::Tcl { + message, + info, + stdout, + .. + } => { + if !stdout.trim().is_empty() { + println!("\n{}", "STDOUT:".bright_black().bold()); + println!("{}", stdout.trim_end()); + } + println!("\n{}", "ERROR:".red().bold()); + for line in message.lines() { + println!(" {}", line.red()); + } + if let Some(info) = info { + // Tcl's `errorInfo` embeds the error message at its + // top and appends the "while executing…" frames + // after it. Rendering the whole thing after the + // ERROR block duplicates every line of the message + // (which for `assert_file_eq` is the entire diff). + // Strip the message prefix off `info` and show + // only the frame tail; skip the section entirely + // if nothing but the message was present. + let frames = strip_message_prefix(info, message); + if !frames.trim().is_empty() { + println!("\n{}", "stack:".bright_black().bold()); + for line in frames.lines() { + println!(" {}", line.bright_black()); + } + } + } + } + other => { + println!(" {}", other.to_string().red()); + } + } + println!("{}\n", bar.red()); +} + +/// Trim Tcl's echo of the error message from the start of +/// `errorInfo`. Tcl formats `errorInfo` as +/// `\n while executing "..."\n ...`, so the frame +/// tail always starts after `message` (whose newlines and +/// whitespace we match verbatim). When the message doesn't appear +/// at the top for some reason, we fall back to returning the raw +/// info unchanged rather than silently swallowing the whole stack. +fn strip_message_prefix<'a>(info: &'a str, message: &str) -> &'a str { + let trimmed = message.trim_end(); + if let Some(rest) = info.strip_prefix(trimmed) { + // Skip any leading whitespace / newlines between the + // message and the first `while executing "..."` frame. + rest.trim_start_matches(['\n', '\r', ' ']) + } else { + info + } +} + +#[cfg(test)] +mod tests { + use super::strip_message_prefix; + + #[test] + fn strip_removes_message_echo_and_leading_whitespace() { + let message = + "assert_file_eq: files differ\n actual: a\n expected: b"; + let info = "assert_file_eq: files differ\n actual: a\n expected: b\n while executing\n\"error \\\"foo\\\"\""; + let out = strip_message_prefix(info, message); + assert!(out.starts_with("while executing"), "got {out:?}"); + } + + #[test] + fn strip_falls_back_when_prefix_missing() { + let info = "totally different\n while executing\n\"foo\""; + let out = strip_message_prefix(info, "assert_file_eq: files differ"); + assert_eq!(out, info); + } +} diff --git a/vw-cli/src/main.rs b/vw-cli/src/main.rs index ce4386b..0073d12 100644 --- a/vw-cli/src/main.rs +++ b/vw-cli/src/main.rs @@ -2,21 +2,28 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; use clap::{Parser, Subcommand, ValueEnum}; use colored::*; -use std::collections::HashSet; use std::fmt; use std::process; use vw_lib::{ - add_dependency_with_token, clear_cache, extract_hostname_from_repo_url, - generate_deps_tcl, get_access_credentials_from_netrc, init_workspace, - list_dependencies, list_testbenches, load_workspace_config, - remove_dependency, run_testbench, update_workspace_with_token, Credentials, - VersionInfo, VhdlStandard, + add_dependency_with_token, clear_cache, generate_deps_tcl, + get_access_credentials_for_repo, get_access_credentials_for_workspace, + init_workspace, list_dependencies, remove_dependency, run_testbench, + update_workspace_with_token, VersionInfo, VhdlStandard, }; +mod bench_runner; +mod cloud; +mod cloud_sync; +mod driver; +mod htcl_test; +mod parallel_load; +mod part_picker; +mod test_ui; + #[derive(Clone, Copy, Debug, ValueEnum)] enum CliVhdlStandard { #[value(name = "2008")] @@ -43,6 +50,38 @@ impl From for VhdlStandard { } } +/// Clap ValueEnum mirror of [`vw_vivado::LogLevel`]. Lives here +/// rather than in vw-vivado so the backend crate stays clap-free. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum CliLogLevel { + /// Show every block raw — including Vivado's banners, tables, + /// hierarchy summaries, and other non-diagnostic noise. The + /// same content the raw `vivado.log` receives. + Debug, + /// Show INFO+ diagnostics. NONE-block noise renders dimmed + /// (vw run) or collapsed (repl). Default. + Info, + /// Show WARNING+ diagnostics. NONE + INFO are elided. + Warning, + /// Show CRITICAL WARNING+ diagnostics. NONE + INFO + WARNING + /// are elided. + Critical, + /// Show only ERRORs. Everything else is elided. + Error, +} + +impl From for vw_vivado::LogLevel { + fn from(lvl: CliLogLevel) -> Self { + match lvl { + CliLogLevel::Debug => vw_vivado::LogLevel::Debug, + CliLogLevel::Info => vw_vivado::LogLevel::Info, + CliLogLevel::Warning => vw_vivado::LogLevel::Warning, + CliLogLevel::Critical => vw_vivado::LogLevel::Critical, + CliLogLevel::Error => vw_vivado::LogLevel::Error, + } + } +} + #[derive(Parser)] #[command(name = "vw")] #[command(about = "A VHDL workspace management tool")] @@ -51,12 +90,52 @@ struct Cli { command: Commands, } +/// What can be done with the workspace's driver. +#[derive(Subcommand)] +enum DriverCommand { + #[command(about = "Build the driver on the helios instance")] + Build { + #[arg( + long, + help = "Build on this machine instead of in a cloud environment" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to build in. Only needed when you have \ + more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying it" + )] + insecure: bool, + #[arg(long, help = "Build with optimizations")] + release: bool, + #[arg( + trailing_var_arg = true, + help = "Further arguments for cargo, e.g. `-p module`" + )] + args: Vec, + }, +} + #[derive(Subcommand)] enum Commands { #[command(about = "Initialize a new workspace")] Init { #[arg(help = "Workspace name")] name: String, + #[arg( + long, + help = "Vivado part id, e.g. xcvp1202-vsva2785-2MP-e-S. \ + If omitted and stdout is a tty, an interactive picker \ + is launched against the current Vivado install." + )] + part: Option, }, #[command(about = "Update workspace dependencies")] Update, @@ -87,23 +166,78 @@ enum Commands { }, #[command(about = "Clear all cached repositories")] Clear, + #[command(about = "Work with the workspace's driver")] + Driver { + #[command(subcommand)] + command: DriverCommand, + }, + #[command( + about = "Remove build output, on the cloud environment if there is one" + )] + Clean { + #[arg( + long, + help = "Remove this machine's build output instead of a cloud \ + environment's" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to clean. Only needed when you have \ + more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying it" + )] + insecure: bool, + }, #[command(about = "List workspace dependencies")] List, #[command(about = "Generate deps.tcl file with all dependency VHDL files")] DepsToTcl, - #[command(about = "Run testbench using NVC")] - Test { - #[arg(help = "Name of the testbench entity to run")] + #[command( + about = "Run VHDL/cosim testbenches with NVC — all in parallel, nextest-style" + )] + Bench { + #[arg( + help = "Filter to testbenches whose name contains this substring; omit to run all" + )] testbench: Option, + #[arg( + long, + help = "Run the testbenches on this machine instead of in a cloud \ + environment" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to run in. Only needed when you have \ + more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying it" + )] + insecure: bool, #[arg(long, help = "VHDL standard", default_value_t = CliVhdlStandard::Vhdl2019)] std: CliVhdlStandard, - #[arg(long, help = "List all available testbenches")] + #[arg( + long, + help = "List matching testbenches instead of running them" + )] list: bool, #[arg( long, - help = "Enable recursive search when looking for testbenches" + help = "Maximum number of testbenches to run concurrently (default: CPU count)" )] - recurse: bool, + concurrency: Option, #[arg( long, value_delimiter = ',', @@ -113,15 +247,10 @@ enum Commands { #[arg( long, value_delimiter = ',', - help = "Runtime flags to pass to NVC (comma-separated or use multiple times)", - requires = "testbench" + help = "Runtime flags to pass to NVC (comma-separated or use multiple times)" )] runtime_flags: Vec, - #[arg( - long, - help = "Build Rust library for testbench before running", - requires = "testbench" - )] + #[arg(long, help = "Build Rust library for testbench before running")] build_rust: bool, #[arg( long, @@ -129,38 +258,385 @@ enum Commands { requires = "testbench" )] scaffold: bool, + /// Internal: run exactly one testbench into this isolated nvc build + /// dir. Used by the parallel runner to fan out per-bench; when set, + /// `testbench` is an exact name (not a filter) and anodization is + /// assumed already done. + #[arg(long, hide = true, requires = "testbench")] + build_dir: Option, + }, + #[command(about = "Run an htcl script against a Vivado worker. \ + With no file, discovers `/design.htcl`.")] + Run { + #[arg(help = "Path to an .htcl source file. Omit to run the \ + workspace's `design.htcl`.")] + file: Option, + #[arg( + long, + help = "Build on this machine instead of in a cloud environment" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to build in. Only needed when you have \ + more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying \ + it. For development services fronted by a self-signed \ + certificate; this gives up any guarantee about who is on \ + the other end, and your access token is sent to whatever \ + answers." + )] + insecure: bool, + #[arg( + long, + help = "Parse and print diagnostics only; don't launch Vivado" + )] + check: bool, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select a non-default `[[target-parts]]` entry by full \ + part ID or unique substring (e.g. `--part 3HP`). \ + Mutually exclusive with `--variant`." + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a non-default `[[workspace.variants]]` entry by \ + exact name. Variants own their parts inline." + )] + variant: Option, + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity to show. `debug` shows everything \ + (including Vivado's non-diagnostic noise); INFO+ \ + filters and dims/collapses non-diagnostic output. \ + The full raw stream always lands in \ + `/target/logs/vivado-*.log`." + )] + log_level: CliLogLevel, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too \ + (WARNING / ERROR / CRITICAL always include the stack)" + )] + info_with_stack: bool, + #[arg( + long = "bunyan", + help = "Emit newline-delimited bunyan JSON log records on \ + stdout instead of the human-readable stream, for \ + piping into `looker` in CI. Vivado INFO → bunyan \ + info (30), WARNING → warn (40), CRITICAL WARNING and \ + ERROR → error (50). Non-diagnostic noise is omitted \ + unless `--log-level=debug`. vw's own status lines \ + stay on stderr, so stdout is pure JSON." + )] + bunyan: bool, + }, + #[command(about = "Launch the vw analyzer LSP server on stdio")] + Analyzer, + #[command( + about = "Interactive htcl REPL backed by a long-lived Vivado worker" + )] + Repl { + #[arg( + long, + help = "Run vivado on this machine instead of in a cloud \ + environment" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to run in. Only needed when you have \ + more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying it" + )] + insecure: bool, + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity shown in scrollback. `debug` shows \ + every block raw; INFO+ collapses non-diagnostic \ + output into a togglable placeholder." + )] + log_level: CliLogLevel, + #[arg( + long = "load", + value_name = "FILE", + conflicts_with_all = [ + "from_synth_checkpoint", + "from_place_checkpoint", + "from_route_checkpoint", + ], + help = "Source FILE into the session as soon as Vivado is up" + )] + initial_load: Option, + #[arg( + long = "from-synth-checkpoint", + conflicts_with_all = [ + "from_place_checkpoint", + "from_route_checkpoint", + ], + help = "Skip design.htcl. On boot, open \ + /target/synth/.dcp instead. Errors if the \ + checkpoint is missing." + )] + from_synth_checkpoint: bool, + #[arg( + long = "from-place-checkpoint", + conflicts_with_all = ["from_route_checkpoint"], + help = "Skip design.htcl. On boot, open \ + /target/place/.dcp instead. Errors if the \ + checkpoint is missing." + )] + from_place_checkpoint: bool, + #[arg( + long = "from-route-checkpoint", + help = "Skip design.htcl. On boot, open \ + /target/route/.dcp instead. Errors if the \ + checkpoint is missing." + )] + from_route_checkpoint: bool, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select a non-default `[[target-parts]]` entry by full \ + part ID or unique substring. Mutually exclusive with \ + `--variant`." + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a non-default `[[workspace.variants]]` entry by \ + exact name." + )] + variant: Option, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too \ + (WARNING / ERROR / CRITICAL always include the stack)" + )] + info_with_stack: bool, + }, + #[command(about = "Parse and analyze htcl. With no args, discovers the \ + workspace's module.htcl AND test/*.htcl and checks both")] + Check { + #[arg(help = "One or more .htcl source files. Empty → discover \ + from the workspace root.")] + files: Vec, + #[arg( + long, + help = "Generate IP on this machine instead of in a cloud \ + environment" + )] + local: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with = "local", + help = "Cloud environment to generate IP in. Only needed when \ + you have more than one." + )] + env: Option, + #[arg( + long, + help = "Accept the service's TLS certificate without verifying it" + )] + insecure: bool, + #[arg( + long, + value_name = "ID", + conflicts_with_all = ["all_parts", "variant", "all_variants"], + help = "Check against a specific `[[target-parts]]` entry by \ + full part ID or unique substring. Default: the workspace's \ + default part." + )] + part: Option, + #[arg( + long = "all-parts", + conflicts_with_all = ["part", "variant", "all_variants"], + help = "Check against every declared `[[target-parts]]` entry \ + instead of just the default" + )] + all_parts: bool, + #[arg( + long, + value_name = "NAME", + conflicts_with_all = ["part", "all_parts", "all_variants"], + help = "Check against a specific `[[workspace.variants]]` entry \ + by exact name. Default: the workspace's default variant." + )] + variant: Option, + #[arg( + long = "all-variants", + conflicts_with_all = ["part", "all_parts", "variant"], + help = "Check against every declared `[[workspace.variants]]` \ + entry (each variant's `.part`) instead of just the \ + default" + )] + all_variants: bool, + #[arg( + long = "ip-generate", + help = "When IP wrappers under `target/ip/` are stale relative \ + to `ip/**.htcl`, regenerate them in-process (runs \ + `vw::configure_ip` + `vw::generate_ip_stubs`, no \ + synthesis) instead of erroring. Fast path for the \ + common `edited an IP config, want to re-check` flow — \ + orders of magnitude cheaper than `vw run`, and less \ + destructive than `rm -rf target/`." + )] + ip_generate: bool, + }, + #[command(about = "Run htcl-level tests (@test procs under test/)")] + Test { + #[arg(help = "Substring filter — run only tests whose name matches")] + filter: Option, + #[arg(long, help = "List discovered tests without running them")] + list: bool, + #[arg( + long, + help = "Max concurrent dedicated-eda Vivado processes", + default_value_t = 2 + )] + test_threads: usize, + #[arg( + long, + value_name = "ID", + conflicts_with = "variant", + help = "Select the workspace-default `[[target-parts]]` entry \ + for tests without their own `@test(target=…)`; matches \ + by full ID or unique substring" + )] + part: Option, + #[arg( + long, + value_name = "NAME", + conflicts_with = "part", + help = "Select a specific `[[workspace.variants]]` entry as the \ + default for tests without their own \ + `@test(variant=…)`" + )] + variant: Option, + #[arg( + long = "log-level", + value_enum, + default_value_t = CliLogLevel::Info, + help = "Minimum severity to show during test evals. See \ + `vw run --help` for the full model." + )] + log_level: CliLogLevel, + #[arg( + long = "info-with-stack", + help = "Attach the Tcl call stack to INFO messages too" + )] + info_with_stack: bool, }, + #[command(subcommand, about = "IP-XACT tooling")] + Ip(IpCommand), + #[command( + subcommand, + name = "htcl-cmd", + about = "Generate htcl wrappers from Vivado command references" + )] + HtclCmd(HtclCmdCommand), + #[command(about = "Manage remote build environments on a vw service")] + Cloud(cloud::CloudArgs), } -/// Helper function to get access credentials for a repository URL from netrc if available -async fn get_access_credentials_for_repo( - repo_url: &str, -) -> Option { - if let Ok(hostname) = extract_hostname_from_repo_url(repo_url) { - if let Ok(Some(creds)) = get_access_credentials_from_netrc(&hostname) { - return Some(creds); - } - } - None +#[derive(Subcommand)] +enum HtclCmdCommand { + #[command( + about = "Generate an htcl wrapper from a Vivado man-page command \ + reference" + )] + Generate { + #[arg(help = "Path to a Vivado man-page file (e.g. \ + /doc/eng/man/add_files)")] + input: Utf8PathBuf, + #[arg(short, long, help = "Output file (defaults to stdout)")] + output: Option, + #[arg( + long, + help = "Command name to wrap (defaults to the input file stem)" + )] + name: Option, + #[arg( + long, + value_name = "FILE", + help = "Per-command constraint overrides (TOML)" + )] + constraints: Option, + }, } -/// Helper function to get access credentials for workspace dependencies from netrc -async fn get_access_credentials_for_workspace( - workspace_dir: &camino::Utf8Path, -) -> Option { - // Load workspace config and check if any dependencies might need authentication - if let Ok(config) = load_workspace_config(workspace_dir) { - for dep in config.dependencies.values() { - if let Some(creds) = - get_access_credentials_for_repo(&dep.repo).await - { - return Some(creds); - } - } - } - None +#[derive(Subcommand)] +enum IpCommand { + #[command(about = "Generate an htcl wrapper from an IP-XACT component")] + Generate { + #[arg(help = "Path to an IP-XACT component XML file")] + input: Utf8PathBuf, + #[arg(short, long, help = "Output file (defaults to stdout)")] + output: Option, + #[arg( + long, + help = "Include parameters whose resolve attribute is not 'user'" + )] + include_internal: bool, + #[arg( + long = "preset", + value_name = "FILE", + help = "Supplementary Vivado preset XML file (`` format). May be given multiple times. The \ + declared values are merged into `@enum(...)` lists in the \ + generated wrapper, on top of the IP-XACT `` \ + entries." + )] + presets: Vec, + #[arg( + long, + help = "Skip auto-discovery of preset files under the Vivado \ + `data/versal/ps_pmc//` tree. Use this if the \ + discovered files are wrong or you only want the explicit \ + `--preset` ones." + )] + no_auto_presets: bool, + #[arg( + long, + value_name = "FILE", + help = "Per-IP TOML overrides file. Refines XML-derived \ + dict-schemas with `@enum(...)` restrictions and \ + per-field default overrides. Silently ignored when \ + the file doesn't exist." + )] + overrides: Option, + }, } +// Netrc credential lookup moved to `vw_lib` so `vw-vivado`'s +// RPC auto-update path can reuse it. See +// `vw_lib::get_access_credentials_for_workspace`. + #[tokio::main] async fn main() { let cli = Cli::parse(); @@ -182,8 +658,11 @@ async fn main() { }); match cli.command { - Commands::Init { name } => { - if let Err(e) = init_workspace(&cwd, name.clone()) { + Commands::Init { name, part } => { + let target_part = resolve_init_target_part(part); + if let Err(e) = + init_workspace(&cwd, name.clone(), target_part.clone()) + { eprintln!("{} {e}", "error:".bright_red()); process::exit(1); } @@ -192,9 +671,17 @@ async fn main() { "✓".bright_green(), name.cyan() ); + match &target_part { + Some(p) => println!(" target part: {}", p.cyan()), + None => println!( + " {} target-parts is empty; edit vw.toml or re-run `vw init` with --part when ready.", + "note:".yellow() + ), + } } Commands::Update => { - let access_creds = get_access_credentials_for_workspace(&cwd).await; + let access_creds = + get_access_credentials_for_workspace(&cwd, false); match update_workspace_with_token(&cwd, access_creds).await { Ok(result) => { for dep in result.dependencies { @@ -233,7 +720,7 @@ async fn main() { recursive, sim_only, } => { - let access_creds = get_access_credentials_for_repo(&repo).await; + let access_creds = get_access_credentials_for_repo(&repo); match add_dependency_with_token( &cwd, repo.clone(), @@ -282,6 +769,44 @@ async fn main() { } } } + Commands::Driver { command } => { + let DriverCommand::Build { + local, + env, + insecure, + release, + args, + } = command; + match driver_build( + &cwd, + local, + env.as_deref(), + insecure, + release, + &args, + ) + .await + { + Ok(true) => {} + // A build that failed is not a vw failure; the message came + // from cargo and has already been printed. + Ok(false) => process::exit(1), + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + } + Commands::Clean { + local, + env, + insecure, + } => { + if let Err(e) = clean(&cwd, local, env.as_deref(), insecure).await { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } Commands::Clear => match clear_cache(&cwd) { Ok(cleared) => { if !cleared.is_empty() { @@ -307,9 +832,8 @@ async fn main() { if deps.is_empty() { println!("No dependencies found in workspace"); } else { - println!("Dependencies:"); - for dep in deps { - let version_info = match dep.version { + let render = |dep: &vw_lib::DependencyInfo| { + let version_info = match &dep.version { VersionInfo::Branch { branch } => { format!(" (branch: {branch})") } @@ -319,15 +843,32 @@ async fn main() { VersionInfo::Locked { commit } => { format!(" ({})", &commit[..8.min(commit.len())]) } + VersionInfo::Local => " (local)".to_string(), VersionInfo::Unknown => String::new(), }; - println!( " {} - {}{}", dep.name.cyan(), - dep.repo, + dep.source, version_info.bright_black() ); + }; + let (test_deps, regular_deps): (Vec<_>, Vec<_>) = + deps.into_iter().partition(|d| d.is_test); + if !regular_deps.is_empty() { + println!("Dependencies:"); + for dep in ®ular_deps { + render(dep); + } + } + if !test_deps.is_empty() { + if !regular_deps.is_empty() { + println!(); + } + println!("Test dependencies:"); + for dep in &test_deps { + render(dep); + } } } } @@ -348,109 +889,3298 @@ async fn main() { process::exit(1); } }, - Commands::Test { + Commands::Bench { testbench, + local, + env, + insecure, std, list, - recurse, + concurrency, ignore, runtime_flags, build_rust, scaffold, + build_dir, } => { - if list { - let bench_dir = cwd.join("bench"); - if !bench_dir.exists() { - println!("No bench dir found in {:}", bench_dir.as_str()); - } else { - let mut ignore_set: HashSet = HashSet::new(); - for ignore_pattern in ignore { - ignore_set.insert(ignore_pattern); - } - - let mist_configs = - vw_lib::sim::find_mist_configs(&bench_dir) - .unwrap_or_default(); - - match list_testbenches(&bench_dir, &ignore_set, recurse) { - Ok(testbenches) => { - if testbenches.is_empty() && mist_configs.is_empty() - { - println!( - "No testbenches found in bench directory" - ); - } else { - println!("Available testbenches:"); - for (name, config) in &mist_configs { - println!( - " {} - {} (mixed-signal: {})", - name.cyan(), - config.entity.bright_black(), - config.netlist.bright_black() - ); - } - for tb in testbenches { - println!( - " {} - {}", - tb.name.cyan(), - tb.path - .display() - .to_string() - .bright_black() - ); - } - } - } - Err(e) => { - eprintln!("{} {e}", "error:".bright_red()); - process::exit(1); - } - } + // Self-heal a fresh checkout like `vw check`. Skip the + // internal `--build-dir` subprocess (the parent runner + // already fetched — re-fetching per bench would race) and + // `--list` (pure metadata, no deps needed). + if build_dir.is_none() && !list { + ensure_workspace_deps(&cwd).await; + } + if let Some(bd) = build_dir { + // Internal single-exec mode — the parallel runner fans out one + // subprocess per bench into an isolated build dir. Run exactly + // `testbench`; anodization was already done by the runner. This + // process's stdout/stderr is captured by the runner and shown + // only if the bench fails. + let name = + testbench.expect("--build-dir requires a testbench name"); + println!("Running testbench: {}", name.cyan()); + if let Err(e) = run_testbench( + &cwd, + name, + std.into(), + true, + &runtime_flags, + build_rust, + scaffold, + &bd, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); } - } else if let Some(testbench_name) = testbench { - println!("Running testbench: {}", testbench_name.cyan()); + } else if scaffold { + // Scaffolding is a single-bench, no-simulation operation. + let name = + testbench.expect("--scaffold requires a testbench name"); match run_testbench( &cwd, - testbench_name.clone(), + name.clone(), std.into(), - recurse, + true, &runtime_flags, build_rust, - scaffold, + true, + vw_lib::BUILD_DIR, ) .await { - Ok(()) => { - if scaffold { - println!( - "{} Scaffolding generated for '{}'", - "✓".bright_green(), - testbench_name - ); - } else { - println!( - "{} Testbench '{}' completed successfully!", - "✓".bright_green(), - testbench_name - ); - println!( - "Waveform saved to: {}", - format!("{testbench_name}.fst").cyan() - ); - } - } + Ok(()) => println!( + "{} Scaffolding generated for '{}'", + "✓".bright_green(), + name + ), Err(e) => { eprintln!("{} {e}", "error:".bright_red()); process::exit(1); } } } else { - eprintln!( - "{} Must specify testbench name or use --list", - "error:".bright_red() - ); - process::exit(1); + // Parallel nextest-style runner: no positional runs every + // testbench; a positional filters by name substring. + // + // `--list` is answered from this machine's own tree even when + // the workspace builds in the cloud. It is a question about + // source, the source here is what the developer is editing, + // and a network round trip to be told the same answer would + // only be slower. + let cloud = if local || list { + None + } else { + match bench_site(env.as_deref(), insecure).await { + Ok(site) => site, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }; + + let outcome = match &cloud { + Some((session, environment)) => { + bench_runner::run_benches_remotely( + session, + environment, + testbench.as_deref(), + concurrency, + std.into(), + &ignore, + ) + .await + } + None => { + let conc = concurrency.unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + }); + bench_runner::run_benches( + &cwd, + testbench.as_deref(), + list, + conc, + std.into(), + &ignore, + ) + .await + } + }; + + if let Err(e) = outcome { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } } } + Commands::Run { + file, + local, + env, + insecure, + check, + part, + variant, + log_level, + info_with_stack, + bunyan, + } => { + // Self-heal a fresh checkout: fetch missing deps the same + // way `vw check` does before resolving `src @dep` imports. + // + // Done here even for a cloud build: the entry file is parsed and + // lowered on this machine, so `src @dep` has to resolve here too. + // The instance fetches its own copy for vivado to read. + ensure_workspace_deps(&cwd).await; + let resolved = match file { + Some(f) => f, + None => match discover_entry_file(&cwd) { + Ok(f) => f, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + }, + }; + // `--check` never launches vivado, so there is nothing to run + // remotely and no reason to make the user wait on a network call + // to find that out. + let cloud = if local || check { + None + } else { + match cloud_site(env.as_deref(), insecure).await { + Ok(site) => site, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }; + let site = match &cloud { + Some((session, environment)) => Site::Remote { + session, + environment, + }, + None => Site::Local, + }; + if let Err(e) = run_htcl( + &resolved, + site, + check, + part.as_deref(), + variant.as_deref(), + log_level.into(), + info_with_stack, + bunyan, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Analyzer => { + init_analyzer_logging(); + vw_analyzer::run_stdio().await; + } + Commands::Repl { + local, + env, + insecure, + log_level, + initial_load, + from_synth_checkpoint, + from_place_checkpoint, + from_route_checkpoint, + part, + variant, + info_with_stack, + } => { + // `--from-*-checkpoint` short-circuits every other + // load path: skip `design.htcl` auto-discovery, skip + // `--load`, and instead resolve the requested DCP + // Rust-side so we can fail with a clear error BEFORE + // spawning Vivado if the checkpoint is missing. + // Clap already gates mutual exclusivity, so at most + // one of the three is true here. + let from_stage = if from_synth_checkpoint { + Some("synth") + } else if from_place_checkpoint { + Some("place") + } else if from_route_checkpoint { + Some("route") + } else { + None + }; + let (resolved_load, initial_source) = if let Some(stage) = + from_stage + { + match resolve_checkpoint_path(&cwd, stage, variant.as_deref()) { + Ok(path) => { + // `src @vivado-cmd` first so the namespaced + // wrapper is defined — a fresh REPL boot + // hasn't loaded any modules yet, and + // calling `vivado_cmd::open_checkpoint` + // without it fires an "undefined proc" + // error. Absolute path in braces so cwd + // shifts inside the worker can't reroute. + let snippet = format!( + "src @vivado-cmd\n\ + vivado_cmd::open_checkpoint -file {{{path}}}\n", + ); + (None, Some(snippet)) + } + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + } else { + // Same `design.htcl` auto-discovery as `vw run`: + // if the user didn't pass `--load`, look for a + // `design.htcl` in the enclosing workspace and + // source it up front. Silent no-op when there's + // no workspace or no `design.htcl` — the REPL + // still boots and the user can `:load` something + // explicitly. + // Self-heal a fresh checkout: fetch missing deps + // the same way `vw check` does before the REPL + // resolves `src @dep`. + ensure_workspace_deps(&cwd).await; + let resolved = initial_load.or_else(|| { + vw_lib::find_workspace_dir(cwd.as_std_path()) + .and_then(|ws| vw_lib::find_design_file(&ws)) + }); + (resolved, None) + }; + // Cloud first, the same as `vw run`: the session drives the + // vivado where this workspace builds. Resolved before the + // alternate screen goes up, so a sync or a service problem is + // reported on an ordinary terminal rather than flashing past + // inside a TUI that is about to exit. + let worker = if local { + vw_repl::Worker::Local + } else { + match remote_worker(env.as_deref(), insecure, &part, &variant) + .await + { + Ok(worker) => worker, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }; + + if let Err(e) = vw_repl::run( + vw_repl::ReplOptions { + log_level: log_level.into(), + initial_load: resolved_load, + initial_source, + part, + variant, + info_with_stack, + }, + worker, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Check { + files, + local, + env, + insecure, + part, + all_parts, + variant, + all_variants, + ip_generate, + } => { + // Cloud first, like every other command that needs vivado. The + // check itself runs here — it is a static analysis of the source + // the developer is editing — but generating the IP wrappers it + // needs is a vivado job, and those come back afterwards. + // + // Resolved once, before anything, so a sync happens at most once + // per check rather than at each of the two places IP generation + // can be triggered from. + let cloud = if local { + None + } else { + match cloud_site(env.as_deref(), insecure).await { + Ok(site) => site, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }; + // Two shapes: + // `vw check FILE [FILE...]` — check the explicit list. + // `vw check` — discover from workspace: + // - `/module.htcl` in normal mode. + // - Every `/test/**/*.htcl` in test-mode + // (test-deps + self-injection visible so `src @` + // and `src @` both resolve). + let discovered = if files.is_empty() { + match discover_check_targets(&cwd) { + Ok(t) => t, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + } else { + files + .into_iter() + .map(|f| CheckTarget { + path: f, + include_test_deps: false, + }) + .collect() + }; + if discovered.is_empty() { + eprintln!( + "{} nothing to check — pass a file, or run from a \ + directory with a `vw.toml` that has a `design.htcl`, \ + `module.htcl`, or `test/*.htcl`", + "note:".bright_yellow(), + ); + return; + } + // Transparently fetch missing dependencies before checking + // — the way `cargo` fetches absent deps rather than erroring + // on an unresolved import. Shared with `run`/`bench`/`repl`. + ensure_workspace_deps(&cwd).await; + let mut had_errors = false; + // Upfront IP regeneration when the user asked for it AND + // wrappers are stale. This runs BEFORE the HTCL/VHDL + // checks so subsequent analyses see the fresh wrappers + // and the staleness gate at the bottom of the check no + // longer trips. Regeneration is the same in-process + // `configure_ip -generate_targets false` + + // `generate_ip_stubs` path used when VHDL diagnostics + // report missing libraries — no synthesis, no full + // Vivado run. Failure here is downgraded to a warning: + // the check will still run and surface whatever it finds + // (including the staleness error if regen didn't take). + if ip_generate { + if let Some(ws) = vw_lib::find_workspace_dir(cwd.as_std_path()) + { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + let project_dir = vw_lib::vw_project_dir(&ws); + let stale = vw_lib::project_needs_wipe( + &ws, + project_dir.as_std_path(), + &cfg.workspace.name, + ) + .unwrap_or(false); + if stale { + println!( + "{} regenerating IP wrappers \ + (--ip-generate, `ip/**.htcl` changed)…", + "note:".bright_yellow(), + ); + if let Err(e) = ensure_ip_generated( + &ws, + cloud.as_ref(), + None, + None, + vw_vivado::LogLevel::Warning, + ) + .await + { + eprintln!( + "{} IP regeneration did not complete \ + cleanly: {e}", + "warning:".yellow(), + ); + } + } + } + } + } + // conflicts_with on the clap args guarantees at most + // one non-Default source at a time. Map both flag + // families to the same PartSelector; the resolver + // decides which config block applies based on the + // workspace shape. + let part_selector = if let Some(v) = variant.as_deref() { + PartSelector::Explicit(v.to_string()) + } else if all_variants { + PartSelector::All + } else { + PartSelector::from_flags(part.as_deref(), all_parts) + }; + for target in &discovered { + let res = check_htcl_with_mode( + &target.path, + target.include_test_deps, + &part_selector, + ) + .await; + match res { + Ok(file_errs) => { + if file_errs { + had_errors = true; + } + } + Err(e) => { + had_errors = true; + eprintln!( + "{} {}: {e}", + "error:".bright_red(), + target.path, + ); + } + } + } + // VHDL static analysis (vhdl_ls) over the workspace's own + // HDL — the same checks the editor surfaces live, run in + // batch alongside the htcl check. Skipped entirely for a + // pure-htcl workspace (nothing rendered into a VHDL library). + if let Some(ws) = vw_lib::find_workspace_dir(cwd.as_std_path()) { + if vw_lib::workspace_has_vhdl(&ws, None) { + // Make sure the VHDL standard library is available — + // fetched into the dep cache on first use so a + // machine without a system rust_hdl install still + // analyzes. On failure, fall back to vhdl_lang's + // built-in search (`None`), which just skips VHDL if + // nothing is found. + let stdlib = vw_lib::ensure_vhdl_stdlib().await.ok(); + let mut vhdl_result = + vw_lib::check_vhdl(&ws, None, stdlib.as_deref()); + // A design that instantiates `entity ip.` / + // `entity xil_defaultlib.` needs Vivado-generated + // wrappers/stubs under `target/`. Whether the + // SPECIFIC referenced entity is present can't be + // judged from the library being non-empty — the BD + // wrappers populate `xil_defaultlib` while a + // standalone XCI IP's stub can still be missing — so + // we let the analyzer be the judge: if it can't + // resolve a unit within those libraries, generate the + // IP in-process (configure_ip + generate_ip_targets) + // and re-analyze. That pass is skip-gated and + // `generate_target`-only, so it's cheap; and we only + // reach it on a real resolution failure, never + // speculatively. A generation that doesn't complete + // cleanly isn't fatal — the re-analysis reports the + // true remaining state. + if matches!(&vhdl_result, Ok(diags) + if diagnostics_need_ip_generation(diags)) + { + println!( + "{} generating IP for `ip`/`xil_defaultlib` \ + (vw::configure_ip)…", + "note:".bright_yellow(), + ); + if let Err(e) = ensure_ip_generated( + &ws, + cloud.as_ref(), + None, + None, + // Terse: the check only cares about VHDL + // resolution, so suppress the Vivado + // firehose (NONE/INFO) and surface just + // Warning/CriticalWarning/Error from the + // IP-generation pass. + vw_vivado::LogLevel::Warning, + ) + .await + { + eprintln!( + "{} IP generation did not complete \ + cleanly: {e}", + "warning:".yellow(), + ); + } + vhdl_result = + vw_lib::check_vhdl(&ws, None, stdlib.as_deref()); + } + match vhdl_result { + Ok(diags) if !diags.is_empty() => { + let cwd_owned = std::env::current_dir().ok(); + let cwd_ref = cwd_owned.as_deref(); + let (mut errs, mut warns) = (0usize, 0usize); + for d in &diags { + let label = match d.severity { + vw_lib::VhdlSeverity::Error => { + errs += 1; + "error:".bright_red() + } + vw_lib::VhdlSeverity::Warning => { + warns += 1; + "warning:".yellow() + } + vw_lib::VhdlSeverity::Info => { + "info:".cyan() + } + vw_lib::VhdlSeverity::Hint => { + "hint:".cyan() + } + }; + let path = render_path(&d.file, cwd_ref); + eprintln!( + "{label} {path}:{}:{}: {}", + d.line, d.column, d.message, + ); + } + eprintln!( + "VHDL: {errs} error(s), {warns} warning(s)" + ); + if errs > 0 { + had_errors = true; + } + } + Ok(_) => {} // no VHDL, or no findings + Err(e) => { + had_errors = true; + eprintln!( + "{} vhdl check failed: {e}", + "error:".bright_red(), + ); + } + } + } + // IP-wrapper staleness. The VHDL check above verifies + // that whatever wrappers are in `target/ip/` PARSE + // and RESOLVE, but not that they match the CURRENT + // `ip/**.htcl` config. If the user edited an IP + // config (added a slot, changed a preset, whatever) + // and hasn't run `vw run`/`vw repl` since, `vw check` + // used to pass silently — the on-disk wrappers still + // reflected the pre-edit config. Compare the current + // source-set fingerprint against the manifest that + // `vw::mark_project_configured` writes on every + // successful generation; a mismatch means the design + // in `target/` is a lie about the design in source. + // + // Same helper `vw run` / `vw repl` use to decide + // whether to wipe the on-disk Vivado project, so + // there's exactly one authoritative notion of + // "wrappers match sources." + // + // Only meaningful for a project on this machine. A + // cloud check has no local vivado project — what it + // fetches is the generated VHDL, not the project that + // produced it — and the instance keeps its own project + // honest, wiping and regenerating it whenever the + // source fingerprint moves. Asking whether a directory + // holding four stub files needs a wipe would be a + // category error, and one that answers "yes" every + // time, because there is no `.xpr` in it to compare + // against. + if cloud.is_none() { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + let name = &cfg.workspace.name; + let project_dir = vw_lib::vw_project_dir(&ws); + match vw_lib::project_needs_wipe( + &ws, + project_dir.as_std_path(), + name, + ) { + Ok(true) if project_dir.exists() => { + had_errors = true; + eprintln!( + "{} IP wrappers under `target/ip/` are \ + stale — the `ip/**.htcl` config has \ + changed since the last generation. Run \ + `vw check --ip-generate` to regenerate \ + just the wrappers in-process (no \ + synthesis) and re-check.", + "error:".bright_red(), + ); + } + // Wipe-needed but the project doesn't exist + // yet is the "fresh workspace" case, which + // the VHDL block above already handles by + // calling `ensure_ip_generated` when it sees + // the missing-library diagnostics. + Ok(_) => {} + Err(e) => eprintln!( + "{} could not compare IP source fingerprint: \ + {e}", + "warning:".yellow(), + ), + } + } + } + } + if had_errors { + process::exit(1); + } + } + Commands::Test { + filter, + list, + test_threads, + part, + variant, + log_level, + info_with_stack, + } => { + if let Err(e) = htcl_test::run_htcl_tests( + &cwd, + filter, + list, + test_threads, + part.as_deref(), + variant.as_deref(), + log_level.into(), + info_with_stack, + ) + .await + { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + Commands::Ip(cmd) => match cmd { + IpCommand::Generate { + input, + output, + include_internal, + presets, + no_auto_presets, + overrides, + } => { + if let Err(e) = run_ip_generate( + &input, + output.as_deref(), + include_internal, + &presets, + no_auto_presets, + overrides.as_deref(), + ) { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }, + Commands::HtclCmd(cmd) => match cmd { + HtclCmdCommand::Generate { + input, + output, + name, + constraints, + } => { + if let Err(e) = run_htcl_cmd_generate( + &input, + output.as_deref(), + name.as_deref(), + constraints.as_deref(), + ) { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + }, + Commands::Cloud(args) => { + if let Err(e) = cloud::run(args).await { + eprintln!("{} {e}", "error:".bright_red()); + process::exit(1); + } + } + } +} + +/// Decide what part to seed a new workspace's `target-parts` with. +/// +/// Explicit `--part

` on the command line always wins. Otherwise +/// launch the interactive picker when stdout is a tty and a Vivado +/// install can be located; on non-tty, no-Vivado, or user cancel +/// (Esc), fall back to `None` and print a hint so `vw init` still +/// succeeds and the user can hand-edit `vw.toml` later. +fn resolve_init_target_part(explicit: Option) -> Option { + if let Some(part) = explicit { + return Some(part); + } + use std::io::IsTerminal; + if !std::io::stdout().is_terminal() { + eprintln!( + "{} `vw init` running non-interactively without `--part`; \ + workspace will have empty target-parts.", + "note:".yellow() + ); + return None; + } + let Some(install) = vw_lib::parts::find_vivado_install() else { + eprintln!( + "{} could not locate Vivado (`which vivado`, $XILINX_VIVADO); \ + workspace will have empty target-parts. Re-run with `--part` \ + once Vivado is on PATH.", + "note:".yellow() + ); + return None; + }; + let parts = vw_lib::parts::enumerate_parts(&install); + if parts.is_empty() { + eprintln!( + "{} Vivado install at {install} contains no plaintext parts \ + catalog; workspace will have empty target-parts.", + "note:".yellow() + ); + return None; + } + match part_picker::pick_part(&parts) { + Ok(Some(id)) => Some(id), + Ok(None) => { + eprintln!( + "{} no part picked; workspace will have empty target-parts.", + "note:".yellow() + ); + None + } + Err(e) => { + eprintln!( + "{} part picker failed: {e}; workspace will have empty target-parts.", + "warning:".yellow() + ); + None + } + } +} + +fn run_htcl_cmd_generate( + input: &Utf8Path, + output: Option<&Utf8Path>, + name: Option<&str>, + constraints_path: Option<&Utf8Path>, +) -> Result<(), String> { + let page = vw_htcl_cmd::load(input.as_std_path(), name) + .map_err(|e| format!("loading {input}: {e}"))?; + let constraints = match constraints_path { + Some(p) => vw_htcl_cmd::ConstraintsTable::load(p.as_std_path()) + .map_err(|e| format!("loading constraints: {e}"))?, + None => vw_htcl_cmd::ConstraintsTable::empty(), + }; + let opts = vw_htcl_cmd::GenerateOptions { + constraints, + ..Default::default() + }; + let text = vw_htcl_cmd::generate(&page, &opts); + match output { + Some(path) => std::fs::write(path, &text) + .map_err(|e| format!("writing {path}: {e}"))?, + None => print!("{text}"), + } + Ok(()) +} + +fn run_ip_generate( + input: &Utf8Path, + output: Option<&Utf8Path>, + include_internal: bool, + explicit_presets: &[Utf8PathBuf], + no_auto_presets: bool, + overrides_path: Option<&Utf8Path>, +) -> Result<(), String> { + let component = + vw_ip::load(input).map_err(|e| format!("loading {input}: {e}"))?; + + // Combine explicit `--preset` files with what we can auto-discover + // under Vivado's `data/versal/ps_pmc//` tree. + let mut preset_paths: Vec = explicit_presets + .iter() + .map(|p| std::path::PathBuf::from(p.as_str())) + .collect(); + if !no_auto_presets { + let discovered = + vw_ip::discover_presets(std::path::Path::new(input.as_str())); + for p in discovered { + if !preset_paths.contains(&p) { + preset_paths.push(p); + } + } + } + for p in &preset_paths { + eprintln!("{:>12} {}", "Sourcing".bright_green().bold(), p.display()); + } + let presets = if preset_paths.is_empty() { + vw_ip::PresetMap::new() + } else { + vw_ip::load_presets(&preset_paths) + .map_err(|e| format!("loading presets: {e}"))? + }; + + // Sub-proc schemas for Xilinx `structured_tcldict` parameters + // (PS_PMC_CONFIG, etc.). Empty when the component isn't a CIPS + // and doesn't have an accompanying schema tree. + let dict_schemas = + vw_ip::load_cips_dict_schemas(std::path::Path::new(input.as_str())); + for name in dict_schemas.keys() { + eprintln!( + "{:>12} schema for {name} ({} fields)", + "Loaded".bright_green().bold(), + dict_schemas[name].fields.len() + ); + } + + // Load per-IP TOML overrides. Missing file → empty overrides; + // the generator falls back to XML-only defaults everywhere. + let overrides = match overrides_path { + Some(p) => { + let ov = + vw_ip::overrides::OverridesFile::load_from(p.as_std_path()) + .map_err(|e| format!("{e}"))?; + if !ov.is_empty() { + eprintln!( + "{:>12} {} shape refinement(s) from {p}", + "Loaded".bright_green().bold(), + ov.shapes.len() + ); + } + ov + } + None => vw_ip::overrides::OverridesFile::default(), + }; + + let opts = vw_ip::GenerateOptions { + user_configurable_only: !include_internal, + overrides, + ..Default::default() + }; + let out = vw_ip::generate(&component, &presets, &dict_schemas, &opts); + match output { + Some(path) => { + let module_dir = match path.parent() { + Some(p) if !p.as_str().is_empty() => p.to_path_buf(), + _ => Utf8PathBuf::from("."), + }; + std::fs::write(path, &out.main) + .map_err(|e| format!("writing {path}: {e}"))?; + // Sibling `.htcl` files that the main module sources. + // For the split gtwiz-versal IP this is 8+5 = 13 files + // (one per intfN, quadN). For small IPs (cips, dcmac) + // the subfiles vec is empty and only main.htcl gets + // written. + for (basename, content) in &out.subfiles { + let sub_path = module_dir.join(basename); + std::fs::write(&sub_path, content) + .map_err(|e| format!("writing {sub_path}: {e}"))?; + } + if !out.subfiles.is_empty() { + eprintln!( + "{:>12} main + {} subfile(s)", + "Wrote".bright_green().bold(), + out.subfiles.len() + ); + } + // Generated IP modules need a workspace toml so `vw test`, + // `vw analyzer`, and the REPL can resolve `src @vivado-cmd` + // (and any other helpers the wrapper calls into). Seed a + // default one alongside `module.htcl` on first generation; + // never clobber an existing user-edited toml. + ensure_module_vw_toml(&module_dir)?; + // Extract `` from the source + // component.xml and write it into the module's vw.toml + // `[targets] supported` list. This lets downstream + // `vw check` catch device-family mismatches statically + // — no Vivado runtime dependency. + let targets = vw_ip::targets::extract_targets( + std::path::Path::new(input.as_str()), + ); + if !targets.supported.is_empty() + || !targets.not_supported.is_empty() + { + if let Err(e) = upsert_targets_in_vw_toml(&module_dir, &targets) + { + eprintln!( + "{} updating {}/vw.toml `[targets]`: {e}", + "warning:".bright_yellow(), + module_dir, + ); + } + } + } + None => { + // stdout mode has no way to represent multiple files; + // fall back to the concatenated single-file form so + // manual `vw ip generate ... > file.htcl` invocations + // still work. + print!("{}", out.into_single()); + } + } + Ok(()) +} + +/// Write a default `vw.toml` to `dir` if one doesn't already exist. +/// +/// Generated IP wrappers all `src @vivado-cmd` for the `ip::check` / +/// `log::error` / property-helper procs that drive their bodies, so +/// every module dir needs a workspace toml that points at the +/// vivado-cmd module. Without it, the analyzer and REPL flag +/// `undefined proc ip::check` on the first line of every freshly- +/// generated module. +/// +/// The function is idempotent: an existing `vw.toml` is left alone so +/// the user can edit it (add deps, rename the workspace) without +/// having their changes overwritten the next time `regenerate.sh` +/// runs. +/// Serialize a `[targets] supported = [...]` section into `dir/vw. +/// toml`, upserting: if the file already declares `[targets]`, +/// replace the `supported` list; otherwise append a fresh section. +/// Other sections (workspace, dependencies) are preserved verbatim. +/// +/// Uses a text-level upsert rather than a serde round-trip so +/// user-authored comments and formatting in other sections don't +/// get flattened. +fn upsert_targets_in_vw_toml( + dir: &Utf8Path, + targets: &vw_ip::targets::ExtractedTargets, +) -> Result<(), String> { + let toml_path = dir.join("vw.toml"); + let mut existing = std::fs::read_to_string(&toml_path) + .map_err(|e| format!("reading {toml_path}: {e}"))?; + // Render the new block once — same shape whether we're + // inserting or replacing. `not-supported` is written only when + // non-empty so IPs whose XML is uniformly blessed produce a + // tidy vw.toml with just `supported`. + let mut new_block = String::from("[targets]\nsupported = [\n"); + for t in &targets.supported { + new_block.push_str(&format!(" \"{t}\",\n")); + } + new_block.push_str("]\n"); + if !targets.not_supported.is_empty() { + new_block.push_str("not-supported = [\n"); + for t in &targets.not_supported { + new_block.push_str(&format!(" \"{t}\",\n")); + } + new_block.push_str("]\n"); + } + + // If a `[targets]` section already exists, replace it in place + // (from its header line up to but not including the next + // `[section]` header or EOF). Otherwise append at the bottom. + if let Some(section_start) = existing.find("[targets]") { + // Find end: the byte just before the next top-level + // section header, or EOF. + let after_hdr = section_start + "[targets]".len(); + let mut end = existing.len(); + let mut cur = after_hdr; + let bytes = existing.as_bytes(); + while cur < bytes.len() { + if bytes[cur] == b'\n' + && cur + 1 < bytes.len() + && bytes[cur + 1] == b'[' + { + end = cur + 1; + break; + } + cur += 1; + } + existing.replace_range(section_start..end, &new_block); + } else { + if !existing.ends_with('\n') { + existing.push('\n'); + } + if !existing.ends_with("\n\n") { + existing.push('\n'); + } + existing.push_str(&new_block); + } + std::fs::write(&toml_path, existing) + .map_err(|e| format!("writing {toml_path}: {e}"))?; + Ok(()) +} + +fn ensure_module_vw_toml(dir: &Utf8Path) -> Result<(), String> { + let toml_path = dir.join("vw.toml"); + if toml_path.exists() { + return Ok(()); + } + let name = dir + .canonicalize_utf8() + .ok() + .as_deref() + .and_then(|p| p.file_name()) + .or_else(|| dir.file_name()) + .unwrap_or("module") + .to_string(); + let (dep_line, note) = match discover_sibling_vivado_cmd(dir) { + Some(p) => (format!("path = \"{p}\""), None), + None => ( + "path = \"../vivado-cmd\"".to_string(), + Some("# TODO: adjust to your vivado-cmd module path"), + ), + }; + let mut content = format!( + "[workspace]\n\ + name = \"{name}\"\n\ + version = \"0.1.0\"\n\ + \n\ + [dependencies.vivado-cmd]\n" + ); + if let Some(n) = note { + content.push_str(n); + content.push('\n'); + } + content.push_str(&dep_line); + content.push('\n'); + std::fs::write(&toml_path, content) + .map_err(|e| format!("writing {toml_path}: {e}"))?; + eprintln!("{:>12} {}", "Created".bright_green().bold(), toml_path); + Ok(()) +} + +/// Walk up from `start` looking for a sibling `vivado-cmd/vw.toml`. +/// Returns the canonical absolute path to the directory if found. +/// Used by [`ensure_module_vw_toml`] to seed the dep path for +/// freshly-generated IP modules. +fn discover_sibling_vivado_cmd(start: &Utf8Path) -> Option { + let abs = start.canonicalize_utf8().ok()?; + let mut cur = abs.as_path(); + while let Some(parent) = cur.parent() { + let candidate = parent.join("vivado-cmd"); + if candidate.join("vw.toml").is_file() { + return Some(candidate); + } + cur = parent; + } + None +} + +/// Read `entry` and recursively resolve its `src` imports. Looks for +/// a `vw.toml` in the entry file's parent chain to discover the +/// workspace; falls back to an empty resolver (so relative/absolute +/// imports still work, but `@name/` imports fail with a clear error) +/// when no workspace is found. +/// +/// A [`CliObserver`] is attached so the loader's progress prints in +/// real time as `Sourcing …` / `Checking …` lines. +/// Transparently fetch missing dependencies before a workspace op — +/// the way `cargo` fetches absent deps rather than erroring on an +/// unresolved import. Cheap, offline no-op when everything is already +/// cached; only reaches the network (via the `vw update` machinery) +/// when a declared git dep isn't materialized (fresh checkout, +/// `vw clear`, or a newly-added dep). Also surgically prunes a stale +/// git lock entry left by a `repo → path` switch. Exits the process on +/// a fetch failure. Shared by `vw check` / `run` / `bench` / `repl` so +/// they all self-heal a fresh checkout the same way. +/// Resolve `/target//.dcp` for a `--from-*-checkpoint` +/// invocation and confirm it exists on disk. Errors when there's no +/// workspace, no top-entity resolution, or no DCP at the derived path. +/// +/// `stage` must be one of `"synth" | "place" | "route"` — matches +/// the on-disk convention in `~/src/htcl/vw/module.htcl` (see the +/// `checkpoint_path` writes in `vw::synth` / `vw::place` / +/// `vw::route`). +fn resolve_checkpoint_path( + cwd: &Utf8Path, + stage: &str, + variant_query: Option<&str>, +) -> Result { + let ws = + vw_lib::find_workspace_dir(cwd.as_std_path()).ok_or_else(|| { + "`--from-*-checkpoint` requires a workspace (nearest `vw.toml`)" + .to_string() + })?; + let cfg = vw_lib::load_workspace_config(&ws) + .map_err(|e| format!("failed to load vw.toml: {e}"))?; + // Resolve the active variant name the same way the auto-project + // does: caller-supplied → workspace default → None. That keeps + // `` consistent with what `vw::synth` / `vw::place` / + // `vw::route` used when they wrote the checkpoint. + let active_variant = if !cfg.workspace.variants.is_empty() { + cfg.workspace + .select_variant(variant_query) + .map_err(|e| e.to_string())? + .map(|v| v.name.clone()) + } else { + None + }; + let top = cfg + .workspace + .resolve_top(active_variant.as_deref()) + .ok_or_else(|| { + format!( + "`--from-{stage}-checkpoint` needs a top-entity: set \ + `top = \"...\"` at the workspace or variant level in \ + `{ws}/vw.toml`", + ) + })?; + let path = ws.join("target").join(stage).join(format!("{top}.dcp")); + if !path.exists() { + return Err(format!( + "checkpoint not found: {path}\n\ + hint: run `vw::{stage}` (or the full flow through it) to \ + produce the checkpoint before using `--from-{stage}-checkpoint`.", + )); + } + Ok(path) +} + +async fn ensure_workspace_deps(cwd: &Utf8Path) { + let Some(ws) = vw_lib::find_workspace_dir(cwd.as_std_path()) else { + return; + }; + // A `repo → path` switch in `vw.toml` leaves the old git pin in + // `vw.lock`, which would shadow the path dep at resolution time. + // Drop just those stale entries — surgically, so no other git dep + // gets re-resolved/bumped. + match vw_lib::prune_stale_path_deps_from_lock(&ws) { + Ok(true) => println!( + "{} updated vw.lock (a dependency changed to a path dep)", + "note:".bright_yellow(), + ), + Ok(false) => {} + Err(e) => { + eprintln!("{} could not update vw.lock: {e}", "warning:".yellow(),) + } + } + if !vw_lib::dependencies_present(&ws) { + println!("{} fetching missing dependencies…", "note:".bright_yellow()); + let creds = get_access_credentials_for_workspace(&ws, false); + if let Err(e) = update_workspace_with_token(&ws, creds).await { + eprintln!( + "{} failed to fetch dependencies: {e}", + "error:".bright_red() + ); + process::exit(1); + } + } +} + +async fn load_htcl_program( + entry: &Utf8Path, +) -> Result> { + load_htcl_program_with_mode(entry, false).await +} + +/// Same as [`load_htcl_program`] but resolves +/// `[test-dependencies]` from the entry's workspace too. Used by +/// `vw test`. Cargo-parity: only the ENTRY workspace's test-deps +/// are pulled in — transitive workspaces don't leak their own +/// test-deps into the resolver. +#[allow(dead_code)] // wired via `crate::htcl_test` +pub(crate) async fn load_htcl_program_for_test( + entry: &Utf8Path, +) -> Result> { + load_htcl_program_with_mode(entry, true).await +} + +async fn load_htcl_program_with_mode( + entry: &Utf8Path, + include_test_deps: bool, +) -> Result> { + let entry_path = std::path::Path::new(entry.as_str()).to_path_buf(); + let workspace_dir = + entry_path.parent().and_then(vw_lib::find_workspace_dir); + let mut resolver = vw_htcl::Resolver::new(); + // Load the workspace config once — its `name` field feeds both + // the progress bar's label AND the self-injection at the + // bottom of this block. + let workspace_cfg = workspace_dir + .as_deref() + .and_then(|ws| vw_lib::load_workspace_config(ws).ok()); + if let Some(ws) = workspace_dir.as_deref() { + // Transitive resolution so a library's `src @other/...` + // import works even when the consumer hasn't redeclared + // `other` in their own `vw.toml`. + if let Ok(paths) = + vw_lib::transitive_dep_cache_paths_with_test(ws, include_test_deps) + { + for (name, path) in paths { + resolver = resolver.with_dep(name, path); + } + } + // Cargo-parity self-reference: a library named `foo` can + // `src @foo/bar` to reach its own siblings without the + // user having to declare `foo` as a dep of itself. Uses + // `with_dep_if_absent` so a legitimately-declared external + // `foo` still wins. + if let Some(cfg) = &workspace_cfg { + resolver = resolver.with_dep_if_absent( + cfg.workspace.name.clone(), + ws.as_std_path().to_path_buf(), + ); + } + } + let dep_paths: Vec<(String, std::path::PathBuf)> = workspace_dir + .as_deref() + .and_then(|ws| vw_lib::transitive_dep_cache_paths(ws).ok()) + .map(|paths| paths.into_iter().collect()) + .unwrap_or_default(); + // Pick up the workspace name from `vw.toml` so the local + // (non-@dep) files' bar shows `Checking metroid` instead of + // `Checking workspace`. Falls back to the literal `workspace` + // when there's no vw.toml or no `name = "…"` field. + let workspace_label = workspace_cfg + .as_ref() + .map(|cfg| cfg.workspace.name.clone()) + .unwrap_or_else(|| "workspace".to_string()); + let observer = std::sync::Arc::new( + parallel_load::MultiProgressObserver::new(dep_paths, workspace_label), + ); + let obs_for_load: std::sync::Arc = + observer.clone(); + let result = parallel_load::load_parallel( + &entry_path, + std::sync::Arc::new(resolver), + obs_for_load, + std::collections::HashMap::new(), + ) + .await; + observer.finish(); + Ok(result?) +} + +/// `entry`. Used by `check_htcl` to pre-flight `src @` +/// imports and produce spanned diagnostics before the loader's +/// hard-abort path fires. Returns an empty set when `entry` isn't +/// inside a workspace or the dep cache can't be read — the caller +/// treats empty as "skip the check", matching the validator's own +/// short-circuit. +#[allow(dead_code)] // legacy entry — new callers use `_with_mode` +fn collect_dep_names(entry: &Utf8Path) -> std::collections::HashSet { + collect_dep_names_with_mode(entry, false) +} + +fn collect_dep_names_with_mode( + entry: &Utf8Path, + include_test_deps: bool, +) -> std::collections::HashSet { + let entry_path = std::path::Path::new(entry.as_str()); + let Some(ws) = entry_path.parent().and_then(vw_lib::find_workspace_dir) + else { + return std::collections::HashSet::new(); + }; + let Ok(paths) = + vw_lib::transitive_dep_cache_paths_with_test(&ws, include_test_deps) + else { + return std::collections::HashSet::new(); + }; + let mut names: std::collections::HashSet = + paths.into_keys().collect(); + // Mirror `load_htcl_program`'s self-injection: a library named + // `foo` can `src @foo/bar` at check time even though `foo` + // isn't in its own [dependencies]. + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + names.insert(cfg.workspace.name); + } + names +} + +fn init_analyzer_logging() { + // Silent by default — see the matching note in vw-analyzer's main. + let filter = tracing_subscriber::EnvFilter::try_from_env("VW_ANALYZER_LOG") + .unwrap_or_else(|_| "vw_analyzer=off".into()); + let _ = tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_ansi(false) + .with_env_filter(filter) + .try_init(); +} + +/// Run parse + signature validation on `file`. Returns `Ok(true)` +/// if any error-severity diagnostics were reported, `Ok(false)` for +/// clean. Warnings don't flip the return value but still print. +#[allow(dead_code)] // legacy entry — new callers use `_with_mode` +async fn check_htcl( + file: &camino::Utf8Path, +) -> Result> { + check_htcl_with_mode(file, false, &PartSelector::Default).await +} + +/// One entry in the `vw check` (no-args) discovery result. Test +/// files get `include_test_deps = true` so the validator can see +/// `@` and `@` imports without the user +/// spelling them out in the vw.toml `[dependencies]` section. +struct CheckTarget { + path: Utf8PathBuf, + include_test_deps: bool, +} + +/// Discover files to check when the user runs `vw check` from a +/// workspace directory with no explicit file list. Picks up +/// `/design.htcl` (project entry) and `/module.htcl` +/// (library entry) — either or both may be present — plus every +/// `/test/**/*.htcl` (checked in test-mode). +/// +/// Errors when we can't find the enclosing workspace at all — +/// otherwise returns an empty vec, letting the caller print +/// "nothing to check" without treating it as a hard failure. +/// Resolve `--part` / `--variant` for this workspace, showing whatever the +/// resolution had to say for itself. +/// +/// The resolving lives in `vw-vivado` because an agent running a build on an +/// instance has to do exactly the same thing; all that is left here is +/// deciding where the notes go, which on a developer's terminal is the +/// terminal. +pub(crate) fn resolve_workspace_selection( + ws: &camino::Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result<(Option, Option), String> { + let selection = vw_vivado::resolve_workspace_selection(ws, part, variant)?; + for note in &selection.notes { + eprintln!("{} {note}", "info:".cyan()); + } + Ok((selection.auto_project, selection.active_variant)) +} + +/// Locate the workspace's `design.htcl` for a bare `vw run` with +/// no file argument. Errors with a targeted message when the +/// workspace doesn't have one — the user's next move is either +/// `vw run ` or creating a `design.htcl`. +fn discover_entry_file( + cwd: &Utf8Path, +) -> Result> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + vw_lib::find_design_file(&ws).ok_or_else(|| { + format!( + "no `design.htcl` in {ws}; pass a file explicitly \ + (`vw run `) or create a `design.htcl`", + ) + .into() + }) +} + +fn discover_check_targets( + cwd: &Utf8Path, +) -> Result, Box> { + let ws = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("not in a vw workspace (no vw.toml in the parent chain)")?; + let mut targets = Vec::new(); + if let Some(design) = vw_lib::find_design_file(&ws) { + targets.push(CheckTarget { + path: design, + include_test_deps: false, + }); + } + let module = ws.join("module.htcl"); + if module.is_file() { + targets.push(CheckTarget { + path: module, + include_test_deps: false, + }); + } + // `ip/module.htcl` is sourced implicitly at runtime by + // `vw::configure_ip` (which walks `/ip/**` via + // `list_ip_htcl_files`), so static discovery didn't include + // it and typos in ip-tree procs never surfaced via + // `vw check`. Add it explicitly so bare `vw check` catches + // the same class of errors the LSP does. + let ip_module = ws.join("ip/module.htcl"); + if ip_module.is_file() { + targets.push(CheckTarget { + path: ip_module, + include_test_deps: false, + }); + } + for path in vw_lib::list_htcl_tests(&ws)? { + let Ok(path) = Utf8PathBuf::from_path_buf(path) else { + continue; + }; + targets.push(CheckTarget { + path, + include_test_deps: true, + }); + } + Ok(targets) +} + +/// How `vw check` decides which `[[target-parts]]` entries to +/// evaluate. Constructed from the CLI's `--part` / `--all-parts` +/// flags; consumed by [`check_htcl_with_mode`] to iterate the +/// selected parts through the compatibility check. +#[derive(Debug, Clone)] +enum PartSelector { + /// No flag → use the workspace's default part. + Default, + /// `--part ` → use the matching entry. + Explicit(String), + /// `--all-parts` → iterate every entry. + All, +} + +impl PartSelector { + fn from_flags(part: Option<&str>, all_parts: bool) -> Self { + if all_parts { + Self::All + } else if let Some(p) = part { + Self::Explicit(p.to_string()) + } else { + Self::Default + } + } + + /// Resolve to a concrete list of part strings against `ws_info`. + /// Handles both the legacy `[[target-parts]]` shape and the + /// new `[[workspace.variants]]` shape (variants own their + /// parts inline; the compat check treats them the same way + /// once the part is extracted). Returns an empty vec for a + /// library workspace (neither block populated) — caller + /// treats that as "no compat check." + fn resolve<'a>( + &self, + ws_info: &'a vw_lib::WorkspaceInfo, + ) -> std::result::Result, String> { + if !ws_info.variants.is_empty() { + return self.resolve_variant_mode(ws_info); + } + if ws_info.target_parts.is_empty() { + return Ok(Vec::new()); + } + match self { + Self::Default => Ok(ws_info + .default_target_part() + .map_err(|e| e.to_string())? + .into_iter() + .collect()), + Self::Explicit(q) => Ok(ws_info + .select_target_part(Some(q)) + .map_err(|e| e.to_string())? + .into_iter() + .collect()), + Self::All => Ok(ws_info + .target_parts + .iter() + .map(|p| p.part.as_str()) + .collect()), + } + } + + fn resolve_variant_mode<'a>( + &self, + ws_info: &'a vw_lib::WorkspaceInfo, + ) -> std::result::Result, String> { + match self { + Self::Default => Ok(ws_info + .default_variant() + .map_err(|e| e.to_string())? + .map(|v| v.part.as_str()) + .into_iter() + .collect()), + Self::Explicit(q) => Ok(ws_info + .select_variant(Some(q)) + .map_err(|e| e.to_string())? + .map(|v| v.part.as_str()) + .into_iter() + .collect()), + Self::All => { + Ok(ws_info.variants.iter().map(|v| v.part.as_str()).collect()) + } + } + } +} + +async fn check_htcl_with_mode( + file: &camino::Utf8Path, + include_test_deps: bool, + part_selector: &PartSelector, +) -> Result> { + // Pre-flight: run the validator's src-import check on the entry + // file BEFORE handing to `load_htcl_program`, which would + // hard-abort on the first unresolved `src @` with a bare + // non-span error. The pre-flight produces the same spanned + // diagnostics the LSP shows, so `vw check` and the editor agree + // on where the missing dep is and how to fix it. Only kicks in + // when the workspace has a `vw.toml` (otherwise dep-names is + // empty and the check is a no-op). + let dep_names = collect_dep_names_with_mode(file, include_test_deps); + if !dep_names.is_empty() { + let entry_text = std::fs::read_to_string(file.as_str())?; + let entry_parsed = vw_htcl::parse(&entry_text); + let pre_diags = vw_htcl::validate_with_all_extras_and_vars( + &entry_parsed.document, + &entry_text, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + &std::collections::HashSet::new(), + &dep_names, + ); + let src_errs: Vec<_> = pre_diags + .iter() + .filter(|d| { + d.severity == vw_htcl::Severity::Error + && d.message.starts_with("unknown src module") + }) + .collect(); + if !src_errs.is_empty() { + let idx = vw_htcl::LineIndex::new(&entry_text); + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let display_path = + render_path(std::path::Path::new(file.as_str()), cwd); + for d in &src_errs { + let (start, _) = idx.range(d.span); + eprintln!( + "{} {display_path}:{}:{}: {}", + "error:".bright_red(), + start.line + 1, + start.character + 1, + d.message, + ); + } + eprintln!("{file}: {} error(s), 0 warning(s)", src_errs.len()); + return Ok(true); + } + } + + let program = if include_test_deps { + load_htcl_program_for_test(file).await? + } else { + load_htcl_program(file).await? + }; + let parsed = vw_htcl::parse(&program.source); + let validator_diags = vw_htcl::validate(&parsed.document, &program.source); + + // Build a per-file `LineIndex` lazily so we only pay for files + // that actually have diagnostics. Keyed by `file_index`. + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let mut indices: std::collections::HashMap = + std::collections::HashMap::new(); + + let mut error_count = 0usize; + let mut warning_count = 0usize; + let mut emit = |severity: Option, + message: &str, + span: vw_htcl::Span| { + let level = match severity { + None | Some(vw_htcl::Severity::Error) => { + error_count += 1; + "error:".bright_red() + } + Some(vw_htcl::Severity::Warning) => { + warning_count += 1; + "warning:".bright_yellow() + } + }; + // Map the span back to its originating file's line/col so the + // displayed location is the file the user actually wrote — not + // the flat dependency-concatenated source the loader produced. + let (display_path, line, col) = match program.locate_span(span) { + Some((idx, file_span)) => { + let loaded = &program.files[idx]; + let index = indices + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(&loaded.source)); + let (start, _) = index.range(file_span); + ( + render_path(&loaded.path, cwd), + start.line + 1, + start.character + 1, + ) + } + None => (file.to_string(), 0, 0), + }; + eprintln!("{} {display_path}:{line}:{col}: {message}", level); + }; + + for err in &parsed.errors { + emit(None, &err.message, err.span); + } + for d in &validator_diags { + emit(Some(d.severity), &d.message, d.span); + } + + // Target-compatibility check. Iterates over the parts the + // selector picked (default, explicit `--part`, or every + // `[[target-parts]]` entry with `--all-parts`). Library + // workspaces (no target parts) skip silently. + let entry_path = std::path::Path::new(file.as_str()); + if let Some(ws) = entry_path.parent().and_then(vw_lib::find_workspace_dir) { + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + let parts = match part_selector.resolve(&cfg.workspace) { + Ok(p) => p, + Err(e) => { + eprintln!("{} {e}", "error:".bright_red()); + error_count += 1; + Vec::new() + } + }; + if !parts.is_empty() { + let dep_targets = vw_lib::collect_dep_targets(&ws); + for (dep, err) in &dep_targets.errors { + eprintln!( + "{} bad `[targets]` pattern in dep `{dep}`: {err}", + "warning:".bright_yellow(), + ); + warning_count += 1; + } + for target_part in &parts { + let mismatches = vw_lib::check_target_compatibility( + Some(target_part), + &dep_targets, + ); + for m in &mismatches { + match m.kind { + vw_lib::TargetMismatchKind::NotSupported => { + eprintln!( + "{} target-part `{}` matches dep `{}`'s \ + `not-supported` list — Xilinx has \ + attested the IP is not usable on this \ + part ({})", + "error:".bright_red(), + m.target_part, + m.dep, + target_mismatch_families_hint(m), + ); + error_count += 1; + } + vw_lib::TargetMismatchKind::Unblessed => { + eprintln!( + "{} target-part `{}` isn't blessed by \ + dep `{}` ({}); the IP may still work \ + but Xilinx hasn't blessed the \ + combination", + "warning:".bright_yellow(), + m.target_part, + m.dep, + target_mismatch_families_hint(m), + ); + warning_count += 1; + } + } + } + } + } + } + } + + if error_count > 0 || warning_count > 0 { + eprintln!("{file}: {error_count} error(s), {warning_count} warning(s)"); + } + Ok(error_count > 0) +} + +/// Render `path` relative to `cwd` when it sits underneath, otherwise +/// fall back to the absolute path. Keeps diagnostic locations short +/// and click-through-able in editors / terminals. +fn render_path( + path: &std::path::Path, + cwd: Option<&std::path::Path>, +) -> String { + if let Some(cwd) = cwd { + if let Ok(rel) = path.strip_prefix(cwd) { + return rel.display().to_string(); + } + } + path.display().to_string() +} + +/// Human-readable summary of what a dep declared in its +/// `[targets]` block. Splits blessed vs. banned families so a dep +/// with only a `not-supported` list (e.g. clk-wizard v1.0, where +/// every entry is `Not-Supported`) doesn't get misreported as +/// "declared families: versal" — the versal families are BANNED +/// there, not blessed. +fn target_mismatch_families_hint(m: &vw_lib::TargetMismatch) -> String { + match ( + m.supported_families.is_empty(), + m.not_supported_families.is_empty(), + ) { + (true, true) => { + "no `[targets]` families declared — the dep has patterns \ + but none carry family names" + .to_string() + } + (false, true) => { + format!("blessed families: {}", m.supported_families.join(", "),) + } + (true, false) => { + format!( + "no blessed families — only `not-supported` entries for {}", + m.not_supported_families.join(", "), + ) + } + (false, false) => { + format!( + "blessed families: {}; also `not-supported` entries for {}", + m.supported_families.join(", "), + m.not_supported_families.join(", "), + ) + } + } +} + +/// For every monomorphized generic encountered while walking `ty` +/// (recursing through user-newtype underlyings), emit its repr to +/// the backend exactly once. Dedup is owned by the caller so +/// repeated invocations across signatures don't re-ship the same +/// proc. +/// Turn a sequence of [`vw_repl::highlight::Piece`]s (a repr-line +/// highlight result) into an ANSI-colored string using the same +/// palette the REPL's ratatui renderer uses. +/// +/// Emits raw 24-bit truecolor escapes (`\x1b[38;2;R;G;Bm`) +/// directly instead of going through `colored`'s `.truecolor()` +/// — `colored` 2.0 downgrades RGB to the nearest ANSI-16 code +/// unless `COLORTERM` explicitly announces truecolor, which +/// happens to map our REPL palette to shades that read as grey +/// (e.g. RGB(120, 200, 120) → `\e[90m` bright_black). The REPL's +/// ratatui backend emits raw truecolor unconditionally and looks +/// correct on every modern terminal; matching its behavior here +/// keeps `vw run` visually aligned with the REPL. +/// +/// Suppression on `NO_COLOR` / non-TTY stdout is preserved via +/// `colored`'s global gate. +fn ansi_from_pieces(pieces: &[vw_repl::highlight::Piece]) -> String { + use vw_repl::highlight::StyleKind; + let colorize = colored::control::SHOULD_COLORIZE.should_colorize(); + let mut out = String::new(); + for p in pieces { + if !colorize { + out.push_str(&p.text); + continue; + } + // Palette constants mirror the ratatui RGB values in + // `vw-repl/src/highlight.rs::{key_style, variant_style, + // scalar_style}` and the DIM modifier on `punct_style`. + // Keep the two backends in sync — if the ratatui palette + // changes, this needs the same edit. + let escape = match p.kind { + StyleKind::Plain => None, + StyleKind::Key => Some("\x1b[38;2;80;150;255m"), + StyleKind::Variant => Some("\x1b[38;2;100;200;200m"), + StyleKind::Punct => Some("\x1b[2m"), + StyleKind::Scalar => Some("\x1b[38;2;120;200;120m"), + }; + match escape { + None => out.push_str(&p.text), + Some(prefix) => { + out.push_str(prefix); + out.push_str(&p.text); + out.push_str("\x1b[0m"); + } + } + } + out +} + +/// Encode a [`vw_vivado::Severity`] as a `u8` matching the ladder's +/// `PartialOrd`. Used for `AtomicU8::fetch_max` on the "worst seen +/// during this session" counter — atomic comparison over the raw +/// integer is thread-safe without needing a mutex, and the mapping +/// is a static one-liner rather than a lookup table. +fn severity_as_u8(s: vw_vivado::Severity) -> u8 { + match s { + vw_vivado::Severity::None => 0, + vw_vivado::Severity::Info => 1, + vw_vivado::Severity::Warning => 2, + vw_vivado::Severity::CriticalWarning => 3, + vw_vivado::Severity::Error => 4, + } +} + +/// Render one classified [`vw_vivado::Block`] to stdout, applying the +/// user-selected log level: +/// +/// - `Block::Diagnostic { severity, .. }` — passes through +/// [`render_chunk`] with the block joined back into a single +/// chunk, provided the severity clears `log_level`. +/// - `Block::None { .. }` — non-diagnostic Vivado stdout (banners, +/// `VHDL Output written …`, block-design `Slave segment …` chatter, +/// a design's own `puts`/`putr`). At `LogLevel::Debug` it renders +/// raw via the Stdout path (every byte). At `Info` it dims each +/// line so users still see noise flowed through without it +/// dominating. At `Warning`+ it's elided entirely — the caller +/// explicitly asked for terse output (only Warning/Critical/Error), +/// e.g. `vw check`'s IP pre-pass, where this stdout is pure barf. +fn render_block( + block: &vw_vivado::Block, + log_level: vw_vivado::LogLevel, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, +) { + use colored::Colorize; + match block { + vw_vivado::Block::Diagnostic { severity, lines } => { + if !log_level.allows(*severity) { + return; + } + let kind = vw_vivado::stream_kind_for(*severity); + let joined = lines.join("\n"); + render_chunk(kind, &joined, proc_table, origin, input_file); + } + vw_vivado::Block::None { lines } => match log_level { + vw_vivado::LogLevel::Debug => { + // Raw pass-through — same style as if it hadn't been + // block-grouped at all. + let joined = lines.join("\n"); + render_chunk( + vw_vivado::StreamKind::Stdout, + &joined, + proc_table, + origin, + input_file, + ); + } + vw_vivado::LogLevel::Info => { + // Dim so users still see noise flowed through. + use std::io::Write; + let mut out = std::io::stdout().lock(); + for line in lines { + let _ = writeln!(out, "{}", line.dimmed()); + } + let _ = out.flush(); + } + // Warning/Critical/Error: terse — drop non-diagnostic + // output entirely. + _ => {} + }, + } +} + +/// Stream-sink rendering for `vw run`. Mirrors the REPL's +/// scrollback colors + stack-frame rewriting so both surfaces +/// look the same: +/// +/// - **Stream kind → ANSI color/prefix** +/// - `Error` → `✗ ` red bold +/// - `Warning` → `⚠ ` orange (Rgb 255,140,0) +/// - `Info` → `· ` dark gray +/// - `Stdout` → no prefix, no color +/// +/// - **Stack-frame rewriting**: lines matching ` at :N +/// in ::proc` are mapped to the real htcl source via +/// [`vw_repl::resolve_stack_frames_with`] + `proc_table`. +/// Adjacent frames pointing at the same proc collapse to one. +/// +/// - **Origin tagging**: warnings/errors that arrive without an +/// `\n at …` trace get one appended pointing at the +/// currently-executing top-level statement (`origin`). Mirrors +/// the REPL's `tag_streamed_message` — Vivado C++ paths +/// bypass `::common::send_msg_id` and emit traceless messages +/// we'd otherwise have no anchor for. +fn render_chunk( + kind: vw_vivado::StreamKind, + chunk: &str, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, +) { + use colored::Colorize; + use std::io::Write; + // Drop a single trailing newline, resolve stack frames to real + // htcl source, and tag traceless warnings/errors with the current + // origin. Shared with the bunyan emitter via this helper so both + // surfaces show identical, source-resolved message text. + let tagged = + resolve_diagnostic_text(kind, chunk, proc_table, origin, input_file); + if tagged.is_empty() { + return; + } + let prefix = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => "✗ ", + vw_vivado::StreamKind::Warning => "⚠ ", + vw_vivado::StreamKind::Info => "· ", + vw_vivado::StreamKind::Stdout => "", + }; + let mut out = std::io::stdout().lock(); + for (i, line) in tagged.lines().enumerate() { + let leading = if i == 0 || prefix.is_empty() { + prefix + } else { + " " + }; + let styled_prefix: String = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => { + leading.red().bold().to_string() + } + vw_vivado::StreamKind::Warning => { + leading.truecolor(255, 140, 0).bold().to_string() + } + vw_vivado::StreamKind::Info => leading.bright_black().to_string(), + vw_vivado::StreamKind::Stdout => leading.to_string(), + }; + let styled_line: String = match kind { + vw_vivado::StreamKind::Error + | vw_vivado::StreamKind::CriticalWarning => line.red().to_string(), + vw_vivado::StreamKind::Warning => { + line.truecolor(255, 140, 0).to_string() + } + vw_vivado::StreamKind::Info => line.bright_black().to_string(), + // Stdout is where the shim's `puts` output lands, + // including compiler-emitted enum reprs like + // CPM_PCIE0_MODES Scalar(None) + // CONFIG Nested( + // … + // ) + // Route it through the REPL's shape-based highlighter + // so `vw run` and the REPL scrollback look identical + // for repr-shaped lines. Non-repr text (plain `puts` + // messages, error messages that reach Stdout, etc.) + // fails the parse and falls through to the raw line. + // The `colored` crate that underpins ansi_from_pieces + // already respects NO_COLOR + tty-detection, so the + // integration inherits the standard "quiet when piped + // or NO_COLOR is set" behavior for free. + vw_vivado::StreamKind::Stdout => { + match vw_repl::highlight::highlight_line_pieces(line) { + Some(pieces) => ansi_from_pieces(&pieces), + None => line.to_string(), + } + } + }; + let _ = writeln!(out, "{styled_prefix}{styled_line}"); + } + let _ = out.flush(); +} + +/// Resolve a diagnostic chunk's `at :N in ::proc` frames back to +/// real htcl source and tag traceless warnings/errors with the current +/// origin. Shared by the human renderer ([`render_chunk`]) and the +/// bunyan emitter ([`emit_bunyan_block`]) so both surfaces show +/// identical, source-resolved text. Trims a single trailing newline; +/// returns `""` for an otherwise-empty chunk. +fn resolve_diagnostic_text( + kind: vw_vivado::StreamKind, + chunk: &str, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, +) -> String { + let trimmed = chunk.trim_end_matches('\n'); + if trimmed.is_empty() { + return String::new(); + } + let resolved = vw_repl::resolve_stack_frames_with( + trimmed, + |name| proc_table.get(name).cloned(), + input_file, + ); + // Tag traceless warnings/errors with the currently-executing + // statement's origin — Vivado's C++ IP-Flow paths bypass + // `::common::send_msg_id` and arrive without an `at …` frame. + match kind { + vw_vivado::StreamKind::Warning | vw_vivado::StreamKind::Error + if !resolved.contains("\n at ") => + { + match origin { + Some(o) => { + let path = o + .file + .as_deref() + .map(vw_repl::display_path) + .unwrap_or_else(|| { + input_file + .map(vw_repl::display_path) + .unwrap_or_else(|| "".into()) + }); + format!("{resolved}\n at {path}:{}", o.line) + } + None => resolved, + } + } + _ => resolved, + } +} + +/// Emit one classified [`vw_vivado::Block`] as a bunyan JSON record on +/// stdout (the `--bunyan` path). Diagnostic blocks map their severity to +/// a bunyan level and honor the `--log-level` filter exactly like the +/// human renderer; non-diagnostic NONE blocks only enter the structured +/// stream at the `--log-level=debug` escape hatch (they always remain in +/// the raw `vivado-*.log`). Filtered-out blocks are a no-op. +fn emit_bunyan_block( + block: &vw_vivado::Block, + log_level: vw_vivado::LogLevel, + proc_table: &std::collections::HashMap, + origin: Option<&vw_repl::Origin>, + input_file: Option<&std::path::Path>, + hostname: &str, + pid: u32, +) { + match block { + vw_vivado::Block::Diagnostic { severity, lines } => { + if !log_level.allows(*severity) { + return; + } + let kind = vw_vivado::stream_kind_for(*severity); + let joined = lines.join("\n"); + let msg = resolve_diagnostic_text( + kind, &joined, proc_table, origin, input_file, + ); + if msg.is_empty() { + return; + } + emit_bunyan_line( + bunyan_level_for(*severity), + "vivado", + Some(bunyan_severity_label(*severity)), + &msg, + hostname, + pid, + ); + } + vw_vivado::Block::None { lines } => { + if !matches!(log_level, vw_vivado::LogLevel::Debug) { + return; + } + let msg = lines.join("\n"); + if msg.trim().is_empty() { + return; + } + emit_bunyan_line( + bunyan_level_for(vw_vivado::Severity::None), + "vivado", + Some(bunyan_severity_label(vw_vivado::Severity::None)), + &msg, + hostname, + pid, + ); + } + } +} + +/// Write a single newline-delimited bunyan record to stdout. `serde_json` +/// handles all escaping, so an embedded quote or newline in `msg` can't +/// break the one-object-per-line protocol looker parses. Every required +/// bunyan field is present (`v`, `name`, `hostname`, `pid`, `level`, +/// `time`, `msg`); the original Vivado severity rides along in a +/// non-standard `severity` field so a viewer can still distinguish +/// CRITICAL WARNING from ERROR (both collapse to level 50). +fn emit_bunyan_line( + level: u8, + component: &str, + severity: Option<&str>, + msg: &str, + hostname: &str, + pid: u32, +) { + use std::io::Write; + let time = + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let record = + bunyan_record(level, component, severity, msg, hostname, pid, &time); + let mut out = std::io::stdout().lock(); + let _ = writeln!(out, "{record}"); + let _ = out.flush(); +} + +/// Build the bunyan record value. Pure (timestamp injected) so the field +/// set and JSON escaping are unit-testable without touching stdout or the +/// clock. `emit_bunyan_line` wraps it with `chrono::Utc::now()`. +fn bunyan_record( + level: u8, + component: &str, + severity: Option<&str>, + msg: &str, + hostname: &str, + pid: u32, + time: &str, +) -> serde_json::Value { + let mut record = serde_json::json!({ + "v": 0, + "name": "vw", + "hostname": hostname, + "pid": pid, + "level": level, + "component": component, + "time": time, + "msg": msg, + }); + if let Some(sev) = severity { + record["severity"] = serde_json::Value::from(sev); + } + record +} + +/// Map a Vivado [`vw_vivado::Severity`] to a bunyan numeric level: +/// INFO → 30, WARNING → 40, CRITICAL WARNING and ERROR → 50, and +/// non-diagnostic NONE → 20 (debug). +fn bunyan_level_for(severity: vw_vivado::Severity) -> u8 { + match severity { + vw_vivado::Severity::None => 20, + vw_vivado::Severity::Info => 30, + vw_vivado::Severity::Warning => 40, + vw_vivado::Severity::CriticalWarning | vw_vivado::Severity::Error => 50, + } +} + +/// The original Vivado severity as a stable lowercase label, preserved in +/// the bunyan `severity` field since CRITICAL WARNING and ERROR share +/// level 50. +fn bunyan_severity_label(severity: vw_vivado::Severity) -> &'static str { + match severity { + vw_vivado::Severity::None => "none", + vw_vivado::Severity::Info => "info", + vw_vivado::Severity::Warning => "warning", + vw_vivado::Severity::CriticalWarning => "critical-warning", + vw_vivado::Severity::Error => "error", + } +} + +async fn ship_generic_reprs( + backend: &mut dyn vw_eda::EdaBackend, + ty: &vw_htcl::TypeExpr, + types: &std::collections::HashMap, + emitted: &mut std::collections::HashSet, +) -> Result<(), Box> { + // TypeExpr::Qualified appears only on overloaded-handler + // first-args; the validator forbids it anywhere else, and + // codegen doesn't need a repr for it. + if matches!(ty, vw_htcl::TypeExpr::Qualified { .. }) { + return Ok(()); + } + let emission = vw_htcl::emit_repr_with_types(ty, types); + for p in &emission.procs { + // The procs are emitted in dependency order; the body of + // each instantiation may reference earlier ones in the + // same emission, so we ship them sequentially through + // the same eval channel. + if emitted.insert(p.clone()) { + backend.eval(p).await?; + } + } + Ok(()) +} + +/// Mirror of `vw-repl/src/lower.rs::overload_specialization_mangle`. +/// If `cmd` is a top-level `proc` whose name is an overload public +/// name AND whose first arg is a qualified-variant annotation, +/// return the mangled internal name to lower it under. Keeps +/// `vw run` in step with the REPL's specialization-rerouting. +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +/// Thin loader wrapper: resolve the entry's `src` imports into one +/// flattened program, then hand it to [`run_loaded_program`]. +/// Work out whether the testbenches belong on a cloud environment, and get +/// the environment ready for them. +/// +/// The same shape as `vw run`'s: cloud first, synchronized before anything +/// runs, and a service that cannot be reached falls back to this machine with +/// a warning rather than a failure. +async fn bench_site( + named: Option<&str>, + insecure: bool, +) -> Result, Box> { + let session = cloud::Session::from_env(insecure)?; + + let environment = match cloud::pick_environment(&session, named).await { + Ok(environment) => environment, + Err(e) if cloud::Session::unreachable(&e) => { + eprintln!( + "{} no vw service reachable ({e}); running the testbenches on \ + this machine", + "warning:".yellow(), + ); + return Ok(None); + } + Err(e) => return Err(e.into()), + }; + + cloud::sync_for_build( + &session, + &environment, + Some(vw_api_types_versions::latest::TargetKind::Vivado), + ) + .await?; + + Ok(Some((session, environment))) +} + +/// Open a vivado session on this workspace's cloud environment, for the REPL. +/// +/// Falls back to a local worker when no service can be reached, with a warning +/// — the same bargain `vw run` strikes, and for the same reason: a developer +/// on a train should still get a REPL, but not silently one that is not the +/// one they meant. +async fn remote_worker( + named: Option<&str>, + insecure: bool, + part: &Option, + variant: &Option, +) -> Result> { + let session = cloud::Session::from_env(insecure)?; + + let environment = match cloud::pick_environment(&session, named).await { + Ok(environment) => environment, + Err(e) if cloud::Session::unreachable(&e) => { + eprintln!( + "{} no vw service reachable ({e}); running vivado on this \ + machine", + "warning:".yellow(), + ); + return Ok(vw_repl::Worker::Local); + } + Err(e) => return Err(e.into()), + }; + + // The instance builds what it was last given, so give it this before the + // first eval can ask for it. + cloud::sync_for_build( + &session, + &environment, + Some(vw_api_types_versions::latest::TargetKind::Vivado), + ) + .await?; + + let backend = cloud::open_vivado_session( + &session, + &environment, + vw_remote::SessionParams { + part: part.clone(), + variant: variant.clone(), + info_with_stack: false, + verbose: false, + }, + ) + .await?; + + // Taken before the backend is handed over, because once an eval is in + // flight the backend is borrowed and Ctrl-C has to work precisely then. + let handle = backend.interrupt_handle(); + + Ok(vw_repl::Worker::Remote { + backend: Box::new(backend), + interrupt: std::sync::Arc::new(move || handle.interrupt()), + }) +} + +/// Build the driver, wherever this workspace builds it. +/// +/// Cloud first, like everything else that needs a toolchain the developer's +/// machine may not have: the driver targets illumos and pins its own compiler, +/// and the helios instance is where both of those are true. +async fn driver_build( + cwd: &Utf8Path, + local: bool, + named: Option<&str>, + insecure: bool, + release: bool, + args: &[String], +) -> Result> { + let workspace = vw_lib::find_workspace_dir(cwd.as_std_path()) + .ok_or("no vw workspace here; run this from one, or from a directory inside it")?; + + if !local { + let session = cloud::Session::from_env(insecure)?; + match cloud::pick_environment(&session, named).await { + Ok(environment) => { + // The instance builds what it was last given. + cloud::sync_for_build( + &session, + &environment, + Some(vw_api_types_versions::latest::TargetKind::Helios), + ) + .await?; + return Ok(driver::build( + &session, + &environment, + release, + args, + ) + .await?); + } + Err(e) if cloud::Session::unreachable(&e) => { + eprintln!( + "{} no vw service reachable ({e}); building on this \ + machine", + "warning:".yellow(), + ); + } + Err(e) => return Err(e.into()), + } + } + + Ok(driver::build_locally(&workspace, release, args).await?) +} + +/// Remove build output, wherever this workspace builds. +/// +/// Cloud first, like `vw run`: what gets cleaned is what would get built. A +/// developer whose builds happen on an instance and whose `target/` here is +/// months stale would find cleaning the local one a waste of a command. +/// +/// Only build output goes. Source on the instance stays, so the next build +/// starts over without anything having to be pushed again. +async fn clean( + cwd: &Utf8Path, + local: bool, + named: Option<&str>, + insecure: bool, +) -> Result<(), Box> { + if !local { + let session = cloud::Session::from_env(insecure)?; + match cloud::pick_environment(&session, named).await { + Ok(environment) => { + cloud::clean_build_output(&session, &environment).await?; + return Ok(()); + } + Err(e) if cloud::Session::unreachable(&e) => { + eprintln!( + "{} no vw service reachable ({e}); cleaning this machine", + "warning:".yellow(), + ); + } + Err(e) => return Err(e.into()), + } + } + + let workspace = vw_lib::find_workspace_dir(cwd.as_std_path()).ok_or( + "no vw workspace here; run this from one, or from a \ + directory inside it", + )?; + let cleaned = vw_sync::clean(&workspace)?; + + if cleaned.existed { + println!( + "{} removed {}/{}", + "\u{2713}".bright_green(), + workspace, + vw_sync::BUILD_OUTPUT, + ); + } else { + println!("{} nothing to remove", "\u{2713}".bright_green()); + } + + Ok(()) +} + +/// Work out whether this run belongs in a cloud environment, and get the +/// environment ready for it. +/// +/// Cloud first: if there is an environment, that is where the build goes, +/// because that is where the machine with the memory and the licence is. A +/// service that cannot be reached is not fatal — plenty of work happens on a +/// train — but it is said out loud, since silently building here when you +/// meant to build there is an afternoon nobody gets back. +async fn cloud_site( + named: Option<&str>, + insecure: bool, +) -> Result, Box> { + let session = cloud::Session::from_env(insecure)?; + + let environment = match cloud::pick_environment(&session, named).await { + Ok(environment) => environment, + Err(e) if cloud::Session::unreachable(&e) => { + eprintln!( + "{} no vw service reachable ({e}); building on this machine", + "warning:".yellow(), + ); + return Ok(None); + } + Err(e) => return Err(e.into()), + }; + + // The instance builds what it was last given, so give it this. + cloud::sync_for_build( + &session, + &environment, + Some(vw_api_types_versions::latest::TargetKind::Vivado), + ) + .await?; + + Ok(Some((session, environment))) +} + +/// Where the vivado driving this run is. +/// +/// Cloud first: an environment is used when there is one, because that is +/// where the machine with the memory and the licence lives. `--local` is how +/// you say you meant this one. +pub(crate) enum Site<'a> { + Local, + Remote { + session: &'a cloud::Session, + environment: &'a str, + }, +} + +#[allow(clippy::too_many_arguments)] +async fn run_htcl( + file: &camino::Utf8Path, + site: Site<'_>, + check_only: bool, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, + bunyan: bool, +) -> Result<(), Box> { + let program = load_htcl_program(file).await?; + run_loaded_program( + file, + site, + &program, + check_only, + part, + variant, + log_level, + info_with_stack, + bunyan, + ) + .await +} + +/// Execute an already-loaded htcl program end to end: parse/validate +/// gates, an optional check-only short-circuit, then spawn Vivado and +/// run every command — RPC handler, prelude shipping, per-command +/// lowering + eval, and severity-driven exit code. +/// +/// Split out from [`run_htcl`] so `vw check`'s IP pre-pass can run an +/// in-memory `src @vw` + `vw::configure_ip` program through the exact +/// same Vivado machinery, without writing a file or shelling out to +/// `vw run`. +/// +/// `file` is the entry path — used for workspace discovery, the raw +/// log location, and proc-trace labels; for the IP pre-pass it's a +/// synthetic path inside the workspace whose on-disk source is never +/// read (the program is already loaded/flattened). +#[allow(clippy::too_many_arguments)] +async fn run_loaded_program( + file: &camino::Utf8Path, + site: Site<'_>, + program: &vw_htcl::LoadedProgram, + check_only: bool, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, + info_with_stack: bool, + bunyan: bool, +) -> Result<(), Box> { + // Under the hood the block renderer decides what to show; the + // backend's `verbose` toggle still gates the unclassified PTY + // firehose (banner, source echo, idle chatter). At log_level = + // Debug the user wants everything, so verbose stays on; at + // higher levels it stays off, and the classifier's NONE-block + // path is the only source of non-diagnostic content — which the + // renderer then dims or collapses. + let verbose = matches!(log_level, vw_vivado::LogLevel::Debug); + // Borrow `source` from the (caller-owned) program instead of + // moving; the stack-frame rewriting needs the LoadedProgram for + // body-span resolution. + let source = program.source.clone(); + let parsed = vw_htcl::parse(&source); + let line_index = vw_htcl::LineIndex::new(&source); + // Compile-time `putr` rewrite map: every `putr ` command + // in the document gets a replacement Tcl string keyed by its + // span. `vw_htcl::lower_command_with_putr` consults the map at + // emit time. See `vw-htcl/src/putr.rs` for the walker. + let putr_map = vw_htcl::putr::rewrite(&source, &parsed.document); + + let mut had_errors = false; + for err in &parsed.errors { + had_errors = true; + let (start, _end) = line_index.range(err.span); + eprintln!( + "{} {}:{}:{}: {}", + "error:".bright_red(), + file, + start.line + 1, + start.character + 1, + err.message + ); + } + if had_errors { + return Err(format!( + "{} parse error(s); aborting", + parsed.errors.len() + ) + .into()); + } + + // Validator gate. `vw check prime.htcl` runs `vw_htcl::validate` + // and returns non-zero on any error; `vw run prime.htcl` used + // to skip the validator entirely and hand the (possibly + // type-broken) program to Vivado, letting silent runtime + // divergence hide real bugs the checker had already found. Run + // the same validator here and abort on any error before + // spawning Vivado — same behavior `check` shows, same exit + // path. Warnings still emit but don't gate execution. + let validator_diags = vw_htcl::validate(&parsed.document, &source); + let mut error_count = 0usize; + let mut warning_count = 0usize; + let cwd_owned = std::env::current_dir().ok(); + let cwd = cwd_owned.as_deref(); + let mut indices: std::collections::HashMap = + std::collections::HashMap::new(); + for d in &validator_diags { + let (display_path, line, col) = match program.locate_span(d.span) { + Some((idx, file_span)) => { + let loaded = &program.files[idx]; + let index = indices + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(&loaded.source)); + let (start, _) = index.range(file_span); + ( + render_path(&loaded.path, cwd), + start.line + 1, + start.character + 1, + ) + } + None => (file.to_string(), 0, 0), + }; + match d.severity { + vw_htcl::Severity::Error => { + error_count += 1; + eprintln!( + "{} {display_path}:{line}:{col}: {}", + "error:".bright_red(), + d.message + ); + } + vw_htcl::Severity::Warning => { + warning_count += 1; + eprintln!( + "{} {display_path}:{line}:{col}: {}", + "warning:".bright_yellow(), + d.message + ); + } + } + } + if error_count > 0 { + eprintln!("{file}: {error_count} error(s), {warning_count} warning(s)"); + return Err( + format!("{error_count} validation error(s); aborting",).into() + ); + } + + if check_only { + let cmd_count = parsed + .document + .stmts + .iter() + .filter(|s| matches!(s, vw_htcl::Stmt::Command(_))) + .count(); + println!( + "{} {file}: parsed OK ({cmd_count} command(s))", + "✓".bright_green() + ); + return Ok(()); + } + + // RPC handler — serves htcl `vw::…` calls whose answers live + // on the tool side (workspace root, design source list, …). + // Constructed with whatever workspace state we can discover + // from the entry file; when no `vw.toml` is found the + // handler still exists but returns an error for `workspace_ + // root`, matching how the same call behaves in the LSP. + let file_path = std::path::Path::new(file.as_str()); + let rpc_workspace_root: Option = file_path + .parent() + .and_then(vw_lib::find_workspace_dir) + .map(|p| p.into_std_path_buf()); + // Auto-project bootstrap: if the enclosing workspace declares + // `[[target-parts]]` or `[[workspace.variants]]`, ship a + // `create_project -in_memory` down the wire before user code + // runs. `--variant ` picks a variant (variant-mode + // workspaces); `--part ` picks a target-parts entry + // (part-mode workspaces). Otherwise the workspace default + // wins. Eliminates the "no open project" failure mode for + // `ip::check` / `get_ipdefs` / etc., and gives the whole + // session a stable part context for downstream + // implementation/timing steps. + let ws_utf8 = rpc_workspace_root + .as_deref() + .and_then(camino::Utf8Path::from_path) + .map(|p| p.to_path_buf()); + let cw_count: vw_vivado::SharedCriticalWarningCount = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + + // Local or remote, what comes back drives the same way: `EdaBackend` is + // the whole of what the rest of this function knows about vivado, and it + // streams either way. Everything assembled above — the RPC handler, the + // project, the raw log — is for a worker on this machine; a remote session + // builds its own from the tree it is holding, because every one of those + // answers is about where the files are. + let mut backend: Box = match site { + Site::Local => { + let (auto_project, active_variant) = match ws_utf8.as_deref() { + Some(ws) => resolve_workspace_selection(ws, part, variant)?, + None => (None, None), + }; + // Seed the RPC handler's preload map with everything the + // entry-file load pulled in. `vw run` is single-shot — no + // batches commit after this — so the initial population IS + // the final state. `compile_htcl_module` (called from + // `vw::configure_ip` if the user's htcl invokes it) then + // skips re-shipping files whose procs are already installed. + let preload: vw_vivado::SharedPreload = { + let mut m = std::collections::HashMap::new(); + for f in &program.files { + if let Some(t) = f.mtime { + m.insert(f.path.clone(), t); + } + } + std::sync::Arc::new(std::sync::RwLock::new(m)) + }; + // Shared CRITICAL WARNING counter. Bumped by the stream sink + // below on each `Severity::CriticalWarning` chunk; read via + // the `critical_warning_count` RPC by `vw::synth` / `vw::place` + // to gate checkpoint writes on a CW-clean phase. Cloned into + // the handler and the sink so both sides see the same atomic. + let rpc_handler = vw_vivado::make_handler_full( + rpc_workspace_root.clone(), + active_variant, + preload, + cw_count.clone(), + ); + // Raw byte-log: `/target/logs/vivado-.log`. + // Failure to create the directory demotes to no-log rather than + // aborting the run — the log is a diagnostic aid, and a + // misconfigured target/ dir shouldn't block a synth flow. + let raw_log = rpc_workspace_root.as_deref().and_then(|ws| { + match vw_vivado::raw_log_path_for_workspace(ws) { + Ok(p) => { + eprintln!( + "{} {}", + "raw vivado log:".bright_black(), + p.display().to_string().bright_black(), + ); + Some(p) + } + Err(e) => { + eprintln!( + "{} raw log unavailable: {e}", + "warning:".yellow() + ); + None + } + } + }); + + Box::new( + vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await + .map_err(|e| format!("failed to start Vivado worker: {e}"))?, + ) + } + Site::Remote { + session, + environment, + } => { + let mut remote = cloud::open_vivado_session( + session, + environment, + vw_remote::SessionParams { + part: part.map(str::to_owned), + variant: variant.map(str::to_owned), + info_with_stack, + verbose, + }, + ) + .await?; + // What the instance is doing before it can run anything. Locally + // this stretch is silent too, but locally the developer knows why + // — the machine's fans are audible and vivado is in their process + // list. Across a network the same silence looks like a hang, and + // the wait is longer. Rendered the way the same notes are on a + // local run, several of them being literally the same notes. + remote.set_note_sink(Box::new(|message: &str| { + eprintln!("{} {message}", "info:".cyan()); + })); + Box::new(remote) + } + }; + + // Build the proc-location table the stream sink uses to map + // Tcl `:N in ::proc` frames back to real htcl source. + // Mirrors what `vw-repl` does per batch — we use the same + // shared helpers (`vw_repl::trace::*`) so REPL and CLI render + // the same. The entry file IS the scratch from build_proc_locations' + // perspective. + let entry_std_path = std::path::Path::new(file.as_str()).to_path_buf(); + let proc_table = std::sync::Arc::new(vw_repl::build_proc_locations( + &parsed.document, + program, + &entry_std_path, + )); + let input_file_for_stack = std::sync::Arc::new(entry_std_path.clone()); + // Shared between the main loop (writes the current origin + // before each eval) and the stream sink (reads it to tag + // unattributed warnings — e.g. Vivado IP-Flow C++ messages + // that bypass `::common::send_msg_id`). Same trick the REPL + // uses with `pending_origins[pending_eval_index]`. + let current_origin = + std::sync::Arc::new(std::sync::Mutex::new(None::)); + // Segmenter state — shared between the sink (which pushes into + // it per chunk) and the outer scope (which flushes any trailing + // NONE after the last eval returns so the tail doesn't get + // silently dropped when Vivado's last write is non-diagnostic). + let block_acc = std::sync::Arc::new(std::sync::Mutex::new( + vw_vivado::BlockAccumulator::new(), + )); + // Track the highest severity classified during the session. We + // check `kind` — not the rendered block — so `--log-level=error` + // can't silence a CRITICAL WARNING into a false-pass exit code. + // Encoded as u8 (0..=4) matching the Severity ladder so a plain + // atomic fetch_max gets us thread-safe "worst so far" semantics + // without a mutex. + let worst_severity = std::sync::Arc::new(std::sync::atomic::AtomicU8::new( + severity_as_u8(vw_vivado::Severity::None), + )); + // Static bunyan record fields (`hostname`, `pid`), resolved once and + // shared with the stdout sink and the trailing-flush below. Only + // consulted under `--bunyan`; cheap enough to compute either way. + let bunyan_host = gethostname::gethostname().to_string_lossy().into_owned(); + let bunyan_pid = std::process::id(); + { + let procs = std::sync::Arc::clone(&proc_table); + let input_file = std::sync::Arc::clone(&input_file_for_stack); + let origin = std::sync::Arc::clone(¤t_origin); + let acc = std::sync::Arc::clone(&block_acc); + let worst = std::sync::Arc::clone(&worst_severity); + let cw = std::sync::Arc::clone(&cw_count); + let host = bunyan_host.clone(); + backend.set_stdout_sink(Box::new(move |kind, chunk: &str| { + let cur_origin = origin.lock().ok().and_then(|g| g.clone()); + worst.fetch_max( + severity_as_u8(vw_vivado::severity_of(kind)), + std::sync::atomic::Ordering::Relaxed, + ); + // Bump the CW counter exposed via the + // `critical_warning_count` RPC. Only count exact + // CriticalWarning (not Error) — htcl checkpoint gates + // want CWs specifically; errors already abort the + // eval before the checkpoint-write branch is reached. + if matches!(kind, vw_vivado::StreamKind::CriticalWarning) { + cw.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + let blocks = acc + .lock() + .map(|mut a| a.push(kind, chunk)) + .unwrap_or_default(); + for block in blocks { + if bunyan { + emit_bunyan_block( + &block, + log_level, + &procs, + cur_origin.as_ref(), + Some(input_file.as_path()), + &host, + bunyan_pid, + ); + } else { + render_block( + &block, + log_level, + &procs, + cur_origin.as_ref(), + Some(input_file.as_path()), + ); + } + } + })); + } + + // Lower structured proc declarations and call sites to plain Tcl + // before sending. Generic commands pass through unchanged. + let table = vw_htcl::signature_table(&parsed.document); + // Ship enum preludes + overload dispatchers before any user + // statements run — same shape as the REPL's prepare path. + // Without these, calls to `Property::Scalar` or to an + // overloaded `handle` would fail at runtime with `invalid + // command name`. + let mut _ignored = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + // Always ship the primitive prelude so user-written newtype + // reprs can call e.g. `string::repr -v $v` for their inner + // values. + for p in vw_htcl::emit_primitive_prelude() { + let _ = backend.eval(&p).await?; + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + let _ = backend.eval(&prelude).await?; + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + let _ = backend.eval(&dispatcher).await?; + } + // Ship monomorphized generic reprs for every type expression + // referenced in any proc signature. This covers user newtypes + // that delegate to a generic repr (e.g. `Properties::repr` + // delegates to `dict_string_Property::repr`); without these + // the user's repr body errors at runtime with `invalid + // command name`. + let mut emitted_generics: std::collections::HashSet = + std::collections::HashSet::new(); + for sig in full_sigs.values() { + if let Some(ret) = sig.return_type.as_ref() { + ship_generic_reprs( + &mut *backend, + ret, + &type_decl_table, + &mut emitted_generics, + ) + .await?; + } + for arg in &sig.args { + if let Some(ty) = arg.type_annotation.as_ref() { + ship_generic_reprs( + &mut *backend, + ty, + &type_decl_table, + &mut emitted_generics, + ) + .await?; + } + } + } + // Per-file LineIndex cache so traceless-warning origins can be + // reported in the *originating file's* coordinates rather than the + // flattened LoadedProgram's. Without this, a warning anchored at + // the entry file ends up rendered at the merged-source line + // (something like `prime.htcl:119767`), which is meaningless. + let merged_line_index = vw_htcl::LineIndex::new(&source); + let mut per_file_line_index: std::collections::HashMap< + usize, + vw_htcl::LineIndex, + > = std::collections::HashMap::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + // Snapshot the origin of THIS statement before shipping + // it, so the stream sink can tag any traceless warning + // Vivado emits during the eval with the right "what was + // running" anchor. Mirrors the REPL's pending_origins + + // pending_eval_index mechanism. + let stmt_origin = { + let (file_path, line, snippet) = match program.locate_span(cmd.span) + { + Some((idx, local)) => { + let file_src = &program.files[idx].source; + let li = per_file_line_index + .entry(idx) + .or_insert_with(|| vw_htcl::LineIndex::new(file_src)); + let (lc, _) = li.range(local); + let snippet = file_src + [local.start as usize..local.end as usize] + .lines() + .next() + .unwrap_or("") + .to_string(); + ( + Some(program.files[idx].path.clone()), + lc.line + 1, + snippet, + ) + } + None => { + // Synthetic span that doesn't lie in any loaded + // file (e.g. a generated dispatcher). Fall back to + // the merged-source line; the entry-file label is + // attached by the renderer when `file` is None. + let (lc, _) = merged_line_index.range(cmd.span); + let snippet = source + [cmd.span.start as usize..cmd.span.end as usize] + .lines() + .next() + .unwrap_or("") + .to_string(); + (None, lc.line + 1, snippet) + } + }; + let origin = vw_repl::Origin { + file: file_path, + line, + snippet, + via: Vec::new(), + }; + if let Ok(mut g) = current_origin.lock() { + *g = Some(origin.clone()); + } + origin + }; + // Overload specializations lower under their mangled + // names so the dispatcher's switch arms can find them. + let lowered = match overload_specialization_mangle(cmd, &overload_table) + { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!() + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &source, + &table, + Some(&mangled), + &putr_map, + &merged_line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &source, + &table, + &putr_map, + &merged_line_index, + ), + }; + // Rewrite `extern::name` → `::name` (the textual pass the + // REPL also runs) so calls to runtime-Tcl/Vivado procs + // reach Vivado as the bare native name instead of the + // literal `extern::` text — without this, every wrapper + // body that forwards via `extern::` errors out at runtime + // with `invalid command name "extern::create_project"`. + let tcl = vw_htcl::rewrite_externs(&lowered).text; + // Wrap with a shim-side origin marker so any traceless + // warning emitted during THIS eval stays anchored to + // `stmt_origin` — see [`vw_repl::wrap_tcl_with_origin_marker`] + // for the race this fixes. + let tcl = vw_repl::wrap_tcl_with_origin_marker(&tcl, &stmt_origin); + // `set VAR ` is a binding — the user asked to name a + // value, not to display it. Vivado's Tcl returns the + // bound value as the eval result, and echoing that would + // leak the raw internal form (e.g. `metroid` from + // `set proj [create_project … -name metroid]` in + // `project.htcl`). Suppress the result echo for set + // bindings so the batch path matches the REPL's + // `is_set_binding` policy (`vw-repl/src/app.rs:2437`). + let is_set_binding = matches!(cmd.kind, vw_htcl::CommandKind::Set); + match backend.eval(&tcl).await { + Ok(out) => { + // Puts output already streamed to stdout via the + // sink; `out.stdout` is empty here by contract. The + // eval's return value gets a newline only when it's + // not already empty AND the source command wasn't a + // set binding. + if !out.value.is_empty() && !is_set_binding { + if bunyan { + // Eval return values are results, not + // diagnostics — surface them at bunyan info so + // stdout stays a pure JSON stream for looker. + emit_bunyan_line( + 30, + "result", + None, + &out.value, + &bunyan_host, + bunyan_pid, + ); + } else if matches!( + log_level, + vw_vivado::LogLevel::Debug | vw_vivado::LogLevel::Info + ) { + // Echo a command's return value only at Info/ + // Debug. At Warning+ the caller asked for terse + // output — e.g. `vw check`'s IP pre-pass, where + // `vw::configure_ip`'s `null` return is pure + // noise. + println!("{}", out.value); + } + } + } + Err(vw_eda::BackendError::Tcl { message, .. }) => { + eprintln!("{} {message}", "vivado:".bright_red()); + } + Err(e) => { + eprintln!("{} {e}", "vivado:".bright_red()); + } + } + } + let _ = backend.shutdown().await; + // Flush any trailing NONE block the last chunk left in the + // accumulator. Vivado's final write is often non-diagnostic + // (banner, resource summary, "synth_design completed" table); + // without this the very end of the session would sit unemitted + // when the caller's eval returns and the sink stops being + // called. render_block honors the log-level here too, so + // trailing noise still gets dimmed at Info+ and shown raw at + // Debug. + let trailing = block_acc.lock().map(|mut a| a.flush()).unwrap_or_default(); + let cur_origin = current_origin.lock().ok().and_then(|g| g.clone()); + for block in trailing { + if bunyan { + emit_bunyan_block( + &block, + log_level, + &proc_table, + cur_origin.as_ref(), + Some(input_file_for_stack.as_path()), + &bunyan_host, + bunyan_pid, + ); + } else { + render_block( + &block, + log_level, + &proc_table, + cur_origin.as_ref(), + Some(input_file_for_stack.as_path()), + ); + } + } + // Severity-driven exit code. If ANY chunk classified as + // CRITICAL WARNING or ERROR during the session, propagate a + // non-zero exit so CI wrappers / scripts can gate on it. We + // read the classification counter — NOT the render decision — + // so a user passing `--log-level=error` can't hide a + // CRITICAL WARNING into a false-pass. The diagnostic itself + // has already streamed through the log-level filter (or was + // suppressed at the caller's request); this is just about the + // process return code. + let worst = worst_severity.load(std::sync::atomic::Ordering::Relaxed); + if worst >= severity_as_u8(vw_vivado::Severity::CriticalWarning) { + let label = if worst >= severity_as_u8(vw_vivado::Severity::Error) { + "error" + } else { + "critical warning" + }; + return Err(format!("session emitted at least one {label}").into()); + } + Ok(()) +} + +/// True when a VHDL diagnostic set contains an error a Vivado IP pass +/// could fix: an unresolved unit or missing library within the +/// tool-generated `ip` / `xil_defaultlib` libraries. Keys off the +/// library name in the analyzer's message — which only appears on a +/// resolution failure (`No such library 'ip'`, `No primary unit +/// 'primary_clock' within library 'xil_defaultlib'`) — rather than the +/// emptiness of the rendered library: the BD wrappers populate +/// `xil_defaultlib` even while a standalone XCI IP's stub is still +/// absent, so emptiness is not a reliable signal. +fn diagnostics_need_ip_generation(diags: &[vw_lib::VhdlDiagnostic]) -> bool { + diags.iter().any(|d| { + matches!(d.severity, vw_lib::VhdlSeverity::Error) + && (d.message.contains("library 'ip'") + || d.message.contains("library 'xil_defaultlib'")) + }) +} + +/// Run `src @vw` + `vw::configure_ip` in-process (spawning Vivado) so a +/// subsequent VHDL check can resolve the design's generated `ip` / +/// `xil_defaultlib` libraries. +/// +/// Called by `vw check` only when +/// [`vw_lib::workspace_needs_ip_generation`] reports the design +/// references those libraries and their wrappers aren't already in +/// `target/` — so a workspace whose IP is already generated never +/// pays for a (multi-minute) Vivado run. The program is built and run +/// entirely in-library: no temp file, no `vw run` sub-process. +async fn ensure_ip_generated( + ws: &camino::Utf8Path, + cloud: Option<&(cloud::Session, String)>, + part: Option<&str>, + variant: Option<&str>, + log_level: vw_vivado::LogLevel, +) -> Result<(), Box> { + // Build the resolver exactly as `load_htcl_program` does so `@vw` + // — and `@vw`'s own transitive `src @vivado-cmd/…` imports — + // resolve. `configure_ip` is a normal (non-test) run, so + // test-dependencies stay out. + let mut resolver = vw_htcl::Resolver::new(); + if let Ok(paths) = vw_lib::transitive_dep_cache_paths_with_test(ws, false) { + for (name, path) in paths { + resolver = resolver.with_dep(name, path); + } + } + // Cargo-parity self-reference so a workspace can `src @/…`. + if let Ok(cfg) = vw_lib::load_workspace_config(ws) { + resolver = resolver.with_dep_if_absent( + cfg.workspace.name, + ws.as_std_path().to_path_buf(), + ); + } + // Synthetic entry inside the workspace — never written to disk. + // Its parent (the workspace) is what `run_loaded_program` uses to + // discover the RPC workspace root; `@vw` resolves through the dep + // map above, not this path. + let entry = ws.join(".vw-configure-ip.htcl"); + // `configure_ip -generate_targets false` makes block-design IPs + // check-ready (their `_wrapper` entities land in library `ip` + // via `make_wrapper`) WITHOUT the minutes-long + // `generate_target -name all` per BD — that generation only feeds + // synthesis, and the check resolves the wrapper's inner BD as an + // unbound `component`. `generate_ip_stubs` then writes the + // instantiation-template stubs for the standalone XCI IPs so + // `entity xil_defaultlib.` (e.g. a bare `clk_wizard` like + // `primary_clock`) also resolves. Neither step synthesizes. + // + // Explicit `src ip` (when the workspace has one) loads + // `ip/module.htcl` and every file it transitively `src`s into + // the LoadedProgram at compile time. Without this, `vw:: + // configure_ip` would auto-load them at runtime via an RPC + // `compile_htcl_module` → `uplevel #0 $tcl` eval, which the + // Tcl side attributes to the anonymous script — so any + // Vivado warning coming from an `ip/*.htcl` proc surfaces as + // `:N` and the stack-frame rewriter can't map it back + // to source (proc_table has no entry for procs the LoadedProgram + // didn't see). Sourcing here puts them in `program.files`, + // `configure_ip`'s `info procs ::ip::configure` gate skips + // the RPC compile, and the rewriter's per-proc lookup + // resolves `configure_cips`, `configure_gtm`, … to their + // real `ip/.htcl:` positions. + let has_ip_module = ws.join("ip/module.htcl").is_file(); + let src = if has_ip_module { + "src @vw\n\ + src ip\n\ + vw::configure_ip -generate_targets false\n\ + vw::generate_ip_stubs\n" + } else { + "src @vw\n\ + vw::configure_ip -generate_targets false\n\ + vw::generate_ip_stubs\n" + }; + let program = + vw_htcl::load_program_source(src, entry.as_std_path(), &resolver)?; + let site = match cloud { + Some((session, environment)) => Site::Remote { + session, + environment, + }, + None => Site::Local, + }; + let run_result = run_loaded_program( + &entry, site, &program, /*check_only=*/ false, part, variant, + log_level, /*info_with_stack=*/ false, /*bunyan=*/ false, + ) + .await; + // Rewrite the just-generated `.vho` instantiation templates into + // black-box VHDL stubs. Done here (not in htcl) — a mechanical + // component→entity splice is far simpler in Rust than in Tcl. Run + // it even if the pre-pass reported a non-fatal error, since the + // templates may already be on disk. + // + // Remotely, the templates are on the instance and so is the splice; what + // comes back is the finished VHDL. Either way the check that follows finds + // the same files at the same paths, which is the whole point — a language + // server that opens a wrapper has to open a real file. + match cloud { + Some((session, environment)) => { + match cloud::fetch_generated_ip(session, environment, ws).await { + Ok(0) => {} + Ok(n) => println!( + "{} fetched {n} generated IP file(s) for the VHDL check", + "note:".bright_yellow(), + ), + Err(e) => eprintln!( + "{} could not fetch the generated IP: {e}", + "warning:".yellow(), + ), + } + } + None => { + if let Ok(n) = vw_lib::write_ip_stubs_from_templates(ws) { + if n > 0 { + println!( + "{} wrote {n} IP stub(s) for the VHDL check", + "note:".bright_yellow(), + ); + } + } + } + } + run_result +} + +#[cfg(test)] +mod bunyan_tests { + use super::*; + + /// The severity→level mapping is the crux of `--bunyan`: INFO→30, + /// WARNING→40, CRITICAL WARNING and ERROR both →50 (error), and + /// non-diagnostic noise →20 (debug). + #[test] + fn severity_maps_to_requested_bunyan_levels() { + assert_eq!(bunyan_level_for(vw_vivado::Severity::Info), 30); + assert_eq!(bunyan_level_for(vw_vivado::Severity::Warning), 40); + assert_eq!(bunyan_level_for(vw_vivado::Severity::CriticalWarning), 50); + assert_eq!(bunyan_level_for(vw_vivado::Severity::Error), 50); + assert_eq!(bunyan_level_for(vw_vivado::Severity::None), 20); + } + + /// Every required bunyan field (per looker's `BunyanEntry`) must be + /// present with the right type, the record must be a single line, and + /// quotes/newlines in the message must survive JSON escaping. + #[test] + fn record_is_single_line_json_with_all_required_fields() { + let rec = bunyan_record( + 50, + "vivado", + Some("critical-warning"), + "CRITICAL WARNING: [Vivado 12-4739] a \"quoted\" bit\n at x.htcl:3", + "host1", + 1234, + "2026-07-19T00:00:00.000Z", + ); + let line = rec.to_string(); + assert!(!line.contains('\n'), "a record must serialize to one line"); + + let v: serde_json::Value = serde_json::from_str(&line).unwrap(); + // Required fields looker rejects a record for missing. + assert_eq!(v["v"], 0); + assert_eq!(v["name"], "vw"); + assert_eq!(v["hostname"], "host1"); + assert_eq!(v["pid"], 1234); + assert_eq!(v["level"], 50); + assert_eq!(v["time"], "2026-07-19T00:00:00.000Z"); + assert!(v["msg"].is_string()); + // Original severity preserved so CW is distinguishable from ERROR. + assert_eq!(v["severity"], "critical-warning"); + assert_eq!(v["component"], "vivado"); + // Escaping round-trips the embedded quote and newline. + let msg = v["msg"].as_str().unwrap(); + assert!(msg.contains('"') && msg.contains('\n')); + } + + /// An info record with no `severity` (the eval-result echo path) still + /// carries every required field and omits the optional `severity`. + #[test] + fn result_record_omits_optional_severity() { + let rec = bunyan_record( + 30, + "result", + None, + "ok", + "h", + 7, + "2026-07-19T00:00:00.000Z", + ); + assert!(rec.get("severity").is_none()); + assert_eq!(rec["level"], 30); + assert_eq!(rec["component"], "result"); + assert_eq!(rec["v"], 0); } } diff --git a/vw-cli/src/parallel_load.rs b/vw-cli/src/parallel_load.rs new file mode 100644 index 0000000..84351d1 --- /dev/null +++ b/vw-cli/src/parallel_load.rs @@ -0,0 +1,740 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parallel htcl loader with per-dep progress rows. +//! +//! Wraps the sync `vw_htcl` load primitives with an async coordinator +//! that: +//! +//! - Recursively discovers and parses files, with `tokio::task:: +//! spawn_blocking` for each file's read+parse so N cores work in +//! parallel. +//! - Stitches the results back into the SAME `LoadedProgram` shape +//! the serial loader produces (flat `source` + `regions`), in the +//! same deterministic DFS-from-entry order — so every downstream +//! consumer (validator, putr, lower, LSP) works unchanged. +//! - Fires observer events (`on_source`, `on_parsed`, `on_dep_ +//! completed`) as parses complete, so the CLI's +//! [`MultiProgress`] UI can render one live row per top-level +//! dep and commit each row when its subtree is done. +//! +//! The loader is dependency-graph aware via `petgraph`. Cycles are +//! detected and rejected with `LoadError::Cycle`. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; + +use futures::future::BoxFuture; +use futures::FutureExt; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use tokio::task::JoinError; +use vw_htcl::{ + parse, CommandKind, Document, ImportEdge, LoadError, LoadedFile, + LoadedProgram, Resolver, SourceRegion, Span, Stmt, +}; + +/// Extended observer trait for the parallel loader. Adds +/// [`on_dep_completed`] so the UI can commit a top-level dep's +/// progress row when its whole subtree has finished parsing. +/// +/// The base `LoadObserver` methods still fire (once per file, from +/// blocking-thread contexts so implementations must be Send + Sync). +pub trait ParallelObserver: Send + Sync { + fn on_source(&self, _raw: &str, _resolved: &Path) {} + fn on_parsed(&self, _file: &Path, _raw: Option<&str>) {} + /// A top-level dep name (`@vivado-cmd`, `@cpm5`, ...) has + /// finished parsing its whole subtree. Used by the CLI to + /// commit the dep's progress row to `Checking @`. + fn on_dep_completed(&self, _dep_name: &str) {} +} + +/// The result of parsing one file. Shared across all consumers via +/// `Arc` so multiple parts of the stitch phase can hold references +/// without cloning the source text. +struct ParsedFile { + path: PathBuf, + source: String, + document: Document, + mtime: Option, + /// Import descriptors in source order, each resolved to a + /// canonical path. Preserving order matters for the stitch + /// phase's chunking output. + imports: Vec, +} + +#[derive(Clone)] +struct ImportInfo { + raw: String, + resolved: PathBuf, + /// Span of the `src` command in this file's local source + /// (the source the parse was against, not the flat source). + src_span: Span, +} + +/// Shared parse-cache. One slot per canonical path; the slot's +/// `Notify` fires when the parse completes, so a second discovery +/// of the same path awaits the first task's result instead of +/// re-parsing. +type ParseCache = Arc>>; + +#[derive(Clone)] +enum ParseSlot { + /// In-progress. Awaiters watch the notify. + Pending(Arc), + /// Completed successfully. + Done(Arc), + /// Failed. Error is stringified because `LoadError` isn't `Clone`. + Failed(String), + /// Preloaded and unchanged (mtime match). Skip; downstream + /// treats this file as if it wasn't there (it's already in a + /// prior batch's LoadedProgram). + Skipped, +} + +/// Public entry point for the parallel loader. Same contract as +/// `vw_htcl::load_program`: reads the entry file, recursively +/// resolves every `src` statement, and returns a `LoadedProgram` +/// whose flat source is DFS-ordered exactly as the serial loader +/// would produce. +/// +/// `preloaded` mirrors the sync loader's cross-batch cache — files +/// listed here whose current mtime matches the stored one are +/// skipped as if already sourced. +pub async fn load_parallel( + entry: &Path, + resolver: Arc, + observer: Arc, + preloaded: HashMap, +) -> Result { + let entry = entry.canonicalize().unwrap_or_else(|_| entry.to_path_buf()); + let cache: ParseCache = Arc::new(Mutex::new(HashMap::new())); + let preloaded = Arc::new(preloaded); + let deps_in_flight: Arc>> = + Arc::new(Mutex::new(HashMap::new())); + + // Recursive parallel discovery+parse. The entry file has no + // dep context; every reachable file inherits from its + // importer's dep bucket (see `bucket_for_path` used by the + // observer wiring). + schedule_parse( + entry.clone(), + None, + resolver.clone(), + observer.clone(), + cache.clone(), + preloaded.clone(), + deps_in_flight.clone(), + ) + .await?; + + // Serially stitch the flat source in the same DFS order the + // sync loader produces. This is what makes the output + // interchangeable with `vw_htcl::load_program` for downstream + // consumers. + stitch(&entry, &cache) +} + +/// Await the parse of `path` (kicking it off first if no other +/// task has). Recurses into imports concurrently, so multiple +/// file trees load in parallel. +fn schedule_parse( + path: PathBuf, + imported_via_raw: Option, + resolver: Arc, + observer: Arc, + cache: ParseCache, + preloaded: Arc>, + deps_in_flight: Arc>>, +) -> BoxFuture<'static, Result<(), LoadError>> { + async move { + // Fast path: skip preloaded files whose mtime hasn't + // changed. Same cross-batch semantics as the sync loader. + if let Some(stored_mtime) = preloaded.get(&path) { + let current_mtime = std::fs::metadata(&path) + .ok() + .and_then(|m| m.modified().ok()); + if current_mtime == Some(*stored_mtime) { + let mut guard = cache.lock().unwrap(); + guard.entry(path.clone()).or_insert(ParseSlot::Skipped); + return Ok(()); + } + } + + // Claim the slot. If someone else already claimed, await + // their notify; if already done, return. + enum Claim { + Done, + Failed(String), + AwaitPending(Arc), + Owned(Arc), + } + let claim = { + let mut guard = cache.lock().unwrap(); + match guard.get(&path).cloned() { + Some(ParseSlot::Done(_)) | Some(ParseSlot::Skipped) => { + Claim::Done + } + Some(ParseSlot::Failed(msg)) => Claim::Failed(msg), + Some(ParseSlot::Pending(n)) => Claim::AwaitPending(n), + None => { + let n = Arc::new(tokio::sync::Notify::new()); + guard.insert(path.clone(), ParseSlot::Pending(n.clone())); + Claim::Owned(n) + } + } + }; + let notify = match claim { + Claim::Done => return Ok(()), + Claim::Failed(msg) => { + return Err(LoadError::Io { + path: path.clone(), + source: std::io::Error::other(msg), + }); + } + Claim::AwaitPending(n) => { + n.notified().await; + return Ok(()); + } + Claim::Owned(n) => n, + }; + + // Do read+parse on a blocking thread. Reading is I/O + // bound; parsing is CPU bound. spawn_blocking is the + // right primitive. + let parsed_path = path.clone(); + let parse_task = + tokio::task::spawn_blocking(move || read_and_parse(&parsed_path)); + + let parsed_result = parse_task.await.map_err(join_err_to_load_err)?; + let parsed = match parsed_result { + Ok(p) => Arc::new(p), + Err(e) => { + let msg = format!("{e}"); + let mut guard = cache.lock().unwrap(); + guard.insert(path.clone(), ParseSlot::Failed(msg)); + notify.notify_waiters(); + return Err(e); + } + }; + + // Extract src operands from the parsed doc, resolve + // each, and schedule parallel parses for the targets. + // The `imports` field on ParsedFile carries the ordered + // (raw, resolved) pairs; we resolve here so we can + // emit the on_source event with the resolved path. + let parent_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let mut imports: Vec = Vec::new(); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + let Some(raw) = import.path.as_deref() else { + let line = line_of(&parsed.source, cmd.span.start) + 1; + let err = LoadError::DynamicPath { + importer: path.clone(), + line, + }; + let mut guard = cache.lock().unwrap(); + guard.insert(path.clone(), ParseSlot::Failed(format!("{err}"))); + notify.notify_waiters(); + return Err(err); + }; + let resolved = + resolver.resolve(&parent_dir, raw).map_err(|source| { + LoadError::Resolve { + importer: path.clone(), + raw: raw.to_string(), + source, + } + })?; + imports.push(ImportInfo { + raw: raw.to_string(), + resolved, + src_span: cmd.span, + }); + } + + // Attach imports to the parsed file so the stitch phase + // can reproduce the byte order. Use the `Arc` trick: + // wrap in a new Arc after mutating. + let parsed_with_imports = Arc::new(ParsedFile { + path: parsed.path.clone(), + source: parsed.source.clone(), + document: parsed.document.clone(), + mtime: parsed.mtime, + imports: imports.clone(), + }); + + // Emit on_source for each not-yet-seen target BEFORE + // recursing. The observer sees Sourcing events in the + // same conceptual order the serial loader would. + for imp in &imports { + let already_seen = cache + .lock() + .unwrap() + .get(&imp.resolved) + .map(|s| !matches!(s, ParseSlot::Failed(_))) + .unwrap_or(false); + if !already_seen { + observer.on_source(&imp.raw, &imp.resolved); + // Track top-level dep in-flight count for + // on_dep_completed firing. + if let Some(dep) = extract_dep_from_raw(&imp.raw) { + let mut deps = deps_in_flight.lock().unwrap(); + *deps.entry(dep).or_insert(0) += 1; + } + } + } + + // Spawn recursive parses concurrently. + let mut recurse_tasks = Vec::new(); + for imp in imports.iter() { + let fut = schedule_parse( + imp.resolved.clone(), + Some(imp.raw.clone()), + resolver.clone(), + observer.clone(), + cache.clone(), + preloaded.clone(), + deps_in_flight.clone(), + ); + recurse_tasks.push(fut); + } + for result in futures::future::join_all(recurse_tasks).await { + result?; + } + + // All this file's imports are done. Publish the parse + // result and notify awaiters. + { + let mut guard = cache.lock().unwrap(); + guard.insert( + path.clone(), + ParseSlot::Done(parsed_with_imports.clone()), + ); + } + notify.notify_waiters(); + + observer.on_parsed(&path, imported_via_raw.as_deref()); + + // If this file is the last outstanding file of a + // top-level dep bucket, fire on_dep_completed. + if let Some(via_raw) = imported_via_raw.as_deref() { + if let Some(dep) = extract_dep_from_raw(via_raw) { + let mut deps = deps_in_flight.lock().unwrap(); + if let Some(count) = deps.get_mut(&dep) { + *count = count.saturating_sub(1); + if *count == 0 { + deps.remove(&dep); + drop(deps); + observer.on_dep_completed(&dep); + } + } + } + } + + Ok(()) + } + .boxed() +} + +/// Sync read+parse for one file. Runs on a `spawn_blocking` +/// thread. Failures propagate as `LoadError`. +fn read_and_parse(path: &Path) -> Result { + let source = std::fs::read_to_string(path).map_err(|e| LoadError::Io { + path: path.to_path_buf(), + source: e, + })?; + let mtime = std::fs::metadata(path).ok().and_then(|m| m.modified().ok()); + let parsed = parse(&source); + if !parsed.errors.is_empty() { + return Err(LoadError::Parse { + path: path.to_path_buf(), + errors: parsed.errors, + }); + } + Ok(ParsedFile { + path: path.to_path_buf(), + source, + document: parsed.document, + mtime, + imports: Vec::new(), + }) +} + +/// Extract the top-level `@name` prefix from a src operand, so +/// the observer can bucket files by top-level dep. `./relative` +/// operands (dep-internal imports) return None — the CLI-side +/// observer resolves them via the loader-observer path. +fn extract_dep_from_raw(raw: &str) -> Option { + let rest = raw.strip_prefix('@')?; + let name = match rest.split_once('/') { + Some((n, _)) => n, + None => rest, + }; + Some(format!("@{name}")) +} + +/// Serially stitch the flat `LoadedProgram` from the parsed +/// cache. Walks in the same DFS-from-entry order the sync +/// loader uses so byte offsets line up with what downstream +/// consumers expect. +fn stitch( + entry: &Path, + cache: &ParseCache, +) -> Result { + let mut program = LoadedProgram::default(); + let mut loaded_files: HashMap = HashMap::new(); + let mut in_progress: HashSet = HashSet::new(); + stitch_file( + entry, + None, + cache, + &mut program, + &mut loaded_files, + &mut in_progress, + )?; + if !program.source.ends_with('\n') { + program.source.push('\n'); + } + Ok(program) +} + +/// Recursive DFS stitch. Same shape as the sync loader's +/// `load_file` but reads pre-parsed content from the cache. +fn stitch_file( + path: &Path, + imported_via: Option, + cache: &ParseCache, + program: &mut LoadedProgram, + loaded_files: &mut HashMap, + in_progress: &mut HashSet, +) -> Result<(), LoadError> { + if loaded_files.contains_key(path) || in_progress.contains(path) { + return Ok(()); + } + let parsed = { + let guard = cache.lock().unwrap(); + match guard.get(path).cloned() { + Some(ParseSlot::Done(p)) => p, + Some(ParseSlot::Skipped) => return Ok(()), + Some(ParseSlot::Failed(msg)) => { + return Err(LoadError::Io { + path: path.to_path_buf(), + source: std::io::Error::other(msg), + }); + } + Some(ParseSlot::Pending(_)) | None => { + return Err(LoadError::Io { + path: path.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::NotFound, + "parallel loader lost parse for this path", + ), + }); + } + } + }; + in_progress.insert(path.to_path_buf()); + + let file_index = program.files.len() as u32; + program.files.push(LoadedFile { + path: path.to_path_buf(), + source: parsed.source.clone(), + imported_via, + mtime: parsed.mtime, + }); + loaded_files.insert(path.to_path_buf(), file_index as usize); + + // Emit chunks + regions in span order, recursing into each + // src target between chunks. + let mut cursor = 0usize; + for imp in &parsed.imports { + // Find this src stmt's span in the doc so we chunk + // around it. `imp.src_span` was recorded at parse time. + let cmd_start = imp.src_span.start as usize; + let cmd_end = imp.src_span.end as usize; + emit_chunk(program, &parsed.source, cursor, cmd_start, file_index); + cursor = cmd_end; + if parsed.source.as_bytes().get(cursor) == Some(&b'\n') { + cursor += 1; + } + stitch_file( + &imp.resolved, + Some(ImportEdge { + importer_file: file_index as usize, + src_span: imp.src_span, + }), + cache, + program, + loaded_files, + in_progress, + )?; + } + emit_chunk( + program, + &parsed.source, + cursor, + parsed.source.len(), + file_index, + ); + if !program.source.ends_with('\n') { + program.source.push('\n'); + } + in_progress.remove(path); + Ok(()) +} + +fn emit_chunk( + program: &mut LoadedProgram, + source: &str, + start: usize, + end: usize, + file_index: u32, +) { + if start >= end { + return; + } + let flat_start = program.source.len() as u32; + program.source.push_str(&source[start..end]); + let flat_end = program.source.len() as u32; + program.regions.push(SourceRegion { + flat_start, + flat_end, + file_index, + file_offset: start as u32, + }); +} + +fn line_of(source: &str, byte: u32) -> u32 { + source[..(byte as usize).min(source.len())] + .bytes() + .filter(|b| *b == b'\n') + .count() as u32 +} + +fn join_err_to_load_err(e: JoinError) -> LoadError { + LoadError::Io { + path: PathBuf::new(), + source: std::io::Error::other(e.to_string()), + } +} + +/// Observer that drives an `indicatif::MultiProgress` panel — one +/// live [`ProgressBar`] per top-level dep, plus one for local / +/// workspace files. +/// +/// On a TTY, each bar's message updates as its inner files fly by. +/// When [`on_dep_completed`] fires, the bar commits with +/// `Checking @` and stays in scrollback. Non-TTY stdout is +/// handled natively by `indicatif` — bars degrade to no-op and +/// we fall back to the `println` path for each event. +/// +/// The observer holds bars behind an internal `Mutex` because +/// [`ProgressBar`] is `Send + Sync` but our bookkeeping around +/// bar creation (first-sight-per-dep) needs synchronized +/// access. Bar updates otherwise happen from many blocking- +/// thread contexts concurrently. +/// +/// **Terminal-side note.** Committed rows are emitted via +/// [`MultiProgress::println`] which uses cursor manipulation to +/// insert content above the bar area rather than a natural scroll. +/// On terminals whose Ctrl-L handler pushes visible content into +/// scrollback before clearing (iTerm2, most xterm-family +/// emulators, tmux), this behaves correctly. On terminals whose +/// Ctrl-L just erases the visible display without preserving +/// unscrolled content (some recent ghostty builds), the committed +/// rows can vanish. That's a terminal behavior, not something the +/// loader can fix at emission time — the tradeoff of the multi- +/// row UI. +pub struct MultiProgressObserver { + multi: MultiProgress, + /// `(name, ProgressBar)` — one per top-level dep. `name` is + /// the `@dep` prefix or the workspace label for local files. + /// Insertion order preserved so scrollback matches source + /// order. + bars: Mutex>, + /// `(depname, cache-directory-abs-path)` pairs used to rewrite + /// resolved paths back into `@dep/relative` labels. + dep_paths: Vec<(String, PathBuf)>, + /// True when stdout is a real terminal. Bars only render + /// when true; non-TTY falls back to plain `println!`. + stdout_is_tty: bool, + /// Human-friendly label for the local workspace's bar. Picked + /// up from `vw.toml`'s `name = "…"` field when available, so + /// `Checking workspace` becomes `Checking metroid` for the + /// metroid workspace. Falls back to the literal `workspace` + /// when no name is configured. + workspace_label: String, +} + +impl MultiProgressObserver { + pub fn new( + dep_paths: Vec<(String, PathBuf)>, + workspace_label: String, + ) -> Self { + use std::io::IsTerminal; + let multi = MultiProgress::new(); + Self { + multi, + bars: Mutex::new(Vec::new()), + dep_paths, + stdout_is_tty: std::io::stdout().is_terminal(), + workspace_label, + } + } + + /// Format a `src` label. `@/` when the + /// path is inside a known dep-cache directory; otherwise + /// strip `@` sigil and `.htcl` suffix. + fn friendly_label(&self, raw: &str, resolved: Option<&Path>) -> String { + if let Some(resolved) = resolved { + let canonical = resolved + .canonicalize() + .unwrap_or_else(|_| resolved.to_path_buf()); + for (name, dep_path) in &self.dep_paths { + let dep_canonical = dep_path + .canonicalize() + .unwrap_or_else(|_| dep_path.clone()); + if let Ok(rel) = canonical.strip_prefix(&dep_canonical) { + let rel_str = rel.display().to_string(); + let rel_str = rel_str.trim_end_matches(".htcl"); + return if rel_str.is_empty() { + format!("@{name}") + } else { + format!("@{name}/{rel_str}") + }; + } + } + } + if !raw.is_empty() { + return raw + .trim_start_matches('@') + .trim_end_matches(".htcl") + .to_string(); + } + resolved + .and_then(|p| p.file_stem()) + .and_then(|s| s.to_str()) + .unwrap_or("?") + .to_string() + } + + /// Bucket a `label` into a top-level dep name for bar routing. + /// `@foo/bar` → `@foo`; `foo/bar` or `foo` → the workspace + /// label (typically the `vw.toml` name). + fn bucket_of(&self, label: &str) -> String { + if let Some(rest) = label.strip_prefix('@') { + let name = match rest.split_once('/') { + Some((n, _)) => n, + None => rest, + }; + format!("@{name}") + } else { + self.workspace_label.clone() + } + } + + /// Return the bar for `bucket`, creating it on first sight. + /// Access is serialized on the bars mutex so parallel task + /// contexts don't race. + fn bar_for(&self, bucket: &str) -> ProgressBar { + let mut guard = self.bars.lock().unwrap(); + if let Some((_, bar)) = guard.iter().find(|(n, _)| n == bucket) { + return bar.clone(); + } + let bar = self.multi.add(ProgressBar::new_spinner()); + bar.set_style( + ProgressStyle::with_template("{prefix:>12.bold.green} {msg}") + .expect("static template compiles"), + ); + bar.set_prefix("Sourcing"); + bar.set_message(bucket.to_string()); + // Steady-tick so spinner-shaped bars animate without needing + // explicit ticks between updates. + bar.enable_steady_tick(std::time::Duration::from_millis(100)); + guard.push((bucket.to_string(), bar.clone())); + bar + } +} + +impl ParallelObserver for MultiProgressObserver { + fn on_source(&self, raw: &str, resolved: &Path) { + let label = self.friendly_label(raw, Some(resolved)); + let bucket = self.bucket_of(&label); + if !self.stdout_is_tty { + println!("{:>12} {label}", "Sourcing"); + return; + } + let bar = self.bar_for(&bucket); + bar.set_prefix("Sourcing"); + bar.set_message(label); + } + + fn on_parsed(&self, file: &Path, raw: Option<&str>) { + let label = self.friendly_label(raw.unwrap_or(""), Some(file)); + let bucket = self.bucket_of(&label); + if !self.stdout_is_tty { + println!("{:>12} {label}", "Checking"); + return; + } + let bar = self.bar_for(&bucket); + bar.set_prefix("Checking"); + bar.set_message(label); + } + + fn on_dep_completed(&self, dep_name: &str) { + use colored::Colorize; + if !self.stdout_is_tty { + return; + } + // Print the committed row THROUGH the MultiProgress. Using + // `multi.println` inserts the text ABOVE the bar area so + // the row settles into normal scrollback flow. + let prefix = "Checking".bright_green().bold(); + let _ = self.multi.println(format!("{:>12} {}", prefix, dep_name)); + // Now remove the bar from the display so it doesn't double- + // render on the next tick. + let mut guard = self.bars.lock().unwrap(); + if let Some(pos) = guard.iter().position(|(n, _)| n == dep_name) { + let (_, bar) = guard.remove(pos); + self.multi.remove(&bar); + } + } +} + +impl MultiProgressObserver { + /// Finalize any bar that wasn't committed by an + /// [`on_dep_completed`] — the local-workspace bar covering + /// non-`@dep` files, and the entry file's own bar — by + /// routing each row through [`MultiProgress::println`] so it + /// enters the normal output stream ABOVE the (now-cleared) + /// bar area. + pub fn finish(&self) { + use colored::Colorize; + if !self.stdout_is_tty { + return; + } + let mut guard = self.bars.lock().unwrap(); + let prefix = "Checking".bright_green().bold(); + // Snapshot the remaining bar names in the current order so + // we can commit + remove without mutating the vec mid-scan. + let remaining: Vec = + guard.iter().map(|(n, _)| n.clone()).collect(); + for name in remaining { + let _ = self.multi.println(format!("{:>12} {}", prefix, name)); + if let Some(pos) = guard.iter().position(|(n, _)| n == &name) { + let (_, bar) = guard.remove(pos); + self.multi.remove(&bar); + } + } + let _ = self.multi.clear(); + } +} diff --git a/vw-cli/src/part_picker.rs b/vw-cli/src/part_picker.rs new file mode 100644 index 0000000..4efe0a0 --- /dev/null +++ b/vw-cli/src/part_picker.rs @@ -0,0 +1,291 @@ +//! Interactive fuzzy picker for selecting an FPGA part from the +//! catalog that ships with the current Vivado install. Invoked by +//! `vw init` when no `--part` flag is passed and stdin is a tty. + +use crossterm::{ + event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}, + execute, + terminal::{ + disable_raw_mode, enable_raw_mode, EnterAlternateScreen, + LeaveAlternateScreen, + }, +}; +use ratatui::{ + layout::{Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, ListState, Paragraph}, + Terminal, +}; +use std::io; +use vw_lib::parts::{matches_query, PartEntry, PartSeries}; + +/// Run the picker and return the selected part id, or `None` if the +/// user cancelled (Esc / Ctrl-C). +pub fn pick_part(parts: &[PartEntry]) -> io::Result> { + if parts.is_empty() { + return Ok(None); + } + + let mut stdout = io::stdout(); + enable_raw_mode()?; + execute!(stdout, EnterAlternateScreen)?; + let backend = ratatui::backend::CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + let mut state = PickerState::new(parts); + let result = loop { + terminal.draw(|f| render(f, &mut state))?; + match event::read()? { + Event::Key(key) if key.kind == KeyEventKind::Press => { + match (key.code, key.modifiers) { + (KeyCode::Esc, _) + | (KeyCode::Char('c'), KeyModifiers::CONTROL) => { + break Ok(None) + } + (KeyCode::Enter, _) => { + break Ok(state.selected_id().map(str::to_owned)) + } + (KeyCode::Tab, _) => state.cycle_family_forward(), + (KeyCode::BackTab, _) => state.cycle_family_backward(), + (KeyCode::Backspace, _) => { + state.query.pop(); + state.refilter(); + } + (KeyCode::Char(c), m) + if !m.contains(KeyModifiers::CONTROL) => + { + state.query.push(c); + state.refilter(); + } + (KeyCode::Up, _) => state.select_prev(), + (KeyCode::Down, _) => state.select_next(), + (KeyCode::PageUp, _) => { + for _ in 0..10 { + state.select_prev(); + } + } + (KeyCode::PageDown, _) => { + for _ in 0..10 { + state.select_next(); + } + } + (KeyCode::Home, _) => state.select_first(), + (KeyCode::End, _) => state.select_last(), + _ => {} + } + } + Event::Resize(_, _) => {} + _ => {} + } + }; + + disable_raw_mode()?; + execute!(terminal.backend_mut(), LeaveAlternateScreen)?; + terminal.show_cursor()?; + result +} + +struct PickerState<'a> { + all: &'a [PartEntry], + query: String, + /// `None` = All families. + family: Option, + filtered: Vec, + list: ListState, +} + +impl<'a> PickerState<'a> { + fn new(all: &'a [PartEntry]) -> Self { + let mut s = Self { + all, + query: String::new(), + family: None, + filtered: Vec::new(), + list: ListState::default(), + }; + s.refilter(); + s + } + + fn refilter(&mut self) { + self.filtered = self + .all + .iter() + .enumerate() + .filter(|(_, p)| { + self.family.is_none_or(|f| p.series == f) + && matches_query(&p.id, &self.query) + }) + .map(|(i, _)| i) + .collect(); + self.list.select((!self.filtered.is_empty()).then_some(0)); + } + + fn selected_id(&self) -> Option<&str> { + let idx = self.list.selected()?; + let src = *self.filtered.get(idx)?; + Some(self.all[src].id.as_str()) + } + + fn select_next(&mut self) { + if self.filtered.is_empty() { + return; + } + let cur = self.list.selected().unwrap_or(0); + self.list + .select(Some((cur + 1).min(self.filtered.len() - 1))); + } + + fn select_prev(&mut self) { + if self.filtered.is_empty() { + return; + } + let cur = self.list.selected().unwrap_or(0); + self.list.select(Some(cur.saturating_sub(1))); + } + + fn select_first(&mut self) { + self.list.select((!self.filtered.is_empty()).then_some(0)); + } + + fn select_last(&mut self) { + if !self.filtered.is_empty() { + self.list.select(Some(self.filtered.len() - 1)); + } + } + + fn family_label(&self) -> String { + match self.family { + None => "All".to_string(), + Some(s) => s.label().to_string(), + } + } + + fn cycle_family_forward(&mut self) { + let seq = family_cycle(); + let idx = seq.iter().position(|x| *x == self.family).unwrap_or(0); + self.family = seq[(idx + 1) % seq.len()]; + self.refilter(); + } + + fn cycle_family_backward(&mut self) { + let seq = family_cycle(); + let idx = seq.iter().position(|x| *x == self.family).unwrap_or(0); + self.family = seq[(idx + seq.len() - 1) % seq.len()]; + self.refilter(); + } +} + +/// Full family-chip cycle order: `All` first, then each series in +/// `PartSeries::all()` order. +fn family_cycle() -> Vec> { + let mut v = vec![None]; + v.extend(PartSeries::all().into_iter().map(Some)); + v +} + +fn render(f: &mut ratatui::Frame<'_>, state: &mut PickerState<'_>) { + let area = f.area(); + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(3), // search box + Constraint::Min(1), // results + Constraint::Length(1), // help + ]) + .split(area); + + render_search(f, chunks[0], state); + render_results(f, chunks[1], state); + render_help(f, chunks[2]); +} + +fn render_search( + f: &mut ratatui::Frame<'_>, + area: Rect, + state: &PickerState<'_>, +) { + let family = state.family_label(); + let chip = format!(" [Family: {family}] "); + let query_line = Line::from(vec![ + Span::styled("> ", Style::default().fg(Color::Cyan)), + Span::raw(&state.query), + Span::styled("_", Style::default().add_modifier(Modifier::SLOW_BLINK)), + ]); + let block = Block::default() + .borders(Borders::ALL) + .title(Line::from(vec![ + Span::raw(" Search parts "), + Span::styled( + format!("({} matches)", state.filtered.len()), + Style::default().fg(Color::DarkGray), + ), + ])) + .title_alignment(ratatui::layout::Alignment::Left) + .title_bottom( + Line::from(chip).alignment(ratatui::layout::Alignment::Right), + ); + f.render_widget(Paragraph::new(query_line).block(block), area); +} + +fn render_results( + f: &mut ratatui::Frame<'_>, + area: Rect, + state: &mut PickerState<'_>, +) { + if state.filtered.is_empty() { + let msg = if state.all.is_empty() { + "No parts found — Vivado install not detected." + } else { + "No matches. Showing only families installed with Vivado; pass --part for others." + }; + let block = Block::default().borders(Borders::ALL); + f.render_widget( + Paragraph::new(msg) + .style(Style::default().fg(Color::DarkGray)) + .block(block), + area, + ); + return; + } + let items: Vec = state + .filtered + .iter() + .map(|&i| { + let p = &state.all[i]; + let series = Span::styled( + format!(" {:<11}", p.series.label()), + Style::default().fg(Color::DarkGray), + ); + ListItem::new(Line::from(vec![Span::raw(&p.id), series])) + }) + .collect(); + let list = List::new(items) + .block(Block::default().borders(Borders::ALL)) + .highlight_style( + Style::default() + .bg(Color::Blue) + .fg(Color::White) + .add_modifier(Modifier::BOLD), + ) + .highlight_symbol("▶ "); + f.render_stateful_widget(list, area, &mut state.list); +} + +fn render_help(f: &mut ratatui::Frame<'_>, area: Rect) { + let help = Line::from(vec![ + Span::styled(" Tab", Style::default().fg(Color::Yellow)), + Span::raw(": family "), + Span::styled("↑↓/PgUp/PgDn", Style::default().fg(Color::Yellow)), + Span::raw(": navigate "), + Span::styled("Enter", Style::default().fg(Color::Yellow)), + Span::raw(": select "), + Span::styled("Esc", Style::default().fg(Color::Yellow)), + Span::raw(": cancel"), + ]); + f.render_widget( + Paragraph::new(help).style(Style::default().fg(Color::DarkGray)), + area, + ); +} diff --git a/vw-cli/src/test_ui.rs b/vw-cli/src/test_ui.rs new file mode 100644 index 0000000..de05e79 --- /dev/null +++ b/vw-cli/src/test_ui.rs @@ -0,0 +1,166 @@ +//! Shared cargo-nextest-style runner UI, used by both `vw test` (HTCL) and +//! `vw bench` (VHDL/cosim testbenches) so the two look identical. +//! +//! [`NextestPanel`] drives an [`indicatif::MultiProgress`] live display: a +//! `Running` progress bar with running/passed/failed counts, one live row +//! per in-flight test (ticking elapsed), and completed `PASS`/`FAIL` lines +//! scrolling above the panel. On a non-TTY the panel is hidden and only the +//! permanent `PASS`/`FAIL` lines and the summary survive. + +use std::io::IsTerminal; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use colored::*; +use indicatif::{ + MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle, +}; + +/// Format a duration the way nextest does: right-aligned seconds with +/// millisecond precision, e.g. `" 1.234s"`. +pub fn format_secs(secs: f64) -> String { + format!("{secs:>6.3}s") +} + +/// Print the final `test result: ok/FAILED. N passed; M failed; finished +/// in T` line shared by both runners. +pub fn print_result_line( + passed: usize, + failed: usize, + elapsed: std::time::Duration, +) { + let outcome = if failed == 0 { + "ok".green().bold().to_string() + } else { + "FAILED".red().bold().to_string() + }; + println!( + "\ntest result: {outcome}. {} passed; {} failed; finished in {}", + passed.to_string().green(), + failed.to_string().red(), + format_secs(elapsed.as_secs_f64()), + ); +} + +/// A cargo-nextest-style live panel. Thread-safe: `&self` methods can be +/// called concurrently from parallel runner tasks (share it via `Arc`). +pub struct NextestPanel { + multi: MultiProgress, + bar: ProgressBar, + is_tty: bool, + running: AtomicUsize, + passed: AtomicUsize, + failed: AtomicUsize, +} + +impl NextestPanel { + /// Set up the panel for `total` items. `noun` is the plural label for + /// the `running N ` header (e.g. `"tests"`, `"testbenches"`). + pub fn new(total: u64, noun: &str) -> Self { + println!("\nrunning {} {noun}", total.to_string().bold()); + let is_tty = std::io::stderr().is_terminal(); + let multi = MultiProgress::new(); + if !is_tty { + multi.set_draw_target(ProgressDrawTarget::hidden()); + } + let bar = multi.add(ProgressBar::new(total)); + bar.set_style( + ProgressStyle::with_template( + "{prefix:>12.cyan.bold} [{elapsed_precise}] \ + {wide_bar:.cyan/blue} {pos}/{len}: {msg}", + ) + .expect("valid template") + .progress_chars("█▉▊▋▌▍▎▏ "), + ); + bar.set_prefix("Running"); + bar.enable_steady_tick(Duration::from_millis(120)); + let panel = Self { + multi, + bar, + is_tty, + running: AtomicUsize::new(0), + passed: AtomicUsize::new(0), + failed: AtomicUsize::new(0), + }; + panel.refresh_counts(); + panel + } + + fn refresh_counts(&self) { + self.bar.set_message(format!( + "{} running, {} passed, {} failed", + self.running.load(Ordering::SeqCst), + self.passed.load(Ordering::SeqCst), + self.failed.load(Ordering::SeqCst), + )); + } + + /// Register a test as started; returns its live row (which ticks its + /// own elapsed until [`finish`](Self::finish) is called). + pub fn start(&self, label: &str) -> ProgressBar { + let row = self.multi.add(ProgressBar::new_spinner()); + row.set_style( + ProgressStyle::with_template( + " [{elapsed_precise}] {msg}", + ) + .expect("valid template"), + ); + row.set_message(label.to_string()); + row.enable_steady_tick(Duration::from_millis(120)); + self.running.fetch_add(1, Ordering::SeqCst); + self.refresh_counts(); + row + } + + /// Mark a test complete: clear its row, print a `PASS`/`FAIL` line above + /// the panel, and advance the bar + counts. + pub fn finish( + &self, + row: &ProgressBar, + label: &str, + passed: bool, + secs: f64, + ) { + row.finish_and_clear(); + self.multi.remove(row); + self.running.fetch_sub(1, Ordering::SeqCst); + if passed { + self.passed.fetch_add(1, Ordering::SeqCst); + } else { + self.failed.fetch_add(1, Ordering::SeqCst); + } + // Right-align PASS/FAIL to 12 cols so it lines up under `Running`. + let word = format!("{:>12}", if passed { "PASS" } else { "FAIL" }); + let status = if passed { + word.green().bold() + } else { + word.red().bold() + }; + self.println(format!("{status} [{}] {}", format_secs(secs), label)); + self.bar.inc(1); + self.refresh_counts(); + } + + /// Print a line above the live panel (plain `println!` on a non-TTY). + pub fn println(&self, line: String) { + if self.is_tty { + let _ = self.multi.println(line); + } else { + println!("{line}"); + } + } + + pub fn passed(&self) -> usize { + self.passed.load(Ordering::SeqCst) + } + + pub fn failed(&self) -> usize { + self.failed.load(Ordering::SeqCst) + } + + /// Tear the live bar down at the end of the run. + pub fn clear(&self) { + self.bar.finish_and_clear(); + self.multi.remove(&self.bar); + } +} diff --git a/vw-core/Cargo.toml b/vw-core/Cargo.toml new file mode 100644 index 0000000..c2f7c7e --- /dev/null +++ b/vw-core/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "vw-core" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Core VHDL analysis and nvc primitives shared by vw-lib and anodizer" +keywords = ["vhdl", "nvc"] +categories = ["development-tools"] + +[dependencies] +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +tokio.workspace = true +regex.workspace = true +dirs.workspace = true +petgraph.workspace = true +vhdl_lang.workspace = true diff --git a/vw-core/src/lib.rs b/vw-core/src/lib.rs new file mode 100644 index 0000000..b112269 --- /dev/null +++ b/vw-core/src/lib.rs @@ -0,0 +1,780 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `vw-core`: low-level VHDL analysis and nvc-runner primitives shared by +//! `vw-lib` (the workflow layer) and `anodizer` (VHDL-record code generation). +//! +//! This crate deliberately has no dependency on git/deps-resolution or any +//! other workflow machinery so that `anodizer` can depend on it without +//! creating a cycle back into `vw-lib`. + +use std::collections::{hash_map::Entry, HashMap, HashSet, VecDeque}; +use std::path::{Path, PathBuf}; +use std::{fmt, fs}; + +use serde::{Deserialize, Serialize}; +use vhdl_lang::{VHDLParser, VHDLStandard}; + +use petgraph::{ + algo::toposort, + graph::{DiGraph, NodeIndex}, +}; + +use crate::mapping::{FileData, SymbolKind, VwSymbol, VwSymbolFinder}; +use crate::nvc_helpers::run_nvc_analysis; +use crate::visitor::walk_design_file; + +pub mod mapping; +pub mod nvc_helpers; +pub mod visitor; + +// ============================================================================ +// Error Types +// ============================================================================ + +#[derive(Debug)] +pub enum VwError { + Config { message: String }, + Dependency { message: String }, + Git { message: String }, + FileSystem { message: String }, + Testbench { message: String }, + NvcSimulation { command: String }, + NvcElab { command: String }, + NvcAnalysis { library: String, command: String }, + CodeGen { message: String }, + Simulation { message: String }, + Io(std::io::Error), + Serialization(toml::ser::Error), + Deserialization(toml::de::Error), + Regex(regex::Error), +} + +impl std::error::Error for VwError {} +impl From for VwError { + fn from(err: std::io::Error) -> Self { + VwError::Io(err) + } +} + +impl From for VwError { + fn from(err: toml::ser::Error) -> Self { + VwError::Serialization(err) + } +} + +impl From for VwError { + fn from(err: toml::de::Error) -> Self { + VwError::Deserialization(err) + } +} + +impl From for VwError { + fn from(err: regex::Error) -> Self { + VwError::Regex(err) + } +} + +pub type Result = std::result::Result; + +impl fmt::Display for VwError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + VwError::NvcSimulation { command } => { + writeln!(f, "NVC simulation failed")?; + writeln!(f, "command:")?; + writeln!(f, "{command}")?; + Ok(()) + } + VwError::NvcElab { command } => { + writeln!(f, "NVC elaboration failed")?; + writeln!(f, "command:")?; + writeln!(f, "{command}")?; + Ok(()) + } + VwError::NvcAnalysis { library, command } => { + writeln!(f, "NVC analysis failed for library '{library}'")?; + writeln!(f, "command:")?; + writeln!(f, "{command}")?; + Ok(()) + } + VwError::CodeGen { message } => { + write!(f, "Code generation failed: {message}") + } + VwError::Simulation { message } => { + write!(f, "Simulation error: {message}") + } + VwError::Config { message } => { + write!(f, "Configuration error: {message}") + } + VwError::Dependency { message } => { + write!(f, "Dependency error: {message}") + } + VwError::Git { message } => { + write!(f, "Git operation failed: {message}") + } + VwError::FileSystem { message } => { + write!(f, "File system error: {message}") + } + VwError::Testbench { message } => { + write!(f, "Testbench error: {message}") + } + VwError::Io(e) => write!(f, "IO error: {e}"), + VwError::Serialization(e) => write!(f, "Serialization error: {e}"), + VwError::Deserialization(e) => { + write!(f, "Deserialization error: {e}") + } + VwError::Regex(e) => write!(f, "Regex error: {e}"), + } + } +} + +// ============================================================================ +// VHDL Standard +// ============================================================================ + +#[derive(Clone, Copy, Debug)] +pub enum VhdlStandard { + Vhdl2008, + Vhdl2019, +} + +impl From for VHDLStandard { + fn from(val: VhdlStandard) -> Self { + match val { + VhdlStandard::Vhdl2008 => VHDLStandard::VHDL2008, + VhdlStandard::Vhdl2019 => VHDLStandard::VHDL2019, + } + } +} + +/// The inverse of [`VhdlStandard`]'s `Display`. +/// +/// Paired with it deliberately: the standard is written into command lines and +/// now also sent to an instance that has to turn it back into the same value, +/// and a spelling that only goes one way is a spelling that eventually +/// disagrees with itself. +impl std::str::FromStr for VhdlStandard { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + match s.trim() { + "2008" => Ok(VhdlStandard::Vhdl2008), + "2019" => Ok(VhdlStandard::Vhdl2019), + other => Err(format!( + "'{other}' is not a vhdl standard vw knows; expected 2008 or \ + 2019" + )), + } + } +} + +impl fmt::Display for VhdlStandard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + VhdlStandard::Vhdl2008 => write!(f, "2008"), + VhdlStandard::Vhdl2019 => write!(f, "2019"), + } + } +} +#[derive(Debug, Serialize, Deserialize)] +pub struct VhdlLsConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub standard: Option, + pub libraries: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub lint: Option>, +} +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct VhdlLsLibrary { + pub files: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub exclude: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_third_party: Option, +} +pub struct RecordProcessor { + pub vhdl_std: VhdlStandard, + pub symbols: HashMap, + pub symbol_to_file: HashMap, + pub tagged_names: HashSet, + pub file_info: HashMap, + pub target_attr: String, +} + +const RECORD_PARSE_ATTRIBUTE: &str = "serialize_rust"; +impl RecordProcessor { + pub fn new(std: VhdlStandard) -> Self { + Self { + vhdl_std: std, + symbols: HashMap::new(), + symbol_to_file: HashMap::new(), + tagged_names: HashSet::new(), + file_info: HashMap::new(), + target_attr: RECORD_PARSE_ATTRIBUTE.to_string(), + } + } +} + +// ============================================================================ +// File Cache - Reduces redundant file reads during build +// ============================================================================ + +/// Cache for parsed file data to avoid redundant parsing during builds. +/// Only caches parsed results, not raw file contents. +pub struct FileCache { + dependencies: HashMap>, + provided_symbols: HashMap>, + entities: HashMap>, +} + +impl FileCache { + pub fn new() -> Self { + Self { + dependencies: HashMap::new(), + provided_symbols: HashMap::new(), + entities: HashMap::new(), + } + } + + /// Get cached file dependencies, reading and parsing file if not cached. + pub fn get_dependencies(&mut self, path: &Path) -> Result<&Vec> { + match self.dependencies.entry(path.to_path_buf()) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let content = fs::read_to_string(path).map_err(|e| { + VwError::FileSystem { + message: format!("Failed to read file {path:?}: {e}"), + } + })?; + let deps = parse_file_dependencies(&content)?; + Ok(e.insert(deps)) + } + } + } + + /// Get cached provided symbols (packages and entities), reading and parsing if not cached. + pub fn get_provided_symbols( + &mut self, + path: &Path, + ) -> Result<&Vec> { + match self.provided_symbols.entry(path.to_path_buf()) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let content = fs::read_to_string(path).map_err(|e| { + VwError::FileSystem { + message: format!("Failed to read file {path:?}: {e}"), + } + })?; + let symbols = parse_provided_symbols(&content)?; + Ok(e.insert(symbols)) + } + } + } + + /// Get cached entities in file, reading and parsing if not cached. + pub fn get_entities(&mut self, path: &Path) -> Result<&Vec> { + match self.entities.entry(path.to_path_buf()) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let content = fs::read_to_string(path).map_err(|e| { + VwError::FileSystem { + message: format!("Failed to read file {path:?}: {e}"), + } + })?; + let entities = parse_entities(&content)?; + Ok(e.insert(entities)) + } + } + } + + /// Get mutable access to the entities cache for functions that only need entity lookups. + pub fn entities_cache_mut(&mut self) -> &mut HashMap> { + &mut self.entities + } +} + +impl Default for FileCache { + fn default() -> Self { + Self::new() + } +} + +/// Parse dependencies from file content (extracted for use by FileCache). +fn parse_file_dependencies(content: &str) -> Result> { + let mut dependencies = Vec::new(); + let mut seen = HashSet::new(); + + // Package imports from "use work.package_name" + let imports = get_package_imports(content)?; + for pkg in imports { + let key = format!("pkg:{}", pkg.to_lowercase()); + if seen.insert(key) { + dependencies.push(VwSymbol::new(None, &pkg, SymbolKind::Package)); + } + } + + // Find direct entity instantiations (instance_name: entity work.entity_name) + let entity_inst_pattern = r"(?i)\w+\s*:\s*entity\s+work\.(\w+)"; + let entity_inst_re = regex::Regex::new(entity_inst_pattern)?; + + for captures in entity_inst_re.captures_iter(content) { + if let Some(entity_name) = captures.get(1) { + let name = entity_name.as_str().to_string(); + let key = format!("ent:{}", name.to_lowercase()); + if seen.insert(key) { + dependencies.push(VwSymbol::new( + None, + &name, + SymbolKind::Entity, + )); + } + } + } + + // Find component declarations + let comp_decl_pattern = r"(?i)component\s+(\w+)"; + let comp_decl_re = regex::Regex::new(comp_decl_pattern)?; + + for captures in comp_decl_re.captures_iter(content) { + if let Some(comp_name) = captures.get(1) { + let name = comp_name.as_str().to_string(); + let key = format!("ent:{}", name.to_lowercase()); + if seen.insert(key) { + dependencies.push(VwSymbol::new( + None, + &name, + SymbolKind::Entity, + )); + } + } + } + + Ok(dependencies) +} + +/// Parse provided symbols (packages and entities) from file content. +fn parse_provided_symbols(content: &str) -> Result> { + let mut symbols = Vec::new(); + + // Find package declarations + let package_pattern = r"(?i)\bpackage\s+(\w+)\s+is\b"; + let package_re = regex::Regex::new(package_pattern)?; + + for captures in package_re.captures_iter(content) { + if let Some(package_name) = captures.get(1) { + symbols.push(VwSymbol::new( + None, + package_name.as_str(), + SymbolKind::Package, + )); + } + } + + // Find entity declarations + let entity_pattern = r"(?i)\bentity\s+(\w+)\s+is\b"; + let entity_re = regex::Regex::new(entity_pattern)?; + + for captures in entity_re.captures_iter(content) { + if let Some(entity_name) = captures.get(1) { + symbols.push(VwSymbol::new( + None, + entity_name.as_str(), + SymbolKind::Entity, + )); + } + } + + Ok(symbols) +} + +/// Parse entity declarations from file content. +pub fn parse_entities(content: &str) -> Result> { + let mut entities = Vec::new(); + + let entity_pattern = r"(?i)\bentity\s+(\w+)\s+is\b"; + let re = regex::Regex::new(entity_pattern)?; + + for captures in re.captures_iter(content) { + if let Some(entity_name) = captures.get(1) { + entities.push(entity_name.as_str().to_string()); + } + } + + Ok(entities) +} + +pub async fn analyze_ext_libraries( + vhdl_ls_config: &VhdlLsConfig, + processor: &mut RecordProcessor, + vhdl_std: VhdlStandard, + build_dir: &str, + cache: &mut FileCache, +) -> Result<()> { + // Collect non-defaultlib library names + let ext_lib_names: Vec = vhdl_ls_config + .libraries + .keys() + .filter(|k| k.as_str() != "defaultlib") + .cloned() + .collect(); + + // Build inter-library dependency graph by scanning for `library ;` + let ext_lib_set: HashSet = ext_lib_names.iter().cloned().collect(); + let mut lib_deps: HashMap> = HashMap::new(); + for lib_name in &ext_lib_names { + let mut deps = Vec::new(); + if let Some(library) = vhdl_ls_config.libraries.get(lib_name) { + for file_path in &library.files { + let expanded = if file_path.starts_with("$HOME") { + if let Some(home) = dirs::home_dir() { + home.join( + file_path + .strip_prefix("$HOME/") + .unwrap_or(file_path), + ) + } else { + PathBuf::from(file_path) + } + } else { + PathBuf::from(file_path) + }; + if let Ok(contents) = fs::read_to_string(&expanded) { + for line in contents.lines() { + let trimmed = line.trim().to_lowercase(); + if let Some(rest) = trimmed.strip_prefix("library ") { + let dep_lib = rest.trim_end_matches(';').trim(); + if ext_lib_set.contains(dep_lib) + && dep_lib != lib_name.to_lowercase() + { + deps.push(dep_lib.to_string()); + } + } + } + } + } + } + lib_deps.insert(lib_name.clone(), deps); + } + + // Topological sort of library names (Kahn's algorithm) + let mut in_degree: HashMap = + ext_lib_names.iter().map(|n| (n.clone(), 0)).collect(); + let mut adj: HashMap> = ext_lib_names + .iter() + .map(|n| (n.clone(), Vec::new())) + .collect(); + for (lib, deps) in &lib_deps { + for dep in deps { + if let Some(neighbors) = adj.get_mut(dep) { + neighbors.push(lib.clone()); + } + if let Some(deg) = in_degree.get_mut(lib) { + *deg += 1; + } + } + } + let mut queue: VecDeque = in_degree + .iter() + .filter(|(_, &d)| d == 0) + .map(|(n, _)| n.clone()) + .collect(); + let mut sorted_libs = Vec::new(); + while let Some(current) = queue.pop_front() { + sorted_libs.push(current.clone()); + if let Some(neighbors) = adj.get(¤t) { + for neighbor in neighbors { + if let Some(deg) = in_degree.get_mut(neighbor) { + *deg -= 1; + if *deg == 0 { + queue.push_back(neighbor.clone()); + } + } + } + } + } + // Fall back to unsorted if cycle detected + if sorted_libs.len() != ext_lib_names.len() { + sorted_libs = ext_lib_names; + } + + // Analyze libraries in dependency order + for lib_name in &sorted_libs { + if let Some(library) = vhdl_ls_config.libraries.get(lib_name) { + // Convert library name to be NVC-compatible (no hyphens) + let nvc_lib_name = lib_name.replace('-', "_"); + + let mut files = Vec::new(); + for file_path in &library.files { + // Convert $HOME paths to absolute paths + let expanded_path = if file_path.starts_with("$HOME") { + let home_dir = dirs::home_dir().ok_or_else(|| { + VwError::FileSystem { + message: "Could not determine home directory" + .to_string(), + } + })?; + home_dir.join( + file_path.strip_prefix("$HOME/").unwrap_or(file_path), + ) + } else { + PathBuf::from(file_path) + }; + files.push(expanded_path); + } + + // Sort files in dependency order (dependencies first) + sort_files_by_dependencies(processor, &mut files, cache)?; + + let file_strings: Vec = files + .iter() + .map(|p| p.to_string_lossy().to_string()) + .collect(); + + run_nvc_analysis( + vhdl_std, + build_dir, + &nvc_lib_name, + &file_strings, + false, + ) + .await?; + } + } + + Ok(()) +} +pub fn find_referenced_files( + testbench_file: &Path, + available_files: &[PathBuf], + cache: &mut FileCache, +) -> Result> { + let mut referenced_files = Vec::new(); + let mut processed_files = HashSet::new(); + let mut files_to_process = vec![testbench_file.to_path_buf()]; + + while let Some(current_file) = files_to_process.pop() { + if processed_files.contains(¤t_file) { + continue; + } + processed_files.insert(current_file.clone()); + + // Don't include the testbench file itself in the referenced files + // (it will be added separately) + if current_file != testbench_file { + referenced_files.push(current_file.clone()); + } + + let dependencies = cache.get_dependencies(¤t_file)?.clone(); + + // Find corresponding files for each dependency + for dep in dependencies { + for available_file in available_files { + if file_provides_symbol(available_file, &dep, cache)? { + if !processed_files.contains(available_file) { + files_to_process.push(available_file.clone()); + } + break; + } + } + } + } + + Ok(referenced_files) +} + +pub fn sort_files_by_dependencies( + processor: &mut RecordProcessor, + files: &mut Vec, + cache: &mut FileCache, +) -> Result<()> { + // Build dependency graph + let mut dependencies: HashMap> = HashMap::new(); + let mut all_symbols: HashMap = HashMap::new(); + + // First pass: collect all symbols provided by each file + for file in files.iter() { + let symbols = analyze_file(processor, file)?; + for symbol in symbols { + match &symbol.kind { + SymbolKind::Package => { + all_symbols.insert(symbol.name.clone(), file.clone()); + let entry = processor + .file_info + .entry(file.to_string_lossy().to_string()) + .or_default(); + entry.add_defined_pkg(&symbol.name); + + // Use cache to get package imports only + let deps = cache.get_dependencies(file)?; + for dep in deps { + if let SymbolKind::Package = dep.kind { + entry.add_imported_pkg(&dep.name); + } + } + } + SymbolKind::Entity => { + all_symbols.insert(symbol.name, file.clone()); + } + _ => {} + } + } + } + + // Second pass: find dependencies for each file + for file in files.iter() { + let deps = cache.get_dependencies(file)?.clone(); + let mut file_deps = Vec::new(); + + for dep in deps { + let dep_name = match &dep.kind { + SymbolKind::Package | SymbolKind::Entity => &dep.name, + _ => continue, + }; + if let Some(provider_file) = all_symbols.get(dep_name) { + if provider_file != file { + file_deps.push(provider_file.clone()); + } + } + } + + dependencies.insert(file.clone(), file_deps); + } + + // Topological sort using Kahn's algorithm + let sorted = topological_sort_files(files.clone(), dependencies)?; + *files = sorted; + + Ok(()) +} +// ============================================================================ +// Internal Helper Functions +// ============================================================================ + +fn get_package_imports(content: &str) -> Result> { + // Find 'use work.package_name' statements + let use_work_pattern = r"(?i)use\s+work\.(\w+)"; + let use_work_re = regex::Regex::new(use_work_pattern)?; + let mut imports = Vec::new(); + + for captures in use_work_re.captures_iter(content) { + if let Some(package_name) = captures.get(1) { + imports.push(package_name.as_str().to_string()); + } + } + Ok(imports) +} + +fn file_provides_symbol( + file_path: &Path, + needed: &VwSymbol, + cache: &mut FileCache, +) -> Result { + let provided = cache.get_provided_symbols(file_path)?; + Ok(provided.iter().any(|s| match (&needed.kind, &s.kind) { + // Package dependency matches package declaration + (SymbolKind::Package, SymbolKind::Package) => { + needed.name.eq_ignore_ascii_case(&s.name) + } + // Entity dependency matches entity declaration + (SymbolKind::Entity, SymbolKind::Entity) => { + needed.name.eq_ignore_ascii_case(&s.name) + } + _ => false, + })) +} + +fn analyze_file( + processor: &mut RecordProcessor, + file: &Path, +) -> Result> { + let parser = VHDLParser::new(processor.vhdl_std.into()); + let mut diagnostics = Vec::new(); + let (_, design_file) = parser.parse_design_file(file, &mut diagnostics)?; + + let mut file_finder = VwSymbolFinder::new(&processor.target_attr); + walk_design_file(&mut file_finder, &design_file); + + let file_str = file.to_string_lossy().to_string(); + + // Add symbols to the map + + for symbol in file_finder.get_symbols() { + match symbol.kind { + SymbolKind::Enum(_) + | SymbolKind::Record(_) + | SymbolKind::Constant(_) => { + let name = symbol.get_name().to_string(); + processor.symbols.insert(name.clone(), symbol.clone()); + processor.symbol_to_file.insert(name, file_str.clone()); + } + _ => {} + } + } + + for tagged_type in file_finder.get_tagged_types() { + processor.tagged_names.insert(tagged_type.clone()); + } + + Ok(file_finder.get_symbols().clone()) +} + +fn topological_sort_files( + files: Vec, + dependencies: HashMap>, +) -> Result> { + let mut dep_graph: DiGraph = DiGraph::default(); + let mut index_map: HashMap = HashMap::new(); + + // initialize the nodes + for file in &files { + let index = dep_graph.add_node(file.clone()); + index_map.insert(file.clone(), index); + } + + // now add edges from files to their dependencies + for (file, deps) in &dependencies { + let source_node = index_map.get(file).ok_or(VwError::Dependency { + message: format!( + "Index map somehow didn't contain file {:?}", + file + ), + })?; + // file depends on every dep in deps + for dep in deps { + let dst_node = index_map.get(dep).ok_or(VwError::Dependency { + message: format!( + "Index map somehow didn't contain dep {:?}", + dep + ), + })?; + dep_graph.add_edge(*source_node, *dst_node, ()); + } + } + + // ok now topological sort + let ordered_files = + toposort(&dep_graph, None).map_err(|_| VwError::Dependency { + message: "Got circular dependency".to_string(), + })?; + + let result: Vec = ordered_files + .iter() + .map(|&idx| dep_graph[idx].clone()) + .rev() + .collect(); + Ok(result) +} + +/// Parse an existing `vhdl_ls.toml` file into a [`VhdlLsConfig`]. +/// +/// A pure, disk-reading loader with no workspace/deps resolution — suitable +/// for the standalone `anodizer` CLI. The `vw` workflow renders a config in +/// memory instead (see `vw_lib::render_vhdl_ls_config`) and hands it to +/// `anodizer` directly. +pub fn load_existing_vhdl_ls_config(path: &Path) -> Result { + let contents = fs::read_to_string(path)?; + let config: VhdlLsConfig = toml::from_str(&contents)?; + Ok(config) +} diff --git a/vw-lib/src/mapping.rs b/vw-core/src/mapping.rs similarity index 93% rename from vw-lib/src/mapping.rs rename to vw-core/src/mapping.rs index 645827b..3bcb959 100644 --- a/vw-lib/src/mapping.rs +++ b/vw-core/src/mapping.rs @@ -199,8 +199,21 @@ impl Visitor for VwSymbolFinder { None }; - // figure out its expression - let expr = decl.expression.as_ref().map(|span| span.item.clone()); + // figure out its expression. VHDL 2019 widened the + // initializer to a `ConditionalExpression`; we only + // ever consumed the simple form here (record-field + // defaults etc.), so unwrap `Simple` and drop + // conditional forms as if the constant had no + // initializer. + let expr = + decl.expression.as_ref().and_then(|span| match &span.item { + vhdl_lang::ast::ConditionalExpression::Simple(e) => { + Some(e.clone()) + } + vhdl_lang::ast::ConditionalExpression::Conditional(_) => { + None + } + }); let type_name = decl.subtype_indication.type_mark.item.clone(); self.symbols.push(VwSymbol::new( diff --git a/vw-lib/src/nvc_helpers.rs b/vw-core/src/nvc_helpers.rs similarity index 94% rename from vw-lib/src/nvc_helpers.rs rename to vw-core/src/nvc_helpers.rs index b7ef851..8618b1d 100644 --- a/vw-lib/src/nvc_helpers.rs +++ b/vw-core/src/nvc_helpers.rs @@ -138,11 +138,13 @@ pub async fn run_nvc_elab( } } +#[allow(clippy::too_many_arguments)] pub async fn run_nvc_sim( std: VhdlStandard, build_dir: &str, lib_name: &str, testbench_name: &String, + wave_dir: &str, rust_lib_path: Option, runtime_flags: &Vec, capture_output: bool, @@ -157,7 +159,7 @@ pub async fn run_nvc_sim( args.push("--dump-arrays".to_string()); args.push("--format=fst".to_string()); - args.push(format!("--wave={testbench_name}.fst")); + args.push(format!("--wave={wave_dir}/{testbench_name}.fst")); let envs = match rust_lib_path { Some(path) => { @@ -200,6 +202,7 @@ pub async fn run_nvc_cosim( lib_name: &str, entity_name: &str, bridge_lib_path: &str, + output_dir: &str, capture_output: bool, ) -> Result, Vec)>, VwError> { let mut args = get_base_nvc_cmd_args(std, build_dir, lib_name); @@ -210,6 +213,9 @@ pub async fn run_nvc_cosim( let envs = vec![ ("GPI_USERS".to_string(), bridge_lib_path.to_string()), ("COCOTB_RUST_MODE".to_string(), "1".to_string()), + // Tells the bench where to write artifacts (Xyce `.prn`, etc.) via + // `rust_cosim::output_dir()`. Contract string == rust_cosim OUTPUT_DIR_ENV. + ("RUST_COSIM_OUTPUT_DIR".to_string(), output_dir.to_string()), ]; if capture_output { diff --git a/vw-lib/src/visitor.rs b/vw-core/src/visitor.rs similarity index 100% rename from vw-lib/src/visitor.rs rename to vw-core/src/visitor.rs diff --git a/vw-eda/Cargo.toml b/vw-eda/Cargo.toml new file mode 100644 index 0000000..f66da6c --- /dev/null +++ b/vw-eda/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vw-eda" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "EDA backend trait and wire protocol for driving vendor TCL interpreters (Vivado, Quartus, ...)" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +async-trait.workspace = true diff --git a/vw-eda/src/lib.rs b/vw-eda/src/lib.rs new file mode 100644 index 0000000..dd57f8f --- /dev/null +++ b/vw-eda/src/lib.rs @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! EDA backend abstraction. +//! +//! Defines the trait that every vendor-specific TCL worker implements +//! and the wire protocol used to talk to it. `vw-vivado` is the first +//! implementation; future `vw-quartus` / `vw-synopsys` crates will +//! implement the same trait, and consumers (`vw run`, `vw repl`, the +//! analyzer) talk only to this abstraction. +//! +//! The protocol is intentionally small: newline-delimited JSON +//! requests, one response per request, monotonic IDs. See the project +//! plan's "Wire protocol" section for the design rationale. + +pub mod protocol; +pub mod stream; + +use async_trait::async_trait; +use thiserror::Error; + +pub use protocol::{ + ErrorPayload, Request, RequestOp, Response, ResponseResult, StreamMessage, + WireMessage, +}; +pub use stream::{StdoutSink, StreamKind}; + +/// Errors returned by an [`EdaBackend`]. +#[derive(Debug, Error)] +pub enum BackendError { + /// The worker process exited or could not be started. + #[error("worker process error: {0}")] + Worker(String), + + /// I/O error while reading or writing the wire protocol. + #[error("wire I/O error: {0}")] + Io(#[from] std::io::Error), + + /// Wire protocol message could not be serialized or parsed. + #[error("wire protocol error: {0}")] + Protocol(#[from] serde_json::Error), + + /// The backend reported a TCL-level error in response to a command. + /// `stdout` carries any output the command produced before erroring, + /// so callers can show context. + #[error("TCL error: {message}")] + Tcl { + message: String, + code: Option, + info: Option, + stdout: String, + }, + + /// Catch-all for backend-specific failures. + #[error("{0}")] + Other(String), +} + +/// Result of an [`EdaBackend::eval`] call. +#[derive(Clone, Debug, Default)] +pub struct EvalOutput { + /// The TCL expression's return value, as a string. + pub value: String, + /// stdout captured during this eval (puts to stdout from the user + /// TCL while the shim's capturing flag was set). Always present + /// and may be empty; trailing newlines are preserved as written. + pub stdout: String, +} + +/// A long-lived TCL worker driven by `vw`. +/// +/// Implementations spawn the vendor process (Vivado, Quartus, ...), +/// inject a small shim that speaks the wire protocol, and translate +/// [`Request`]s into [`Response`]s. The trait is intentionally narrow: +/// callers issue commands, the backend runs them, and the protocol is +/// the contract. +#[async_trait] +pub trait EdaBackend: Send { + /// Human-readable backend name, e.g. `"vivado"`. + fn name(&self) -> &str; + + /// Evaluate a TCL command string and return its result plus any + /// stdout the command produced. + /// + /// Equivalent to issuing a [`RequestOp::Eval`] request and + /// extracting both the return value and the captured-puts payload. + /// Most callers should use this in preference to + /// [`EdaBackend::send`] until the structured-eval machinery + /// (phase 4) lands. + async fn eval(&mut self, tcl: &str) -> Result; + + /// Issue an arbitrary request and return the raw response. + /// + /// The default implementation in concrete backends is the + /// preferred place to add `eval_structured` and future ops without + /// changing the trait surface. + async fn send( + &mut self, + request: Request, + ) -> Result; + + /// Install a sink called once per chunk of output as it is produced, + /// rather than after the command finishes. + /// + /// On the trait rather than on one backend because it is the only way a + /// caller can show a long command's progress, and a caller should not have + /// to know which backend it is driving to get that. With a sink set, + /// chunks are not also accumulated into [`EvalOutput::stdout`] — the sink + /// owns them. + fn set_stdout_sink(&mut self, sink: StdoutSink); + + /// Cleanly shut the worker down. Backends should make this + /// idempotent so that `Drop` can fall back to it without + /// double-shutdown errors. + async fn shutdown(&mut self) -> Result<(), BackendError>; +} diff --git a/vw-eda/src/protocol.rs b/vw-eda/src/protocol.rs new file mode 100644 index 0000000..0559eec --- /dev/null +++ b/vw-eda/src/protocol.rs @@ -0,0 +1,244 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Newline-delimited JSON wire protocol between `vw` and a vendor +//! TCL worker. +//! +//! v0 implements the `eval` op only. The `eval_structured` op (Phase 4 +//! of the project plan) will land as an additional [`RequestOp`] +//! variant without breaking the wire format. + +use serde::{Deserialize, Serialize}; + +/// A request sent from `vw` to the worker. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Request { + /// Monotonic request id chosen by the sender. The worker echoes + /// it in the matching [`Response`]. + pub id: u64, + #[serde(flatten)] + pub op: RequestOp, +} + +/// The operation a [`Request`] performs. +/// +/// Serialized with `op` as the discriminator (`{"op": "eval", "tcl": +/// "..."}`), matching the project plan's spec. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum RequestOp { + /// Evaluate a TCL command in the worker's interpreter and return + /// the result as a string. + Eval { tcl: String }, + /// Cleanly shut the worker down. Issued by [`crate::EdaBackend::shutdown`]. + Shutdown, +} + +/// A response from the worker for a single request. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Response { + pub id: u64, + #[serde(flatten)] + pub result: ResponseResult, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ResponseResult { + Ok { + ok: OkMarker, + #[serde(default)] + result: serde_json::Value, + }, + Err { + ok: ErrMarker, + error: ErrorPayload, + }, +} + +/// Streaming notification emitted by the worker between request and +/// response. `puts` writes from inside an eval are forwarded as these +/// so callers can show output live rather than waiting for the eval +/// to complete (necessary for any long-running synthesis or +/// implementation command). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StreamMessage { + /// Id of the in-flight request this stream chunk belongs to. + pub id: u64, + /// `"stdout"` today; reserved for `"stderr"` etc. later. + pub stream: String, + /// The chunk's bytes, including any trailing newline as written. + pub data: String, +} + +/// One wire-level message read from the worker. Either a streaming +/// chunk for an in-flight request, the request's final response, or +/// an unsolicited RPC call FROM the worker asking `vw` (Rust side) to +/// compute a value. Discriminated by structural inspection: +/// - stream chunks have a `stream` field, +/// - responses have `ok`, +/// - RPC calls have `rpc` set to `true` plus a `method` field. +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum WireMessage { + Stream(StreamMessage), + Response(Response), + Rpc(RpcCall), +} + +/// An RPC call FROM the worker (shim) TO `vw`. The shim's htcl +/// library uses this to reach Rust-implemented externs like +/// `vw::workspace_root` and `vw::design_sources` — anything whose +/// answer lives on the tool side, not in Vivado. +/// +/// Wire shape: `{"id": M, "rpc": true, "method": "...", "args": ...}` +/// The `rpc: true` marker keeps the untagged `WireMessage` union +/// unambiguous — plain responses have `ok`, streams have `stream`, +/// RPC calls have `rpc`. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RpcCall { + pub id: u64, + /// Always `true` — used as an untagged-enum discriminator. See + /// [`RpcMarker`] for the deserializer. + pub rpc: RpcMarker, + pub method: String, + #[serde(default)] + pub args: serde_json::Value, +} + +/// Marker that always serializes to the literal `true`. Same +/// pattern as [`OkMarker`] — distinguishes RPC calls from Responses +/// in the untagged `WireMessage` union. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct RpcMarker(#[serde(deserialize_with = "deserialize_true")] pub bool); + +impl RpcMarker { + pub const TRUE: RpcMarker = RpcMarker(true); +} + +/// Marker that always serializes to the literal `true`. Lets us use +/// the same `ok` field as a discriminator without a custom serializer. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct OkMarker(#[serde(deserialize_with = "deserialize_true")] pub bool); + +impl OkMarker { + pub const TRUE: OkMarker = OkMarker(true); +} + +fn deserialize_true<'de, D: serde::Deserializer<'de>>( + de: D, +) -> Result { + let v = bool::deserialize(de)?; + if v { + Ok(true) + } else { + Err(serde::de::Error::custom("expected `true`")) + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +pub struct ErrMarker(#[serde(deserialize_with = "deserialize_false")] pub bool); + +impl ErrMarker { + pub const FALSE: ErrMarker = ErrMarker(false); +} + +fn deserialize_false<'de, D: serde::Deserializer<'de>>( + de: D, +) -> Result { + let v = bool::deserialize(de)?; + if !v { + Ok(false) + } else { + Err(serde::de::Error::custom("expected `false`")) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ErrorPayload { + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub info: Option, +} + +impl Response { + pub fn ok(id: u64, result: serde_json::Value) -> Self { + Self { + id, + result: ResponseResult::Ok { + ok: OkMarker::TRUE, + result, + }, + } + } + + pub fn err(id: u64, error: ErrorPayload) -> Self { + Self { + id, + result: ResponseResult::Err { + ok: ErrMarker::FALSE, + error, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_eval_request() { + let req = Request { + id: 1, + op: RequestOp::Eval { + tcl: "puts hi".into(), + }, + }; + let s = serde_json::to_string(&req).unwrap(); + assert!(s.contains("\"op\":\"eval\"")); + assert!(s.contains("\"tcl\":\"puts hi\"")); + let back: Request = serde_json::from_str(&s).unwrap(); + match back.op { + RequestOp::Eval { tcl } => assert_eq!(tcl, "puts hi"), + _ => panic!(), + } + } + + #[test] + fn round_trip_ok_response() { + let r = Response::ok(7, serde_json::json!("hi")); + let s = serde_json::to_string(&r).unwrap(); + let back: Response = serde_json::from_str(&s).unwrap(); + match back.result { + ResponseResult::Ok { result, .. } => { + assert_eq!(result, serde_json::json!("hi")) + } + _ => panic!(), + } + } + + #[test] + fn round_trip_err_response() { + let r = Response::err( + 8, + ErrorPayload { + message: "boom".into(), + code: Some("E1".into()), + info: None, + }, + ); + let s = serde_json::to_string(&r).unwrap(); + let back: Response = serde_json::from_str(&s).unwrap(); + match back.result { + ResponseResult::Err { error, .. } => { + assert_eq!(error.message, "boom"); + assert_eq!(error.code.as_deref(), Some("E1")); + } + _ => panic!(), + } + } +} diff --git a/vw-eda/src/stream.rs b/vw-eda/src/stream.rs new file mode 100644 index 0000000..69ec608 --- /dev/null +++ b/vw-eda/src/stream.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! How a backend hands output to its caller while a command is still +//! running. +//! +//! Separate from the request/response protocol because it is not a reply to +//! anything: a synthesis run produces output for minutes before it produces a +//! result, and a caller that only saw the result would have nothing to show +//! for the wait. Every backend streams the same way, so a caller written +//! against one works against another — including one that is not on this +//! machine. + +/// Tag attached to each chunk a [`StdoutSink`] receives, so the +/// caller can route it to the right UI lane. The shim's +/// `puts`-interception path always produces [`StreamKind::Stdout`] +/// — user TCL has no way to "label" a write. The PTY-line filter +/// classifies Vivado's standard message format +/// (`ERROR:`/`WARNING:`/`CRITICAL WARNING:`/`INFO:`) into the +/// corresponding kind. +/// +/// A consumer that doesn't care (e.g. `vw run` capturing for +/// stdout pass-through) can ignore the kind and treat every chunk +/// identically; the REPL uses it to colour error/warning lines. +#[derive( + Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum StreamKind { + /// User TCL `puts` output, or any other chunk we don't have a + /// reason to label otherwise. Default. + Stdout, + /// Vivado `INFO:` line — usually low-importance chatter from + /// the message system. + Info, + /// Vivado `WARNING:` line. + Warning, + /// Vivado `CRITICAL WARNING:` line. Semantically means "your + /// run may fail because of this" — Vivado nests this severity + /// between WARNING and ERROR. Distinct from + /// [`StreamKind::Error`] so log-level filtering can treat them + /// separately (`--log-level=error` hides critical warnings; + /// `--log-level=critical` keeps them). + CriticalWarning, + /// Vivado `ERROR:` line. Distinct from the final + /// [`BackendError::Tcl`] returned by `eval` — these are emitted + /// *during* an eval and the final error often refers back to + /// them ("failed due to earlier errors"). + Error, +} + +/// Sink for streamed output during an eval. Called once per chunk +/// the worker observes — from the shim's `puts` interception (Tcl +/// user output) or from the PTY-line filter (Vivado's own message +/// system). The [`StreamKind`] tags the chunk so the caller can +/// route warnings and errors to a more attention-grabbing UI +/// surface than ordinary stdout. +pub type StdoutSink = Box; diff --git a/vw-htcl-cmd/Cargo.toml b/vw-htcl-cmd/Cargo.toml new file mode 100644 index 0000000..ad3a891 --- /dev/null +++ b/vw-htcl-cmd/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vw-htcl-cmd" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Generate documented htcl command wrappers from Vivado man-page references" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +serde.workspace = true +thiserror.workspace = true +toml.workspace = true +winnow.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-htcl-cmd/src/constraints.rs b/vw-htcl-cmd/src/constraints.rs new file mode 100644 index 0000000..7d491d5 --- /dev/null +++ b/vw-htcl-cmd/src/constraints.rs @@ -0,0 +1,223 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Per-command signature augmentations layered on top of the +//! man-page-derived wrapper. +//! +//! UG835 gives us each command's flag/positional list and types, but +//! it has no language for the semantic refinements an htcl wrapper +//! benefits from — mutually-exclusive call modes (`set_property`'s +//! `-dict` vs `-name/-value/-objects` pair), inter-argument +//! requirements (`tuser_width @requires has_tuser`), reclassifying +//! a positional into a keyword-form arg with a default. Those live +//! in a TOML file the wrapper-module author hand-maintains alongside +//! the auto-generated `cmd/*.htcl` files. +//! +//! File shape: +//! +//! ```toml +//! [.args.] +//! default = "..." # adds/replaces @default(...) +//! enum = ["a", "b"] # adds/replaces @enum(a, b) +//! clear_enum = true # drops any @enum the man-page emitted +//! one_of = ["other"] # adds @one_of(other) +//! requires = ["a", "b"] # adds @requires(a, b) +//! conflicts = ["a"] # adds @conflicts(a) +//! ``` +//! +//! The generator applies overrides during signature emission. The +//! body emission then follows the post-override arg classification +//! — flipping a flag from `@enum(0, 1)` to `@default("")` makes it +//! a value-taking arg, and the body forwards `-flag $value` +//! instead of `if {$flag} { lappend cmd -flag }`. + +use std::collections::HashMap; +use std::path::Path; + +use serde::Deserialize; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConstraintsError { + #[error("reading {path}: {source}")] + Io { + path: std::path::PathBuf, + #[source] + source: std::io::Error, + }, + #[error("parsing {path}: {source}")] + Parse { + path: std::path::PathBuf, + #[source] + source: toml::de::Error, + }, +} + +/// Per-arg overrides for one command. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct ArgOverride { + /// New `@default(...)` value. Replaces any inherited default. + #[serde(default)] + pub default: Option, + /// New `@enum(...)` choices. Replaces any inherited enum. + #[serde(default, rename = "enum")] + pub enum_: Option>, + /// Drop any inherited `@enum`. Use when the man-page parsing + /// modeled an arg as `@enum(0, 1)` (boolean toggle) but it's + /// actually value-taking. + #[serde(default)] + pub clear_enum: bool, + /// `@one_of(...)` declarations to add. Empty means no addition. + #[serde(default)] + pub one_of: Vec, + /// `@requires(...)` declarations to add. + #[serde(default)] + pub requires: Vec, + /// `@conflicts(...)` declarations to add. + #[serde(default)] + pub conflicts: Vec, + /// Override whether this arg carries a Vivado typed Tcl_Obj + /// handle (e.g. a bd_cell, get_bd_pins result). `None` keeps + /// the generator default (a name-based allowlist of well-known + /// typed-arg names like `objects`/`cells`/`pin`/...). `Some(true)` + /// forces this arg to be treated as typed; `Some(false)` forces + /// it to be treated as a plain string. + /// + /// Typed args are passed directly via `-flag $value` in the + /// wrapper body — never through `[list]` or `lappend` — because + /// list-construction shimmers Vivado's internal typed Tcl_Obj to + /// a plain string, and downstream consumers like + /// `set_property -objects` reject the stringified path. + #[serde(default)] + pub typed: Option, + + /// Per-arg type annotation: any valid htcl type expression + /// (`bd_cell`, `list`, `string`, etc.). Wins over + /// the inferred type from the typed-handle name table when + /// both are present. Set when an arg's name doesn't match + /// the table's plural-aware heuristic, or when the man-page + /// `object` placeholder is actually a specific type. + #[serde(default, rename = "type")] + pub arg_type: Option, + + /// Force this arg to be emitted positionally in the wrapper + /// body: `... $value` instead of `... - $value`. The + /// generator's default is to always route required args + /// through the flag form (matches most Vivado commands), but + /// a few — notably `get_property`'s trailing `` — + /// reject `-objects` with `[Common 17-170] Unknown option`. + /// Set `positional = true` in `cmd-constraints.toml` for + /// those specific args. Defaults to false. + #[serde(default)] + pub positional: bool, +} + +/// All overrides for one command, indexed by arg ident. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct CommandOverride { + /// Per-arg overrides. The key is the htcl proc-arg identifier + /// (matches `Argument::ident`). + #[serde(default)] + pub args: HashMap, + /// Override the command's return type. The string is taken + /// verbatim and emitted as the proc's 4th-word annotation + /// (`proc NAME { args } { body }`), so any valid + /// htcl type expression works: `bd_cell`, `list`, + /// `dict`, `unit`. Use this when the Returns: + /// phrase auto-mapping is ambiguous or wrong for a specific + /// command. + #[serde(default)] + pub returns: Option, +} + +/// The complete set of overrides loaded from the constraints file. +/// Lookups are by command name (`set_property`, `create_project`, +/// …) — missing entries return `None` and the generator emits the +/// pure man-page-derived wrapper. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub struct ConstraintsTable { + commands: HashMap, +} + +impl ConstraintsTable { + /// Empty table — every command falls back to the pure man-page + /// signature. Used when no `--constraints` was passed. + pub fn empty() -> Self { + Self::default() + } + + /// Load from a TOML file at `path`. + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| { + ConstraintsError::Io { + path: path.to_path_buf(), + source: e, + } + })?; + toml::from_str(&text).map_err(|e| ConstraintsError::Parse { + path: path.to_path_buf(), + source: e, + }) + } + + /// Per-command overrides, or `None` when nothing is declared + /// for `command`. + pub fn for_command(&self, command: &str) -> Option<&CommandOverride> { + self.commands.get(command) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_table_returns_no_overrides() { + let t = ConstraintsTable::empty(); + assert!(t.for_command("set_property").is_none()); + } + + #[test] + fn parses_full_arg_override_block() { + let toml = r#" + [set_property.args.dict] + default = "" + clear_enum = true + one_of = ["name"] + requires = ["objects"] + + [set_property.args.name] + default = "" + one_of = ["dict"] + requires = ["value", "objects"] + "#; + let t: ConstraintsTable = toml::from_str(toml).unwrap(); + let sp = t.for_command("set_property").unwrap(); + let dict = sp.args.get("dict").unwrap(); + assert_eq!(dict.default.as_deref(), Some("")); + assert!(dict.clear_enum); + assert_eq!(dict.one_of, vec!["name".to_string()]); + assert_eq!(dict.requires, vec!["objects".to_string()]); + + let name = sp.args.get("name").unwrap(); + assert_eq!(name.default.as_deref(), Some("")); + assert_eq!(name.one_of, vec!["dict".to_string()]); + assert_eq!( + name.requires, + vec!["value".to_string(), "objects".to_string()] + ); + } + + #[test] + fn missing_command_returns_none() { + let toml = r#" + [set_property.args.dict] + default = "" + "#; + let t: ConstraintsTable = toml::from_str(toml).unwrap(); + assert!(t.for_command("create_project").is_none()); + assert!(t.for_command("set_property").is_some()); + } +} diff --git a/vw-htcl-cmd/src/generate.rs b/vw-htcl-cmd/src/generate.rs new file mode 100644 index 0000000..41aa62b --- /dev/null +++ b/vw-htcl-cmd/src/generate.rs @@ -0,0 +1,986 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Emit an htcl wrapper proc for a parsed [`ManPage`]. +//! +//! Shape, for a command `add_files`: +//! +//! ```htcl +//! # Preserve the underlying Vivado command so the wrapper can forward +//! # to it after shadowing the global name. +//! if {[info commands __viv_add_files] eq "" && [info commands add_files] ne ""} { +//! rename add_files __viv_add_files +//! } +//! +//! ## Adds one or more source files ... +//! proc add_files { +//! ## (Optional) The fileset to add to. +//! @default("") fileset +//! ## (Optional) Do not recurse ... +//! @enum(0, 1) @default(0) norecurse +//! ## Positional operands ... +//! @default("") operands +//! } { +//! set cmd [list __viv_add_files] +//! if {$fileset ne ""} { lappend cmd -fileset $fileset } +//! if {$norecurse} { lappend cmd -norecurse } +//! if {$operands ne ""} { lappend cmd {*}$operands } +//! return [{*}$cmd] +//! } +//! ``` +//! +//! The wrapper keeps the command's natural name and shadows the +//! builtin; a guarded `rename` stashes the original under +//! `` so the body forwards to it without recursing. All +//! arguments are addressed by keyword (`-fileset value`); boolean flags +//! take a `0`/`1` value at the htcl layer and lower to flag +//! presence/absence on the Vivado command line. + +use std::fmt::Write; + +use vw_htcl::emit::{Command, Doc, Item, Word}; + +use crate::constraints::{ArgOverride, ConstraintsTable}; +use crate::model::{ArgKind, Argument, ManPage}; + +/// Arg names whose values are Vivado typed `Tcl_Obj` handles — +/// `bd_cell`, `bd_pin`, etc. — and therefore must be passed to the +/// underlying command **directly** (`-flag $value`) rather than +/// through `[list]` or `lappend`. List construction shimmers Tcl's +/// internal typed representation away, leaving the handle as a +/// plain path string; downstream code paths in Vivado (notably +/// `set_property -objects`) reject the stringified path with +/// `[Common 17-161] Invalid option value`. +/// +/// Curated list of the obvious cases. Per-arg override via +/// `cmd-constraints.toml`'s `typed = true|false` covers the long +/// tail. +const TYPED_ARG_NAMES: &[&str] = &[ + "object", + "objects", + "of_objects", + "cell", + "cells", + "pin", + "pins", + "port", + "ports", + "intf_pin", + "intf_pins", + "intf_port", + "intf_ports", + "net", + "nets", + "intf_net", + "intf_nets", + // File-object handles: `get_files` returns a Tcl_Obj carrying + // Vivado's internal file representation. Passing that through + // `lappend flags {*}$files` shimmers the internal rep to a plain + // path string, which `make_wrapper -files` (and any downstream + // command taking a file-object list) then rejects with + // `[Common 17-161] Invalid option value`. + "file", + "files", + // Filesets: `get_filesets` returns fileset objects. Same + // shimmer pitfall as files/cells. + "fileset", + "filesets", +]; + +fn is_typed_arg(name: &str, override_: Option) -> bool { + match override_ { + Some(t) => t, + None => TYPED_ARG_NAMES.contains(&name), + } +} + +/// Map a typed-arg name to its concrete `TypeExpr` text, when +/// known. Drives the `name: TYPE` annotation emitted in the +/// generated proc args. Plural names (`cells`, `pins`) map to +/// `list`; singulars (`cell`, `pin`) map to the bd_* type +/// directly. Generic catch-alls (`object`, `objects`, +/// `of_objects`) can refer to any Vivado handle class, so we +/// leave them untyped at this layer — the type system doesn't +/// have unions in v1. +/// +/// Returning `None` means "the arg is typed (don't list-wrap in +/// the body) but we don't have a precise type expression for +/// it" — the generator emits the arg without an annotation. +fn typed_arg_type(name: &str) -> Option<&'static str> { + match name { + "cell" => Some("bd_cell"), + "cells" => Some("list"), + "pin" => Some("bd_pin"), + "pins" => Some("list"), + "port" => Some("bd_port"), + "ports" => Some("list"), + "net" => Some("bd_net"), + "nets" => Some("list"), + "intf_pin" => Some("bd_intf_pin"), + "intf_pins" => Some("list"), + "intf_port" => Some("bd_intf_port"), + "intf_ports" => Some("list"), + "intf_net" => Some("bd_intf_net"), + "intf_nets" => Some("list"), + // object / objects / of_objects: any handle class — no + // precise type until we have unions. + // + // file / files / fileset / filesets: no concrete newtype + // in the current type-decl set — leave the annotation off + // (returned as `string` today from `get_files` / + // `get_filesets`); still routed through the typed-arg + // fast path so we don't shimmer the internal rep away. + _ => None, + } +} + +#[derive(Clone, Debug)] +pub struct GenerateOptions { + /// Prefix for the stashed original command (`rename add_files + /// __viv_add_files`). Kept for backwards compatibility — the + /// lowering pass now generates the rename plumbing, so this + /// field has no effect. + pub rename_prefix: String, + /// Emit each command's `See Also` list as a doc-comment footer. + pub include_see_also: bool, + /// Per-command signature augmentations loaded from + /// `cmd-constraints.toml`. The generator merges these onto the + /// man-page-derived shape so wrapper authors can declare + /// mutually-exclusive call modes, value-taking flags + /// misclassified by the man page, etc., without hand-editing + /// the generated files. + pub constraints: ConstraintsTable, +} + +impl Default for GenerateOptions { + fn default() -> Self { + Self { + rename_prefix: "__viv_".to_string(), + include_see_also: true, + constraints: ConstraintsTable::empty(), + } + } +} + +/// Generate the htcl wrapper text for `page`. +pub fn generate(page: &ManPage, opts: &GenerateOptions) -> String { + let cmd = &page.name; + // Wrapper body forwards to the underlying Vivado proc via + // `extern::` (which the lowering rewrites to the bare native + // name). The wrapper itself lives inside `namespace eval + // vivado { ... }` so it doesn't shadow the global name the + // body is forwarding to — that's what frees Vivado's own + // internal Tcl from accidentally hitting our typed wrappers + // when it calls a sibling builtin. + let forwarded = format!("extern::{}", page.name); + + let overrides = opts.constraints.for_command(&page.name); + let effective = effective_args(page, overrides); + + let mut out = String::new(); + writeln!( + out, + "# Generated by `vw htcl-cmd generate` from the Vivado command \ + reference." + ) + .unwrap(); + writeln!(out, "# Do not edit by hand.").unwrap(); + writeln!(out).unwrap(); + + // Wrappers live in `vivado_cmd::`, NOT `vivado::`. Vivado has + // its own internal `::vivado` namespace and code paths that + // behave differently depending on the calling namespace — + // notably `set_property -dict -objects ...` rejects valid cell + // handles when invoked from inside `::vivado`. Picking a name + // Vivado doesn't use means our wrapper bodies never collide + // with Vivado-internal namespace state. + writeln!(out, "namespace eval vivado_cmd {{").unwrap(); + writeln!(out).unwrap(); + + // Proc doc comment: the command Description, then a See-Also footer. + emit_proc_doc(&mut out, page, opts); + + // Proc args (structured) and body (compact Tcl). + let args = build_args(page, &effective); + let body = build_body(&forwarded, &effective); + // Resolve return type. Priority: + // 1. Explicit override in `cmd-constraints.toml`. + // 2. The page's `Returns:` section, if present. + // 3. Phrases in the `Description:` section — Vivado very + // rarely uses a dedicated Returns: header, so this is + // actually the common path. The phrase table is the same + // either way. + // 4. Fallback to `string`. The emitted body ALWAYS ends with + // `return [extern::_vw_global_call ...]`, which in Tcl + // yields whatever the underlying command returns — a + // string, possibly empty. The htcl validator rejects + // value-returning procs with no return-type annotation, + // so we must always emit one. `string` is the safe + // universal fallback for the commands whose Returns: + // prose doesn't match any of the specific phrases. + let return_type = overrides + .and_then(|o| o.returns.as_deref()) + .map(String::from) + .or_else(|| infer_return_type(page.returns.as_deref())) + .or_else(|| infer_return_type(Some(page.description.as_slice()))) + .or_else(|| Some("string".to_string())); + emit_proc(&mut out, cmd, &args, return_type.as_deref(), &body); + + writeln!(out).unwrap(); + writeln!(out, "}}").unwrap(); + + // Trim trailing whitespace line-by-line (empty doc comments emit a + // trailing space) and guarantee a single trailing newline. + let mut cleaned: String = out + .lines() + .map(str::trim_end) + .collect::>() + .join("\n"); + cleaned.push('\n'); + cleaned +} + +/// Write the proc-level doc comments above the `proc` line so they +/// attach to it. The output is structured as +/// +/// ```text +/// ##

+/// ## +/// ## +/// ## +/// ## +/// ``` +/// +/// where the summary is the first sentence of the source description +/// (LSP-clients use it for inline annotations like +/// `CompletionItem::detail`) and the body is everything after, +/// rendered as separate paragraphs. The body paragraphs are +/// re-wrapped at ~78 columns so the on-disk file stays readable +/// without preserving the man-page's source wrap. +fn emit_proc_doc(out: &mut String, page: &ManPage, opts: &GenerateOptions) { + let raw: Vec = + page.description.iter().map(|l| sanitize_doc(l)).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + + match summary { + None => { + writeln!(out, "## Wrapper for the Vivado `{}` command.", page.name) + .unwrap(); + } + Some(s) => { + emit_paragraph_lines(out, &s, "## ", 78); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + writeln!(out, "##").unwrap(); + emit_paragraph_lines(out, paragraph, "## ", 78); + } + } + + if opts.include_see_also && !page.see_also.is_empty() { + writeln!(out, "##").unwrap(); + writeln!(out, "## See also: {}", page.see_also.join(", ")).unwrap(); + } +} + +fn emit_paragraph_lines( + out: &mut String, + text: &str, + prefix: &str, + width: usize, +) { + let body_width = width.saturating_sub(prefix.len()); + for line in vw_htcl::doc::wrap_paragraph(text, body_width) { + writeln!(out, "{prefix}{line}").unwrap(); + } +} + +/// One argument plus whatever overrides from `cmd-constraints.toml` +/// apply to it. `default`, `enum_values`, `one_of`, `requires`, +/// `conflicts` are the *final* values the wrapper should emit; +/// constraint resolution has already happened. +/// +/// `kind` is derived: a constraint that clears the enum and adds a +/// default to a man-page-Boolean arg flips it to value-taking, so +/// the body-emitter forwards `-flag $value` instead of `if {$f} { +/// lappend cmd -flag }`. +#[derive(Clone, Debug)] +struct EffectiveArg { + ident: String, + flag: Option, + kind: ArgKind, + /// `None` → no default (required); `Some(text)` → emit `@default(text)`. + default: Option, + /// `None` → no enum; `Some(vec)` → emit `@enum(...)`. + enum_values: Option>, + one_of: Vec, + requires: Vec, + conflicts: Vec, + description: Vec, + /// True when this arg carries a Vivado typed `Tcl_Obj` handle + /// — body emission passes it directly (`-flag $value`) rather + /// than threading it through a list. See [`TYPED_ARG_NAMES`]. + typed: bool, + /// The arg's declared type expression, if known. Emitted in + /// the proc args as `name: TYPE`. Set from + /// [`typed_arg_type`] for the typed-handle allowlist; an + /// explicit per-arg `type = "..."` override in + /// `cmd-constraints.toml` wins over the inferred value. + arg_type: Option, + /// True when the wrapper body should pass this arg + /// positionally (`... $value`) instead of via `- $value`. + /// Set by the `positional = true` constraint override — + /// needed for the small subset of Vivado commands + /// (`get_property`'s trailing ``, notably) that + /// reject the `-flag` form with `[Common 17-170] Unknown + /// option`. + positional: bool, +} + +fn effective_args( + page: &ManPage, + overrides: Option<&crate::constraints::CommandOverride>, +) -> Vec { + page.arguments + .iter() + .map(|arg| { + effective_arg(arg, overrides.and_then(|o| o.args.get(&arg.ident))) + }) + .collect() +} + +/// Translate legacy `"0"`/`"1"` boolean defaults into `"false"`/ +/// `"true"` when the arg is still typed as a bool (i.e., no +/// `clear_enum` or explicit type override that would fall through +/// to the value-taking path). Anything else passes through verbatim. +fn bool_translate_if_needed( + raw: &str, + kind: &ArgKind, + over: &ArgOverride, +) -> String { + let is_bool_arg = matches!(kind, ArgKind::Boolean) + && !over.clear_enum + && over.arg_type.is_none(); + if !is_bool_arg { + return raw.to_string(); + } + match raw { + "0" => "false".to_string(), + "1" => "true".to_string(), + _ => raw.to_string(), + } +} + +fn effective_arg(arg: &Argument, over: Option<&ArgOverride>) -> EffectiveArg { + let empty = ArgOverride::default(); + let over = over.unwrap_or(&empty); + + // Default value: explicit override wins; else man-page heuristic. + // Boolean args now emit as `@default(false) name: bool` instead + // of `@enum(0, 1) @default(0) name` — Vivado's man pages + // universally document them as toggles, and htcl already has a + // `bool` type. Callers write `-quiet true` instead of + // `-quiet 1`. + let mut default: Option = match &arg.kind { + ArgKind::Boolean => Some("false".to_string()), + ArgKind::Value | ArgKind::Positional => { + (!arg.required).then(|| "".to_string()) + } + }; + if let Some(d) = over.default.as_deref() { + // Translate `"0"`/`"1"` in an override to `false`/`true` if + // the arg is still typed as a bool. Otherwise the override + // wins verbatim. + default = Some(bool_translate_if_needed(d, &arg.kind, over)); + } + + // Enum: no `@enum` for booleans anymore — the `bool` type + // annotation carries the constraint. Non-boolean args still + // pick up whatever the override declares. + let mut enum_values: Option> = None; + if let Some(v) = &over.enum_ { + enum_values = Some(v.clone()); + } + + // Kind: an override that clears the (former) `@enum(0, 1)` on a + // man-page-Boolean and gives a string-typed default is signaling + // "actually value-taking" — body-emit should forward + // `-flag $value`, not `if {$flag} { ... }`. This is the exact + // shape `set_property -dict` needs. The signal is `clear_enum` + // in the override; without it we keep the Boolean shape and + // emit as a bool toggle. + let kind = if matches!(arg.kind, ArgKind::Boolean) && over.clear_enum { + if arg.flag.is_some() { + ArgKind::Value + } else { + ArgKind::Positional + } + } else { + arg.kind + }; + + let typed = is_typed_arg(&arg.ident, over.typed); + let arg_type = over + .arg_type + .clone() + .or_else(|| typed_arg_type(&arg.ident).map(String::from)) + // Default booleans to `bool` when there's no override and + // no allowlist entry. + .or_else(|| { + matches!(kind, ArgKind::Boolean).then(|| "bool".to_string()) + }); + + EffectiveArg { + ident: arg.ident.clone(), + flag: arg.flag.clone(), + kind, + default, + enum_values, + one_of: over.one_of.clone(), + requires: over.requires.clone(), + conflicts: over.conflicts.clone(), + description: arg.description.clone(), + typed, + arg_type, + positional: over.positional, + } +} + +/// Build the structured arg list as an emit [`Doc`]: per-argument doc +/// comments followed by an `@attr… ident` declaration. The doc +/// comments follow the same `summary, blank, body` shape the +/// proc-level docs use, so LSP clients can split brief/detail from +/// extended documentation consistently. +fn build_args(_page: &ManPage, effective: &[EffectiveArg]) -> Doc { + let mut doc = Doc::new(); + for (i, arg) in effective.iter().enumerate() { + if i > 0 { + doc.push(Item::Blank); + } + let raw: Vec = + arg.description.iter().map(|l| sanitize_doc(l)).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + + let body_width = 76usize; + if let Some(s) = summary.as_deref() { + for line in vw_htcl::doc::wrap_paragraph(s, body_width) { + doc.push(Item::DocComment(line)); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + doc.push(Item::DocComment(String::new())); + for line in vw_htcl::doc::wrap_paragraph(paragraph, body_width) + { + doc.push(Item::DocComment(line)); + } + } + } + + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: effective_attr_words(arg), + body: None, + })); + } + doc +} + +/// The attribute words + identifier for one effective argument. +fn effective_attr_words(arg: &EffectiveArg) -> Vec { + let mut words = Vec::new(); + if let Some(values) = &arg.enum_values { + let inner = values + .iter() + .map(|v| format_attribute_value(v)) + .collect::>() + .join(", "); + words.push(Word::Raw(format!("@enum({inner})"))); + } + if let Some(default) = &arg.default { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } + if !arg.one_of.is_empty() { + words.push(Word::Raw(format!("@one_of({})", arg.one_of.join(", ")))); + } + if !arg.requires.is_empty() { + words + .push(Word::Raw(format!("@requires({})", arg.requires.join(", ")))); + } + if !arg.conflicts.is_empty() { + words.push(Word::Raw(format!( + "@conflicts({})", + arg.conflicts.join(", ") + ))); + } + match arg.arg_type.as_deref() { + Some(ty) => { + // Emit `name: TYPE` as two adjacent bare words. The + // proc-args parser tokenizes `name`, `:`, and TYPE + // independently — the layout reads as the user would + // write it. + words.push(Word::Bare(format!("{}:", arg.ident))); + words.push(Word::Bare(ty.to_string())); + } + None => { + words.push(Word::Bare(arg.ident.clone())); + } + } + words +} + +fn format_attribute_value(s: &str) -> String { + let is_int = !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + let is_ident = !s.is_empty() + && s.bytes().enumerate().all(|(i, b)| { + if i == 0 { + b.is_ascii_alphabetic() || b == b'_' + } else { + b.is_ascii_alphanumeric() || b == b'_' + } + }); + if is_int || is_ident { + s.to_string() + } else { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } +} + +/// Build the proc body. +/// +/// Args are split into two cohorts: +/// +/// - **Non-typed args** (booleans, strings, positionals whose name +/// isn't in the typed-handle allowlist) accumulate into a `flags` +/// list via `lappend`. Each arg's kind drives its emit form — +/// `Boolean` → `if {$x} { lappend flags -flag }`, `Value` → +/// `if {$x ne ""} { lappend flags -flag $x }`, `Positional` → +/// `if {$x ne ""} { lappend flags {*}$x }`. These values are all +/// strings, so the lappend / `{*}`-expansion that follows is +/// safe — string values don't have a typed Tcl_Obj to shimmer. +/// - **Typed-handle args** (`-objects`/`-cell`/etc., per +/// [`TYPED_ARG_NAMES`] or per-arg `typed = true` override) are +/// passed **directly** to the underlying command via +/// `-flag $value`. Putting them through `[list]` or `lappend` +/// would shimmer Vivado's internal typed Tcl_Obj to a string, +/// and downstream code paths like `set_property -objects` reject +/// the stringified path with `[Common 17-161] Invalid option +/// value '...' specified for 'objects'`. +/// +/// The invocation site branches on which typed args are present so +/// no typed-arg flag appears in the call when its variable is +/// empty. With N typed args this is 2^N branches; in practice N is +/// 0 or 1 for almost every Vivado command, and never more than 2. +fn build_body(orig: &str, effective: &[EffectiveArg]) -> String { + let mut body = String::new(); + + let non_typed: Vec<&EffectiveArg> = + effective.iter().filter(|a| !a.typed).collect(); + let typed: Vec<&EffectiveArg> = + effective.iter().filter(|a| a.typed).collect(); + + // Non-typed accumulator. `flags` is a plain Tcl list — only + // ever contains string values, so list-construction shimmer is + // a non-issue. + writeln!(body, "set flags [list]").unwrap(); + for arg in &non_typed { + let id = &arg.ident; + let required = arg.default.is_none(); + match arg.kind { + ArgKind::Boolean => { + let flag = arg.flag.as_deref().unwrap_or(id); + writeln!(body, "if {{${id}}} {{ lappend flags -{flag} }}") + .unwrap(); + } + ArgKind::Value => { + let flag = arg.flag.as_deref().unwrap_or(id); + if required { + writeln!(body, "lappend flags -{flag} ${id}").unwrap(); + } else { + writeln!( + body, + "if {{${id} ne \"\"}} {{ lappend flags -{flag} ${id} }}" + ) + .unwrap(); + } + } + ArgKind::Positional => { + if required { + writeln!(body, "lappend flags {{*}}${id}").unwrap(); + } else { + writeln!( + body, + "if {{${id} ne \"\"}} \ + {{ lappend flags {{*}}${id} }}" + ) + .unwrap(); + } + } + } + } + + // Typed-arg branching. Direct invocation per combination of + // typed args that are non-empty, so the typed values never + // touch a Tcl list. + emit_typed_invocation(&mut body, orig, &typed, 0); + + body +} + +/// Emit the typed-arg branch tree. At each level we split on +/// "this typed arg present?" and recurse; at the leaves we emit +/// `return [extern::_vw_global_call extern:: {*}$flags …]` +/// with whatever subset of typed args was present. The `extern::` +/// prefix on the helper is what the htcl analyzer sees — the +/// lowerer's `rewrite_externs` pass strips it, so the actual +/// runtime call is `::_vw_global_call :: …`. +/// +/// The `::_vw_global_call` helper (defined in the shim at +/// `::` namespace) is what keeps Vivado's internal Tcl (XDC +/// parsing, OOC synth flows) from resolving unqualified commands +/// to *our* wrappers just because we're currently inside +/// `::vivado_cmd::`. Without a global-namespace call, a +/// wrapper that forwards to `synth_ip` — which itself sources XDC +/// files whose scripts call `create_clock`, `get_ports`, … +/// positionally — would put those XDC scripts in the +/// `::vivado_cmd::` namespace context. The XDC's unqualified +/// `create_clock` would then land on our typed wrapper (which +/// expects `-flag` args) and its kwargs prologue would crash on +/// the positional port name. +/// +/// We use the helper rather than `namespace eval ::` / +/// `namespace inscope ::` / `uplevel #0`, all of which internally +/// serialize their args to a script string via `concat` and +/// re-parse them. That round-trip loses Tcl_Obj internal reps — +/// bd_cell handles become plain paths like `/cpm5`, which +/// Vivado's `set_property -objects` then rejects as "Invalid +/// option value". The helper's `{*}$cmd {*}$args` expansion is a +/// direct arg-passing, not a script re-parse, so object identity +/// is preserved end-to-end. +fn emit_typed_invocation( + body: &mut String, + orig: &str, + typed: &[&EffectiveArg], + depth: usize, +) { + let indent = " ".repeat(depth); + if typed.is_empty() { + writeln!( + body, + "{indent}return [extern::_vw_global_call {orig} {{*}}$flags]" + ) + .unwrap(); + return; + } + if let Some((first, rest)) = typed.split_first() { + // Required typed args have no `ne ""` guard — they're + // always passed. Optional typed args branch on presence. + let id = &first.ident; + let flag = first.flag.as_deref().unwrap_or(id); + let required = first.default.is_none(); + if required { + emit_typed_invocation_with( + body, + orig, + rest, + &[(*first, flag)], + depth, + ); + } else { + writeln!(body, "{indent}if {{${id} ne \"\"}} {{").unwrap(); + emit_typed_invocation_with( + body, + orig, + rest, + &[(*first, flag)], + depth + 1, + ); + writeln!(body, "{indent}}} else {{").unwrap(); + emit_typed_invocation(body, orig, rest, depth + 1); + writeln!(body, "{indent}}}").unwrap(); + } + } +} + +/// Inner: we've decided to include `included` typed args; the +/// remaining `rest` still need branching. At the leaf we emit a +/// `return` with `{*}$flags` and each included typed arg as +/// `-flag $var`. +fn emit_typed_invocation_with( + body: &mut String, + orig: &str, + rest: &[&EffectiveArg], + included: &[(&EffectiveArg, &str)], + depth: usize, +) { + let indent = " ".repeat(depth); + if rest.is_empty() { + // Same `extern::_vw_global_call` helper as the no-typed + // leaf — see [`emit_typed_invocation`] for the rationale + // (including why the `extern::` prefix is on the helper). + let mut line = format!( + "{indent}return [extern::_vw_global_call {orig} {{*}}$flags" + ); + for (arg, flag) in included { + // `positional = true` in cmd-constraints.toml opts an + // arg OUT of the default `- $value` shape and + // into bare `$value`. Vivado's `get_property` is the + // canonical case: its trailing `` is + // positional-only (using `-objects` fires + // `[Common 17-170] Unknown option '-objects'`). The + // opt-in is per-arg — the same command may still want + // `-min` / `-max` / `-quiet` in flag form. + if arg.positional { + write!(line, " ${id}", id = arg.ident).unwrap(); + continue; + } + match arg.kind { + ArgKind::Positional => { + // Typed positional args carry Vivado object + // handles (`get_files`, `get_filesets`, + // `get_cells`, …). At the Tcl call level + // Vivado's commands mostly take these via + // `- $value`, not positional — see + // the `make_wrapper -files [get_files ...]` + // example in the man page. Emit the flag + // form so both shimmer avoidance AND flag + // routing land at once; a bare positional + // yields `[Common 17-161] Invalid option + // value` because Vivado can't tell which + // slot it was intended for. Commands that + // genuinely need the positional shape use + // the `positional = true` override above. + write!(line, " -{flag} ${id}", id = arg.ident).unwrap(); + } + _ => { + write!(line, " -{flag} ${id}", id = arg.ident).unwrap(); + } + } + } + line.push(']'); + writeln!(body, "{line}").unwrap(); + return; + } + if let Some((first, more)) = rest.split_first() { + let id = &first.ident; + let flag = first.flag.as_deref().unwrap_or(id); + let required = first.default.is_none(); + if required { + let mut new_included = included.to_vec(); + new_included.push((*first, flag)); + emit_typed_invocation_with(body, orig, more, &new_included, depth); + } else { + writeln!(body, "{indent}if {{${id} ne \"\"}} {{").unwrap(); + let mut new_included = included.to_vec(); + new_included.push((*first, flag)); + emit_typed_invocation_with( + body, + orig, + more, + &new_included, + depth + 1, + ); + writeln!(body, "{indent}}} else {{").unwrap(); + emit_typed_invocation_with(body, orig, more, included, depth + 1); + writeln!(body, "{indent}}}").unwrap(); + } + } +} + +/// Emit `proc { } ? { }` with args and +/// body each indented two spaces. When `return_type` is Some, emits +/// it as the 4th htcl word between the args block and the body — +/// brace-wrapping if the type expression contains whitespace so it +/// parses as a single word. +fn emit_proc( + out: &mut String, + name: &str, + args: &Doc, + return_type: Option<&str>, + body: &str, +) { + let args_text = args.to_string(); + writeln!(out, "proc {name} {{").unwrap(); + for line in args_text.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + match return_type { + Some(ty) => { + // Wrap with `{ … }` if the type expression contains + // whitespace (the htcl parser would otherwise see + // multiple words). + let needs_brace = ty.chars().any(char::is_whitespace); + if needs_brace { + writeln!(out, "}} {{{ty}}} {{").unwrap(); + } else { + writeln!(out, "}} {ty} {{").unwrap(); + } + } + None => { + writeln!(out, "}} {{").unwrap(); + } + } + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Infer a return-type annotation from the Vivado man-page's +/// `Returns:` prose. The phrase-table is intentionally small — +/// matches the recurring shapes Vivado uses across hundreds of +/// commands. Unmatched phrasings return `None`; the +/// `cmd-constraints.toml` `returns = "…"` override picks up +/// whatever doesn't match. +/// +/// Matched on the joined, lowercased text — Vivado's prose is +/// short (usually one or two lines) so we don't need a real NLP +/// pipeline. +fn infer_return_type(returns: Option<&[String]>) -> Option { + let lines = returns?; + let joined = lines.join(" ").to_ascii_lowercase(); + let text = joined.trim(); + if text.is_empty() { + return None; + } + // Order matters: more-specific phrases first. Each entry is + // (substring needle, type). A real future implementation + // could swap in regex; substring search is good enough for + // the v1 phrase set. + let table: &[(&str, &str)] = &[ + // Singular creator/current handles. These fire BEFORE the + // `nothing` catchall so a page like "returns the name of + // the newly created cell object, or returns nothing if + // the command fails" gets the meaningful type instead of + // the failure-sentinel `unit`. + // + // Ordering within this group: exact BD types before the + // generic `cell/pin/port/net` forms so `intf_port` + // outranks `port`, etc. + ("newly created interface port object", "bd_intf_port"), + ("newly created interface pin object", "bd_intf_pin"), + ("newly created interface net object", "bd_intf_net"), + ("current interface port object", "bd_intf_port"), + ("current interface pin object", "bd_intf_pin"), + ("current interface net object", "bd_intf_net"), + ("newly created cell object", "bd_cell"), + ("newly created pin object", "bd_pin"), + ("newly created port object", "bd_port"), + ("newly created net object", "bd_net"), + ("newly created master address segment object", "bd_addr_seg"), + ("newly created address segment object", "bd_addr_seg"), + ("newly created address segment", "bd_addr_seg"), + ("current ip integrator cell instance object", "bd_cell"), + ("current cell object", "bd_cell"), + ("current pin object", "bd_pin"), + ("current port object", "bd_port"), + ("current net object", "bd_net"), + ("current instance object", "bd_cell"), + // Report-shaped commands whose prose starts "Returns a + // list of strings …" — `list` even when the rest + // of the description mentions "nothing". + ("returns a list of strings", "list"), + // Name-of-object pages return `string` (a path/name). + ("returns the name of the design object", "string"), + ("name of the design object", "string"), + // Vivado's stock "This command returns a value, or list of + // values, or returns an error if it fails" idiom used on + // `get_property` (and other query commands whose return is + // a single string). Placed BEFORE the "nothing" catchall + // because the same page's description often also contains + // "returns nothing" as a side-note about missing-property + // behavior — matching "nothing" first would incorrectly + // land the wrapper on `unit`, silently swallowing the + // property value. + ("returns a value, or list of values", "string"), + // "nothing" / "Tcl_OK on success" — side-effecting commands. + // Placed AFTER the creator/current patterns so those + // pull the actual return type from the descriptive prose + // before falling to the failure-sentinel. + ("returns nothing", "unit"), + ("nothing", "unit"), + // Lists of typed handles. Order matters within each type + // family: more-specific "intf" phrasings first so a plain + // "list of pins" doesn't shadow "list of intf_pins" for a + // page whose prose mentions both. + // + // The `list of objects` variants catch phrasing + // Vivado uses on the `get_bd_*` pages ("Gets a list of pin + // objects", "Gets a list of net objects", …). Without them, + // those procs land untyped and the REPL renders their + // return value as one wrapped wall of text instead of + // one-per-line via `list::repr`. + ("a list of intf_pins", "list"), + ("a list of interface pins", "list"), + ("list of intf_pin objects", "list"), + ("list of interface pin objects", "list"), + ("a list of intf_ports", "list"), + ("a list of interface ports", "list"), + ("list of intf_port objects", "list"), + ("list of interface port objects", "list"), + ("a list of intf_nets", "list"), + ("a list of interface nets", "list"), + ("list of intf_net objects", "list"), + ("list of interface net objects", "list"), + ("a list of cells", "list"), + ("a list of bd_cells", "list"), + ("list of cell objects", "list"), + ("a list of pins", "list"), + ("a list of bd_pins", "list"), + ("list of pin objects", "list"), + ("a list of ports", "list"), + ("list of port objects", "list"), + ("a list of nets", "list"), + ("list of net objects", "list"), + // Singular handles. + ("the cell created", "bd_cell"), + ("the new cell", "bd_cell"), + ("the pin created", "bd_pin"), + ("the port created", "bd_port"), + ("the net created", "bd_net"), + // Property values. + ("the property value", "string"), + ("the value of the property", "string"), + ("a list of properties", "list"), + // Generic strings (catch-all when prose says "string" + // explicitly). + ("returns a string", "string"), + ]; + for (needle, ty) in table { + if text.contains(needle) { + return Some((*ty).into()); + } + } + None +} + +/// Make doc-comment text safe to embed inside the proc arg-list braces. +/// +/// The htcl parser captures a proc's arg list as a braced word and +/// brace-matches it raw (only `{`, `}`, `\` are special); a per-arg +/// `##` doc comment with an unbalanced brace or a stray backslash would +/// corrupt that match. Neutralize the three offenders — braces become +/// parentheses, backslashes become slashes — which keeps the prose +/// legible while guaranteeing the generated wrapper parses. +fn sanitize_doc(s: &str) -> String { + s.replace('\\', "/") + .replace('{', "(") + .replace('}', ")") + .trim_end() + .to_string() +} diff --git a/vw-htcl-cmd/src/lib.rs b/vw-htcl-cmd/src/lib.rs new file mode 100644 index 0000000..d90c42c --- /dev/null +++ b/vw-htcl-cmd/src/lib.rs @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado command reference → htcl wrapper generation. +//! +//! The dual of [`vw_ip`]: where that crate turns an IP-XACT component +//! into a configuration-interface proc, this one turns a Vivado Tcl +//! command's plain-text reference page (under +//! `/doc/eng/man`) into a documented, typed htcl wrapper for +//! that command. +//! +//! Each generated wrapper keeps the command's natural name and shadows +//! the Vivado builtin, forwarding to a `rename`-stashed copy of the +//! original. The payoff is the htcl surface: hover documentation drawn +//! from the man page, `@enum`/`@default` validation on flags, and +//! keyword call sites the analyzer can check — all on the real command +//! names. +//! +//! ```no_run +//! let page = vw_htcl_cmd::load("/opt/Vivado/doc/eng/man/add_files", None)?; +//! let htcl = vw_htcl_cmd::generate(&page, &Default::default()); +//! print!("{htcl}"); +//! # Ok::<(), vw_htcl_cmd::Error>(()) +//! ``` + +pub mod constraints; +pub mod generate; +pub mod model; +pub mod parse; + +pub use constraints::{ + ArgOverride, CommandOverride, ConstraintsError, ConstraintsTable, +}; +pub use generate::{generate, GenerateOptions}; +pub use model::{ArgKind, Argument, ManPage}; +pub use parse::parse_man_page; + +use std::path::Path; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("reading man page `{path}`: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, + #[error("cannot derive a command name from `{0}` (no file stem)")] + NoName(String), +} + +pub type Result = std::result::Result; + +/// Load and parse a man page from disk. +/// +/// The command name comes from `name_override` when given, otherwise +/// from the file stem (`.../man/add_files` → `add_files`). +pub fn load( + path: impl AsRef, + name_override: Option<&str>, +) -> Result { + let path = path.as_ref(); + let text = std::fs::read_to_string(path).map_err(|source| Error::Io { + path: path.display().to_string(), + source, + })?; + let name = match name_override { + Some(n) => n.to_string(), + None => path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| Error::NoName(path.display().to_string()))? + .to_string(), + }; + Ok(parse_man_page(&name, &text)) +} diff --git a/vw-htcl-cmd/src/model.rs b/vw-htcl-cmd/src/model.rs new file mode 100644 index 0000000..4034f67 --- /dev/null +++ b/vw-htcl-cmd/src/model.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The structured model of a Vivado command reference ("man page"). +//! +//! Vivado ships a plain-text reference page per Tcl command under +//! `doc/eng/man`. Each page follows a regular shape: +//! +//! ```text +//! Description: +//! +//! +//! +//! Arguments: +//! +//! -fileset - (Optional) +//! -norecurse - (Optional) +//! - (Required) +//! +//! Examples: +//! ... +//! +//! See Also: +//! +//! * import_files +//! * read_ip +//! ``` +//! +//! [`crate::parse`] turns that text into a [`ManPage`]; [`crate::generate`] +//! turns a [`ManPage`] into an htcl wrapper proc. + +/// A parsed Vivado command reference page. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ManPage { + /// The command name (e.g. `add_files`). Derived from the source + /// file stem, not the page body — the body never repeats it. + pub name: String, + /// The `Description:` section, de-indented, one entry per source + /// line. Empty lines are preserved as empty strings so paragraph + /// breaks survive into the emitted doc comment. + pub description: Vec, + /// The `Arguments:` section, one entry per documented flag or + /// positional operand, in declared order. + pub arguments: Vec, + /// Command names listed under `See Also:`. + pub see_also: Vec, + /// The raw `Returns:` prose, one entry per source line. Many + /// Vivado man pages don't include this section — it's `None` + /// in that case, and the generator emits the wrapper without + /// a return-type annotation (the REPL falls back to the + /// untyped heuristic for those). + pub returns: Option>, +} + +/// How an argument maps onto the underlying Vivado command line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ArgKind { + /// A `-flag` with no value placeholder: a boolean toggle. Emitted + /// as `@enum(0, 1) @default(0)` and forwarded as a bare `-flag` + /// when set. + Boolean, + /// A `-flag `: forwarded as `-flag $value` when non-empty. + Value, + /// A trailing positional operand (``, ``, …): + /// forwarded by list-expansion (`{*}$operands`) at the end of the + /// command line. + Positional, +} + +/// One documented argument of a command. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Argument { + pub kind: ArgKind, + /// The htcl proc-arg identifier the caller uses as `-`. + /// Equal to `flag` for flags; derived from the `` for + /// positionals. May be de-collided with a suffix. + pub ident: String, + /// The underlying Vivado flag name without its leading dash + /// (`fileset`, `norecurse`). `None` for positionals, which have no + /// flag on the command line. + pub flag: Option, + /// Whether the man page marked the argument `(Required)`. Required + /// arguments are emitted without an `@default`, so htcl forces the + /// caller to supply them. + pub required: bool, + /// Whether this is a generic operand placeholder synthesized by the + /// generator (the page documented no positional), rather than one + /// taken from the page text. + pub synthesized: bool, + /// The argument's prose description, de-indented, one entry per + /// source line (empty strings preserve paragraph breaks). + pub description: Vec, +} + +impl Argument { + /// `true` for flags (`-flag` / `-flag `), `false` for + /// positionals. + pub fn is_flag(&self) -> bool { + matches!(self.kind, ArgKind::Boolean | ArgKind::Value) + } +} diff --git a/vw-htcl-cmd/src/parse.rs b/vw-htcl-cmd/src/parse.rs new file mode 100644 index 0000000..a2639aa --- /dev/null +++ b/vw-htcl-cmd/src/parse.rs @@ -0,0 +1,554 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parse a Vivado command reference page into a [`ManPage`]. +//! +//! Following the convention in [`vw_htcl::parser`], the outer loop is +//! hand-rolled — it owns line grouping and section recovery, which a +//! pure combinator grammar models awkwardly for free-form reference +//! text — while the structural inner pieces (an argument header's +//! `-flag - (marker)` shape, a `See Also` bullet) are +//! parsed with [`winnow`]. +//! +//! The grammar the inner parsers recognize, per argument block: +//! +//! ```text +//! flag-header := '-' ident placeholder? ' - ' marker? prose +//! pos-header := '<' .. '>' '...'? ' - ' marker? prose +//! marker := '(' ('Optional' | 'Required') ')' +//! ``` +//! +//! A `placeholder` (anything between the flag name and the ` - ` +//! separator) makes a flag a *value* flag; its absence makes it a +//! *boolean* toggle. + +use winnow::ascii::space0; +use winnow::combinator::{opt, preceded}; +use winnow::token::take_while; +use winnow::ModalResult; +use winnow::Parser; + +use crate::model::{ArgKind, Argument, ManPage}; + +/// Parse `text` (the contents of one man page) into a [`ManPage`]. +/// `name` is the command name (the source file stem). +pub fn parse_man_page(name: &str, text: &str) -> ManPage { + let normalized = text.replace('\r', ""); + let sections = split_sections(&normalized); + + let description = section_lines(§ions, "Description") + .map(dedent_block) + .unwrap_or_default(); + + let arguments = section_lines(§ions, "Arguments") + .map(|lines| parse_arguments(&dedent_block(lines))) + .unwrap_or_default(); + + let see_also = section_lines(§ions, "See Also") + .or_else(|| section_lines(§ions, "See also")) + .map(parse_see_also) + .unwrap_or_default(); + + // The `Returns:` section is optional and usually one or two + // lines. Some pages spell it `Return Value` or `Return value` + // — accept both. + let returns = section_lines(§ions, "Returns") + .or_else(|| section_lines(§ions, "Return Value")) + .or_else(|| section_lines(§ions, "Return value")) + .map(dedent_block); + + let mut page = ManPage { + name: name.to_string(), + description, + arguments, + see_also, + returns, + }; + finalize_arguments(&mut page); + page +} + +// --------------------------------------------------------------------------- +// Sectioning (hand-rolled outer loop). +// --------------------------------------------------------------------------- + +/// A man page is a flat list of `Header:` sections. Returns each +/// section's title (without the trailing colon) paired with its raw +/// body lines, in document order. +fn split_sections(text: &str) -> Vec<(String, Vec)> { + let mut sections: Vec<(String, Vec)> = Vec::new(); + for line in text.lines() { + if let Some(title) = section_header(line) { + sections.push((title, Vec::new())); + } else if let Some((_, body)) = sections.last_mut() { + body.push(line.to_string()); + } + // Lines before the first header (a leading blank line, usually) + // are dropped. + } + sections +} + +/// Recognize a section header line — a capitalized label at column +/// zero ending in a colon, e.g. `Arguments:` or `See Also:`. Returns +/// the label without the colon. +fn section_header(line: &str) -> Option { + // Headers sit flush left; body text is indented. Cheap reject + // first. + if line.is_empty() || line.starts_with(' ') { + return None; + } + let stripped = line.strip_suffix(':')?; + if stripped.is_empty() + || !stripped + .chars() + .all(|c| c.is_ascii_alphabetic() || c == ' ') + || !stripped.starts_with(|c: char| c.is_ascii_uppercase()) + { + return None; + } + Some(stripped.to_string()) +} + +/// The body lines of the first section whose title equals `title`. +fn section_lines<'a>( + sections: &'a [(String, Vec)], + title: &str, +) -> Option<&'a [String]> { + sections + .iter() + .find(|(t, _)| t == title) + .map(|(_, body)| body.as_slice()) +} + +/// Strip the uniform two-space indent man-page bodies carry, leaving +/// any deeper (bullet / code) indentation intact, and drop leading and +/// trailing blank lines. Interior blank lines are preserved. +fn dedent_block(lines: &[String]) -> Vec { + let mut out: Vec = lines + .iter() + .map(|l| l.strip_prefix(" ").unwrap_or(l).trim_end().to_string()) + .collect(); + while out.first().is_some_and(|l| l.is_empty()) { + out.remove(0); + } + while out.last().is_some_and(|l| l.is_empty()) { + out.pop(); + } + out +} + +// --------------------------------------------------------------------------- +// Arguments. +// --------------------------------------------------------------------------- + +/// Group the (already de-indented) argument-section lines into blocks +/// — runs of consecutive non-blank lines — then turn each block into +/// an [`Argument`]. Blocks that aren't an argument header (`Note:`, +/// `Tip:`, free prose) are folded into the preceding argument's +/// description. +fn parse_arguments(lines: &[String]) -> Vec { + let mut args: Vec = Vec::new(); + for block in blocks(lines) { + let first = &block[0]; + match parse_arg_header(first) { + Some(header) => { + let mut description = Vec::new(); + let head = header.prose.trim().to_string(); + if !head.is_empty() { + description.push(head); + } + for line in &block[1..] { + description.push(line.trim().to_string()); + } + args.push(Argument { + kind: header.kind, + // Provisional: the positional's placeholder name, or + // empty for a flag. `finalize_arguments` sanitizes + // and de-collides it into the final identifier. + ident: header.name_hint.unwrap_or_default(), + flag: header.flag, + required: header.required, + synthesized: false, + description, + }); + } + None => { + // A `Note:` / `Tip:` / prose continuation block. Attach + // it to the most recent argument, separated by a blank + // line, so the context survives into hover. + if let Some(prev) = args.last_mut() { + prev.description.push(String::new()); + for line in &block { + prev.description.push(line.trim().to_string()); + } + } + } + } + } + args +} + +/// Split lines into blocks of consecutive non-empty lines. +fn blocks(lines: &[String]) -> Vec> { + let mut out: Vec> = Vec::new(); + let mut cur: Vec = Vec::new(); + for line in lines { + if line.trim().is_empty() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } else { + cur.push(line.clone()); + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// The structured outcome of parsing an argument header's first line. +struct ArgHeader { + kind: ArgKind, + /// Flag name without the dash, or `None` for a positional. + flag: Option, + /// A positional's identifier hint, recovered from its `` + /// (e.g. `` → `hw_sio_linkgroups`). `None` for a + /// flag, whose identifier comes from its flag name. + name_hint: Option, + required: bool, + /// The description text that followed the ` - ` separator on the + /// header line. + prose: String, +} + +/// Parse the first line of an argument block. Returns `None` when the +/// line is not a flag/positional header (so the caller treats the +/// block as a note attached to the previous argument). +fn parse_arg_header(line: &str) -> Option { + let mut input = line; + if let Ok(flag) = flag_lead.parse_next(&mut input) { + let (placeholder, prose) = split_separator(input); + let kind = if placeholder.trim().is_empty() { + ArgKind::Boolean + } else { + ArgKind::Value + }; + return Some(ArgHeader { + kind, + flag: Some(flag.to_string()), + name_hint: None, + required: is_required(&prose), + prose, + }); + } + if let Ok(inner) = positional_lead.parse_next(&mut input) { + let (_ellipsis, prose) = split_separator(input); + return Some(ArgHeader { + kind: ArgKind::Positional, + flag: None, + name_hint: first_ident_token(inner), + required: is_required(&prose), + prose, + }); + } + None +} + +/// The first `[A-Za-z_][A-Za-z0-9_]*` token in a positional's +/// placeholder text (`hw_sio_linkgroups`, `arg1 arg2 ...` → `arg1`). +/// `None` when the placeholder has no identifier-shaped run (`[0:750]`). +fn first_ident_token(inner: &str) -> Option { + let mut chars = inner.char_indices().peekable(); + while let Some(&(start, c)) = chars.peek() { + if c.is_ascii_alphabetic() || c == '_' { + let mut end = start; + for (i, c) in inner[start..].char_indices() { + if c.is_ascii_alphanumeric() || c == '_' { + end = start + i + c.len_utf8(); + } else { + break; + } + } + return Some(inner[start..end].to_string()); + } + chars.next(); + } + None +} + +/// `-ident` — consume the dash and flag name, leaving the rest of the +/// line in `input`. Returns the flag name without the dash. +fn flag_lead<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + preceded('-', ident).parse_next(input) +} + +/// `<...>` (with an optional trailing `...`) — consume the angle-bracket +/// placeholder that introduces a positional operand, leaving the rest +/// of the line in `input`. Returns the text inside the brackets. +fn positional_lead<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + '<'.parse_next(input)?; + let inner = take_while(0.., |c: char| c != '>').parse_next(input)?; + '>'.parse_next(input)?; + let _ = opt("...").parse_next(input)?; + // Note: do not consume the space after `>` — it is part of the + // ` - ` separator that `split_separator` looks for. + Ok(inner) +} + +/// An htcl-identifier run: `[A-Za-z0-9_]+`. +fn ident<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input) +} + +/// Split a header remainder on its first ` - ` separator into the +/// (placeholder, description) halves. With no separator the whole +/// remainder is taken as the description (and the placeholder is +/// empty), which makes a bare `-flag` a boolean toggle. +fn split_separator(rest: &str) -> (String, String) { + match rest.find(" - ") { + Some(idx) => ( + rest[..idx].trim().to_string(), + rest[idx + 3..].trim().to_string(), + ), + None => (String::new(), rest.trim().to_string()), + } +} + +/// Whether an argument's description marks it `(Required)`. +fn is_required(prose: &str) -> bool { + prose + .trim_start() + .to_ascii_lowercase() + .starts_with("(required") +} + +// --------------------------------------------------------------------------- +// See Also. +// --------------------------------------------------------------------------- + +/// Extract the command names from `See Also` bullet lines +/// (` * get_clocks`). +fn parse_see_also(lines: &[String]) -> Vec { + let mut out = Vec::new(); + for line in lines { + if let Ok(name) = see_also_entry.parse_next(&mut line.as_str()) { + if !name.is_empty() { + out.push(name.to_string()); + } + } + } + out +} + +/// ` * ` — a See-Also bullet. Returns the command name. +fn see_also_entry<'s>(input: &mut &'s str) -> ModalResult<&'s str> { + space0.parse_next(input)?; + '*'.parse_next(input)?; + space0.parse_next(input)?; + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input) +} + +// --------------------------------------------------------------------------- +// Finalization: identifiers, de-collision, synthesized operands. +// --------------------------------------------------------------------------- + +/// Assign final htcl identifiers, de-collide duplicates, and synthesize +/// a generic trailing operand when the page documented no positional. +fn finalize_arguments(page: &mut ManPage) { + let mut used: std::collections::HashSet = + std::collections::HashSet::new(); + let mut has_positional = false; + + let args = std::mem::take(&mut page.arguments); + for mut arg in args { + let base = match arg.kind { + ArgKind::Positional => { + has_positional = true; + if arg.ident.is_empty() { + "operands".to_string() + } else { + arg.ident.clone() + } + } + _ => arg.flag.clone().unwrap_or_else(|| "arg".to_string()), + }; + let base = sanitize_ident(&base); + // A duplicate flag (the page listing `-foo` twice) is dropped; + // a positional that collides with a flag is renamed. + if used.contains(&base) { + if arg.is_flag() { + continue; + } + arg.ident = unique_ident(&base, &mut used); + } else { + used.insert(base.clone()); + arg.ident = base; + } + page.arguments.push(arg); + } + + if !has_positional { + let ident = unique_ident("operands", &mut used); + page.arguments.push(Argument { + kind: ArgKind::Positional, + ident, + flag: None, + required: false, + synthesized: true, + description: vec![ + "Positional operands passed through to the underlying \ + command (object patterns, names, files, …)." + .to_string(), + ], + }); + } +} + +/// First unused identifier in the `base`, `base_2`, `base_3`, … family. +fn unique_ident( + base: &str, + used: &mut std::collections::HashSet, +) -> String { + let base = sanitize_ident(base); + if used.insert(base.clone()) { + return base; + } + for n in 2.. { + let candidate = format!("{base}_{n}"); + if used.insert(candidate.clone()) { + return candidate; + } + } + unreachable!("exhausted identifier suffixes") +} + +/// Coerce an arbitrary string into the htcl proc-arg grammar +/// (`[A-Za-z_][A-Za-z0-9_]*`). Non-conforming characters become +/// underscores; a digit-leading or empty result gets a leading +/// underscore. Never produces the Tcl-reserved varargs name `args`. +fn sanitize_ident(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 1); + for c in s.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c); + } else { + out.push('_'); + } + } + let needs_lead = out + .as_bytes() + .first() + .map(|b| b.is_ascii_digit()) + .unwrap_or(true); + if needs_lead { + out.insert(0, '_'); + } + if out == "args" { + out = "args_".to_string(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header(line: &str) -> ArgHeader { + parse_arg_header(line).expect("expected an argument header") + } + + #[test] + fn classifies_value_flag() { + let h = header("-fileset - (Optional) The fileset."); + assert_eq!(h.kind, ArgKind::Value); + assert_eq!(h.flag.as_deref(), Some("fileset")); + assert!(!h.required); + assert!(h.prose.starts_with("(Optional)")); + } + + #[test] + fn classifies_boolean_flag() { + let h = header("-norecurse - (Optional) Do not recurse."); + assert_eq!(h.kind, ArgKind::Boolean); + assert_eq!(h.flag.as_deref(), Some("norecurse")); + } + + #[test] + fn required_value_flag() { + let h = header("-period - (Required) The period."); + assert_eq!(h.kind, ArgKind::Value); + assert!(h.required); + } + + #[test] + fn multiword_placeholder_is_value_flag() { + let h = header("-waveform - (Optional) Edges."); + assert_eq!(h.kind, ArgKind::Value); + } + + #[test] + fn classifies_positional_and_recovers_name() { + let h = header(" - (Required) Objects to remove."); + assert_eq!(h.kind, ArgKind::Positional); + assert_eq!(h.name_hint.as_deref(), Some("hw_sio_linkgroups")); + assert!(h.required); + } + + #[test] + fn positional_without_marker_is_optional() { + let h = header(" - Version of the library."); + assert_eq!(h.kind, ArgKind::Positional); + assert!(!h.required); + } + + #[test] + fn non_header_block_is_rejected() { + assert!(parse_arg_header("Note: this is a note.").is_none()); + assert!(parse_arg_header("Plain prose continuation.").is_none()); + } + + #[test] + fn first_ident_token_extraction() { + assert_eq!(first_ident_token("name").as_deref(), Some("name")); + assert_eq!(first_ident_token("arg1 arg2 ...").as_deref(), Some("arg1")); + assert_eq!(first_ident_token("[0:750]"), None); + } + + #[test] + fn sanitize_ident_never_yields_varargs() { + assert_eq!(sanitize_ident("args"), "args_"); + assert_eq!(sanitize_ident("64bit"), "_64bit"); + assert_eq!(sanitize_ident("a-b.c"), "a_b_c"); + } + + #[test] + fn de_collides_positional_against_flag() { + // A flag `-name` and a positional `` must not collide. + let page = parse_man_page( + "demo", + "\nArguments:\n\n -name - (Optional) The flag.\n\n \ + - (Required) The operand.\n", + ); + let idents: Vec<&str> = + page.arguments.iter().map(|a| a.ident.as_str()).collect(); + assert!(idents.contains(&"name")); + assert!(idents.contains(&"name_2")); + } + + #[test] + fn drops_duplicate_flag() { + let page = parse_man_page( + "demo", + "\nArguments:\n\n -quiet - (Optional) Quietly.\n\n \ + -quiet - (Optional) Quietly again.\n", + ); + let quiets = + page.arguments.iter().filter(|a| a.ident == "quiet").count(); + assert_eq!(quiets, 1); + } +} diff --git a/vw-htcl-cmd/tests/generate.rs b/vw-htcl-cmd/tests/generate.rs new file mode 100644 index 0000000..2794570 --- /dev/null +++ b/vw-htcl-cmd/tests/generate.rs @@ -0,0 +1,310 @@ +// Integration tests: parse synthetic and real man pages, then prove the +// generated htcl re-parses cleanly through `vw_htcl` (the same parser +// `vw check` and the LSP use). + +use std::path::Path; + +use vw_htcl_cmd::{generate, parse_man_page, ArgKind, GenerateOptions}; + +/// A man page exercising every argument shape: required value flag, +/// optional value flag, boolean flag, a multi-word placeholder, +/// required positional, optional positional, and a `Note:` block that +/// must fold into the preceding argument. +const SAMPLE: &str = " +Description: + + Creates a thing. Pass it a list like {a b c} and it just works. + + Returns the created thing, or an error if it fails. + +Arguments: + + -period - (Required) The period, must be > 0. + + -name - (Optional) The name of the thing. + + -waveform - (Optional) Edge times. + + -add - (Optional) Add instead of replace. + + -quiet - (Optional) Execute quietly. + + Note: errors on the command line are still returned. + + - (Required) The source objects. + +Examples: + + make_thing -period 10 + +See Also: + + * destroy_thing + * get_things +"; + +fn assert_reparses(htcl: &str) { + let parsed = vw_htcl::parse(htcl); + assert!( + parsed.errors.is_empty(), + "generated htcl failed to parse: {:#?}\n---\n{htcl}", + parsed.errors + ); +} + +#[test] +fn parses_every_argument_shape() { + let page = parse_man_page("make_thing", SAMPLE); + + assert_eq!(page.name, "make_thing"); + assert_eq!(page.see_also, vec!["destroy_thing", "get_things"]); + + let by_ident = |id: &str| { + page.arguments + .iter() + .find(|a| a.ident == id) + .unwrap_or_else(|| panic!("missing arg {id}")) + }; + + let period = by_ident("period"); + assert_eq!(period.kind, ArgKind::Value); + assert!(period.required); + + let name = by_ident("name"); + assert_eq!(name.kind, ArgKind::Value); + assert!(!name.required); + + let waveform = by_ident("waveform"); + assert_eq!(waveform.kind, ArgKind::Value, "multi-word placeholder"); + + let add = by_ident("add"); + assert_eq!(add.kind, ArgKind::Boolean); + + // The `Note:` block must have folded into -quiet's description. + let quiet = by_ident("quiet"); + assert!( + quiet + .description + .iter() + .any(|l| l.contains("still returned")), + "Note block did not fold into -quiet: {:?}", + quiet.description + ); + + let objects = by_ident("objects"); + assert_eq!(objects.kind, ArgKind::Positional); + assert!(objects.required); + // A positional was documented, so none is synthesized. + assert!(page.arguments.iter().all(|a| !a.synthesized)); +} + +#[test] +fn generated_wrapper_reparses() { + let page = parse_man_page("make_thing", SAMPLE); + let htcl = generate(&page, &GenerateOptions::default()); + + // Doc braces are neutralized so the arg-list brace match survives. + assert!( + htcl.contains("{a b c}".replace('{', "(").replace('}', ")").as_str()) + ); + // Natural name + extern-prefixed forward (lowering autogen + // produces the rename plumbing at session startup). + assert!(htcl.contains("proc make_thing {")); + assert!(!htcl.contains("rename")); + // New body shape: non-typed args accumulate into `flags` via + // lappend (safe — strings only). Typed args (objects, cell, + // pin, ...) are passed directly via `-flag $value` so Vivado's + // typed Tcl_Obj survives. + assert!( + htcl.contains("lappend flags -period $period"), + "non-typed value-flag should lappend into flags: {htcl}" + ); + assert!( + htcl.contains("if {$add} { lappend flags -add }"), + "boolean should lappend into flags only when true: {htcl}" + ); + // `objects` is in TYPED_ARG_NAMES → must NOT be lappended + // (would shimmer the typed handle). Must appear in a direct + // invocation as `-objects $objects` or as positional `$objects`. + assert!( + !htcl.contains("lappend flags {*}$objects"), + "typed arg `objects` must not be lappended: {htcl}" + ); + assert!( + !htcl.contains("lappend cmd"), + "old `cmd`-accumulator shape should be gone: {htcl}" + ); + assert!( + htcl.contains("extern::make_thing {*}$flags"), + "direct invocation with {{*}}$flags expected: {htcl}" + ); + assert!( + htcl.contains("$objects"), + "objects must be referenced in the invocation: {htcl}" + ); + + assert_reparses(&htcl); +} + +#[test] +fn synthesizes_operand_when_no_positional() { + let page = parse_man_page( + "current_thing", + "\nDescription:\n\n Gets the current thing.\n\nArguments:\n\n \ + -quiet - (Optional) Quietly.\n", + ); + let synth: Vec<_> = + page.arguments.iter().filter(|a| a.synthesized).collect(); + assert_eq!(synth.len(), 1, "exactly one synthesized operand"); + assert_eq!(synth[0].kind, ArgKind::Positional); + assert!(!synth[0].required); + + assert_reparses(&generate(&page, &GenerateOptions::default())); +} + +#[test] +fn empty_man_page_still_generates_valid_wrapper() { + // No Description, no Arguments — the generator must still emit a + // parseable, self-contained wrapper. + let page = parse_man_page("noop", ""); + let htcl = generate(&page, &GenerateOptions::default()); + assert!(htcl.contains("proc noop {")); + assert_reparses(&htcl); +} + +/// Smoke test over the real Vivado man pages when a local install is +/// present: every page must generate htcl that re-parses cleanly. +#[test] +fn real_man_pages_reparse() { + let dir = "/home/ry/Xilinx/2025.1/Vivado/doc/eng/man"; + if !Path::new(dir).exists() { + eprintln!("skipping: {dir} not present"); + return; + } + let mut checked = 0; + let mut failures = Vec::new(); + let mut stack = vec![std::path::PathBuf::from(dir)]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + continue; + } + let stem = match path.file_name().and_then(|s| s.to_str()) { + Some(s) + if s.chars().all(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' + }) => + { + s + } + _ => continue, // skip tmp.* / *_Copy junk + }; + let text = std::fs::read_to_string(&path).unwrap(); + let page = parse_man_page(stem, &text); + let htcl = generate(&page, &GenerateOptions::default()); + let parsed = vw_htcl::parse(&htcl); + if !parsed.errors.is_empty() { + failures.push(format!("{stem}: {:?}", parsed.errors)); + } + checked += 1; + } + } + eprintln!("checked {checked} real man pages"); + assert!(checked > 500, "expected many man pages, saw {checked}"); + assert!( + failures.is_empty(), + "{} man pages produced unparseable htcl:\n{}", + failures.len(), + failures.join("\n") + ); +} + +// --- return-type emission (step 5) ----------------------------------------- + +#[test] +fn returns_section_emits_type_annotation() { + let src = " +Description: + + Returns a list of cells matching the search. + +Arguments: + + -hierarchical - (Optional) Search hierarchically. + +Returns: + + a list of cells + +See Also: + + * get_cells +"; + let page = parse_man_page("get_things", src); + assert!(page.returns.is_some(), "Returns: section should be parsed"); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + // The `proc get_things { … } list { … }` shape. + assert!( + htcl.contains("list {"), + "expected return-type annotation in: {htcl}" + ); +} + +#[test] +fn returns_section_nothing_emits_unit() { + let src = " +Description: + + Sets things. + +Arguments: + + -quiet - (Optional) Quiet. + +Returns: + + Returns nothing. + +See Also: + + * unset_things +"; + let page = parse_man_page("set_things", src); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + assert!( + htcl.contains(" unit {"), + "expected `unit` return annotation in: {htcl}" + ); +} + +#[test] +fn page_without_returns_section_emits_no_annotation() { + let src = " +Description: + + Does a thing. + +Arguments: + + -x - (Required) Thing. + +See Also: + + * other +"; + let page = parse_man_page("do_a_thing", src); + assert!(page.returns.is_none()); + let htcl = generate(&page, &GenerateOptions::default()); + assert_reparses(&htcl); + // No return-type annotation present. + assert!( + !htcl.contains("unit {") + && !htcl.contains("bd_cell {") + && !htcl.contains("list<"), + "unannotated page should not synthesize a return type: {htcl}" + ); +} diff --git a/vw-htcl/Cargo.toml b/vw-htcl/Cargo.toml new file mode 100644 index 0000000..91aecab --- /dev/null +++ b/vw-htcl/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "vw-htcl" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "htcl language layer: parser, AST, name resolution, signature checking, TCL emission" + +[dependencies] +serde.workspace = true +thiserror.workspace = true +winnow.workspace = true +camino.workspace = true +vw-quote = { path = "../vw-quote" } + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-htcl/src/ast.rs b/vw-htcl/src/ast.rs new file mode 100644 index 0000000..a533063 --- /dev/null +++ b/vw-htcl/src/ast.rs @@ -0,0 +1,585 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Concrete syntax tree for htcl. +//! +//! Every node carries a [`Span`] so the same tree drives diagnostics, +//! hover, navigation, and source-faithful lowering back to TCL. The +//! tree is concrete in the sense that it retains enough information to +//! recover the original source (comments, blank lines, word forms); +//! later passes derive a stripped AST for analysis. + +use crate::span::Span; + +#[derive(Clone, Debug)] +pub struct Document { + pub stmts: Vec, + pub span: Span, +} + +// `Stmt::Command(Command)` is ~320 bytes while the other variants +// are <50; clippy flags the size disparity and suggests boxing +// `Command`. We don't box because: +// - Commands are by far the most common variant (often >95% of +// Stmt instances in real source), so the boxed-pointer +// indirection on the hot path would cost more than the +// wasted bytes in rare Comment/Error variants. +// - Boxing would ripple through ~50 pattern-match sites +// (`let Stmt::Command(cmd) = ...`) and complicate the AST's +// "by-value clone-and-mutate" rewrite passes. +#[derive(Clone, Debug)] +#[allow(clippy::large_enum_variant)] +pub enum Stmt { + Command(Command), + Comment(Comment), + Error(ParseFailure), +} + +impl Stmt { + pub fn span(&self) -> Span { + match self { + Stmt::Command(c) => c.span, + Stmt::Comment(c) => c.span, + Stmt::Error(e) => e.span, + } + } +} + +/// A single TCL command — a whitespace-separated sequence of words, +/// terminated by newline, semicolon, or EOF. +#[derive(Clone, Debug)] +pub struct Command { + pub words: Vec, + pub span: Span, + pub kind: CommandKind, + /// Doc comments (`##`) immediately preceding the command, in + /// source order with the `##` prefix stripped. + pub doc_comments: Vec, + /// Source span covering the whole `##` block — from the first + /// `#` of the first line to the newline after the last line. + /// `None` when the command has no doc comments. Used by the + /// analyzer to answer "is the cursor inside this command's doc + /// block?" so `[NAME]` references inside `##` text can resolve + /// via goto/hover. + pub doc_comments_span: Option, +} + +/// Recognized command shapes. Generic covers any unrecognized command; +/// specific variants exist so downstream passes (symbol tables, the +/// LSP, the structured-proc work in Phase 2) can act on them without +/// re-parsing. +#[derive(Clone, Debug)] +pub enum CommandKind { + Generic, + Set, + Proc(Proc), + Src(SrcImport), + NamespaceEval(NamespaceEval), + /// A `type = ` declaration. Compile-time only + /// — never lowered to Tcl. Together with the required + /// `::repr` / `from` / `to` procs (enforced by the + /// validator), introduces a new newtype the rest of the program + /// can reference in return-type annotations and (later) arg + /// annotations. + TypeDecl(TypeDecl), + /// An `enum = { }` declaration. Compile-time + /// only — the lowerer emits the auto-generated constructor / + /// repr / accessor procs through the repr-codegen path, NOT via + /// shipping the source verbatim. Variants are + /// `IDENT (':' TYPE)?`; the optional `:TYPE` payload makes + /// empty-payload variants first-class. + EnumDecl(EnumDecl), +} + +/// A `namespace eval { }` block. +/// +/// Recognized at parse time so that any `proc` declarations inside +/// the braces register in the document's signature table under the +/// qualified name `::` (Tcl namespace semantics), and +/// the analyzer can offer the same hover / completion / signature +/// help / goto experience for namespaced procs as for top-level +/// ones. The body parses as a script just like a proc body, so +/// nested `namespace eval` blocks compose. +#[derive(Clone, Debug)] +pub struct NamespaceEval { + /// Bare-text namespace name when extractable (the common case), + /// `None` when the name word couldn't be reduced to literal text + /// (e.g. it contains substitutions). Multi-segment names like + /// `foo::bar` are preserved as-is and the analyzer uses them as + /// the full prefix. + pub name: Option, + pub name_span: Span, + pub body_span: Span, + /// The body parsed into statements. Spans are absolute (whole- + /// source) coordinates, same convention as [`Proc::body`]. + pub body: Vec, +} + +/// A `src ` import — load and evaluate another htcl module. +/// +/// The path's *form* is classified at load time, not here: leading +/// `@name/` resolves through the workspace's `vw.toml` dependencies, +/// a leading `/` is filesystem-absolute, anything else is relative to +/// the importing file's directory. `path` is `None` only when the +/// path word couldn't be reduced to literal text (e.g. it contains +/// `$var` / `[cmd]` substitutions); those imports are diagnosed +/// downstream rather than parsed structurally. +#[derive(Clone, Debug)] +pub struct SrcImport { + pub path: Option, + pub path_span: Span, +} + +/// A `proc` declaration. +/// +/// The outer shape (name, args span, body span) comes from the Phase 0 +/// parser. The structured args grammar (Phase 2) is reparsed from +/// `args_span` and stored in [`signature`](Self::signature). When +/// `signature` is `None` the args body couldn't be parsed at all +/// (e.g. mid-edit syntax error); diagnostics for that live in the +/// document's parse-error list. +#[derive(Clone, Debug)] +pub struct Proc { + /// Bare-text proc name when it could be extracted; `None` for + /// programmatically-named procs (e.g. names built from + /// substitution). + pub name: Option, + pub name_span: Span, + pub args_span: Span, + pub body_span: Span, + pub signature: Option, + /// Optional return-type annotation: the 4th word of a + /// `proc NAME { args } TYPE { body }` declaration, parsed as a + /// [`TypeExpr`]. `None` means "no annotation present" — the + /// proc still works, but downstream type-driven machinery + /// (REPL repr printer, hover, future call-site validation) + /// falls back to its untyped path. Bracketed forms like + /// `{dict}` are unwrapped before type-parsing. + pub return_type: Option, + /// Source span of the 4th-word type slot when present; `None` + /// when the proc has no annotation. The span covers the outer + /// word including any wrapping braces, so diagnostics can + /// underline the entire annotation. + pub return_type_span: Option, + /// The body parsed into statements, with spans in absolute + /// (whole-source) coordinates. Populated by a post-pass after the + /// outer parse; empty until then and for bodies that are pure + /// braced text with no commands. Lowering still ships the body + /// verbatim from [`body_span`](Self::body_span) — this field + /// exists so navigation, hover, and analysis can see *into* a + /// proc body. Nested procs declared here have their own `body` + /// populated recursively. + pub body: Vec, + /// Attributes attached to this proc declaration itself (as + /// opposed to [`ProcArg::attributes`], which live on individual + /// args). Populated by the parser from `@name(…)` items that + /// appear at statement position immediately BEFORE the `proc` + /// keyword. Mirrors the doc-comment attachment pattern the + /// parser already uses. Currently used by `vw test` to + /// recognize `@test`- and `@test(dedicated-eda)`-marked procs. + pub attributes: Vec, +} + +impl Proc { + /// Look up a proc-level attribute by name. Returns the first + /// match — attributes with duplicate names are unusual but not + /// rejected at the parse level. + pub fn attribute(&self, name: &str) -> Option<&Attribute> { + self.attributes.iter().find(|a| a.name == name) + } +} + +/// A `type NAME = UNDERLYING` declaration. +/// +/// Introduces a newtype wrapper around an existing type. The +/// validator requires the user to ALSO define three procs in the +/// `::` namespace: `repr` (rendering to a `string`), `from` +/// (lifting an underlying value, with optional validation), and +/// `to` (extracting the underlying value). See the validator for +/// the exact signature shapes enforced. +/// +/// Compile-time only — never lowered to Tcl. The newtype's runtime +/// representation is identical to the underlying type; the +/// distinction lives entirely in the analyzer / printer / future +/// type-checker. +#[derive(Clone, Debug)] +pub struct TypeDecl { + /// Bare-text type name when extractable; `None` for + /// programmatically-named declarations (vanishingly rare; + /// kept consistent with [`Proc::name`]'s convention). + pub name: Option, + pub name_span: Span, + /// The underlying type, parsed from the right-hand side of `=`. + /// `None` when the right-hand side couldn't be parsed as a + /// type expression (e.g. mid-edit). Diagnostics for that live + /// in the document's parse-error list. + pub underlying: Option, + /// Span of the underlying-type word (outer, including any + /// wrapping braces) so diagnostics can underline it. + pub underlying_span: Span, +} + +/// An `enum = { }` declaration. The body is +/// brace-wrapped and newline-separated; each variant is +/// `IDENT (':' TYPE)?`. +/// +/// Compile-time only — codegen lowers this to a `namespace eval +/// { … }` block containing auto-generated constructors, +/// `repr`/`from`/`to`, and `tag`/`payload` accessors. The +/// validator enforces variant-name uniqueness within an enum and +/// that variant payload types reference known type names. +#[derive(Clone, Debug)] +pub struct EnumDecl { + /// Bare-text enum name when extractable. + pub name: Option, + pub name_span: Span, + /// Declared variants, in source order. + pub variants: Vec, + /// Span of the brace-wrapped variants block (outer, including + /// the braces) so diagnostics can underline it. + pub body_span: Span, +} + +/// One variant inside an [`EnumDecl`]. `payload` is `None` for +/// empty-payload variants (e.g. `North` in `enum Direction = { +/// North; South: int }`). +#[derive(Clone, Debug)] +pub struct EnumVariant { + pub name: String, + pub name_span: Span, + pub payload: Option, + /// Span of the payload-type word, or zero-length at the + /// variant's end position for empty-payload variants. + pub payload_span: Span, + /// Span covering the full `NAME (':' TYPE)?` form. + pub span: Span, +} + +/// Side-table entry produced by the validator's overload-classifier +/// pass: for a public proc name that resolved to an enum-overload +/// set, records which enum drives dispatch and where each variant's +/// specialization lives. The codegen step uses this to synthesize +/// the public dispatcher proc; the analyzer uses it to render +/// overload information in hover / signature help. +#[derive(Clone, Debug)] +pub struct OverloadInfo { + /// The public, user-facing proc name (e.g. `handle_prop`). + pub public_name: String, + /// The enum this overload set dispatches on (e.g. `Property`). + pub enum_name: String, + /// Shared arg name across all overload arms (e.g. `v`). The + /// validator enforces every arm uses the same name so the + /// dispatcher can pass the payload via the kwargs protocol + /// (`- `) without per-arm gymnastics. + pub dispatch_arg_name: String, + /// One entry per variant, in declaration order on the enum. + pub variants: Vec, + /// Span on the first overload's name — used as the diagnostic + /// anchor for overload-set-wide errors. + pub anchor_span: Span, +} + +#[derive(Clone, Debug)] +pub struct OverloadVariant { + /// The variant short-name (e.g. `Scalar`, `Nested`). + pub variant_name: String, + /// The mangled internal proc name the specialization runs + /// under at runtime (e.g. `__handle_prop__Scalar`). + pub mangled_proc_name: String, + /// Span of the variant-arg annotation on the specialization's + /// first argument — diagnostic anchor when something's + /// specifically wrong with this arm. + pub dispatch_arg_span: Span, +} + +/// A type expression — the syntactic form of a type used in +/// `proc NAME { args } TYPE { body }` return annotations and on the +/// right-hand side of `type NAME = TYPE` declarations. +/// +/// Newtypes (`bd_cell`, `widget`, user inventions) and primitives +/// (`string`, `int`, `bool`, `unit`) share the [`Named`] variant — +/// the distinction lives in the validator's type table, not the +/// AST. Containers (`list`, `dict`, and any future shape +/// with the same `name` surface) are [`Generic`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TypeExpr { + Named { + name: String, + span: Span, + }, + Generic { + name: String, + name_span: Span, + args: Vec, + /// Full span including `<` … `>`. + span: Span, + }, + /// `Enum::Variant` — a qualified path naming a single variant + /// of a declared enum. Only legal as an arg-type annotation on + /// an overloaded handler proc (the dispatch indicator); the + /// validator rejects this variant anywhere else (return types, + /// generic args, nested positions). + Qualified { + namespace: String, + variant: String, + namespace_span: Span, + variant_span: Span, + /// Full span covering `namespace::variant`. + span: Span, + }, +} + +impl TypeExpr { + pub fn name(&self) -> &str { + match self { + TypeExpr::Named { name, .. } | TypeExpr::Generic { name, .. } => { + name + } + TypeExpr::Qualified { namespace, .. } => namespace, + } + } + + pub fn span(&self) -> Span { + match self { + TypeExpr::Named { span, .. } + | TypeExpr::Generic { span, .. } + | TypeExpr::Qualified { span, .. } => *span, + } + } +} + +/// Structured proc-argument signature. +/// +/// One entry per declared argument, in source order. The order is the +/// canonical positional order used when lowering keyword-arg call +/// sites to Tcl-positional calls for the EDA backend. +#[derive(Clone, Debug)] +pub struct ProcSignature { + pub args: Vec, + pub span: Span, + /// The declared return type, copied here from [`Proc::return_type`] + /// at parse time so the signature-table-based lookup paths + /// (REPL formatter, hover) don't have to re-walk back to the + /// Proc node. `None` for unannotated procs. + pub return_type: Option, +} + +impl ProcSignature { + pub fn find(&self, name: &str) -> Option<&ProcArg> { + self.args.iter().find(|a| a.name == name) + } +} + +#[derive(Clone, Debug)] +pub struct ProcArg { + pub name: String, + pub name_span: Span, + pub doc_comments: Vec, + /// Source span covering the whole `##` block that attaches to + /// this arg — `None` when the arg has no doc comments. See + /// [`Command::doc_comments_span`] for the analyzer-side rationale. + pub doc_comments_span: Option, + pub attributes: Vec, + /// Optional `: TYPE` annotation on the arg. `Some` when the + /// source carries `name: bd_cell` style; `None` when the arg + /// is untyped (the legacy form). Used by the validator (full + /// shape check on newtype repr/from/to procs) and by the + /// analyzer's hover / signature-help displays. + pub type_annotation: Option, + pub span: Span, +} + +impl ProcArg { + pub fn attribute(&self, name: &str) -> Option<&Attribute> { + self.attributes.iter().find(|a| a.name == name) + } +} + +/// Raw attribute as parsed: name plus zero or more comma-separated +/// values. Semantic interpretation (default, required, enum, range, +/// requires, conflicts, deprecated) lives in the validators, not +/// here — keeping the AST shape unopinionated lets new attribute +/// names land without a parser change. +#[derive(Clone, Debug)] +pub struct Attribute { + pub name: String, + pub name_span: Span, + pub values: Vec, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub enum AttributeValue { + Integer { + value: i64, + span: Span, + }, + String { + value: String, + span: Span, + }, + Ident { + value: String, + span: Span, + }, + /// A `key=value` item in an attribute value list, e.g. + /// `@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S)` where + /// `part=xcvm3358…` produces a `Keyed { key: "part", value: + /// Ident(…) }`. Positional items in the same list continue to + /// use the plain [`Integer`]/[`String`]/[`Ident`] variants, + /// so old consumers keep working. + Keyed { + key: String, + key_span: Span, + value: Box, + span: Span, + }, +} + +impl AttributeValue { + pub fn span(&self) -> Span { + match self { + AttributeValue::Integer { span, .. } + | AttributeValue::String { span, .. } + | AttributeValue::Ident { span, .. } + | AttributeValue::Keyed { span, .. } => *span, + } + } + + /// If this item is a `key=value` pair, return `(key, + /// inner_value)`. `None` for positional items. + pub fn as_keyed(&self) -> Option<(&str, &AttributeValue)> { + match self { + AttributeValue::Keyed { key, value, .. } => Some((key, value)), + _ => None, + } + } + + /// Render the value back to a Tcl-style literal, suitable for + /// comparison against a runtime arg or for emitting in lowered + /// Tcl. Integers and idents stringify as-is; strings get + /// double-quoted with naive escaping. + pub fn to_tcl_literal(&self) -> String { + match self { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } => value.clone(), + AttributeValue::String { value, .. } => { + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } + AttributeValue::Keyed { key, value, .. } => { + format!("{key}={}", value.to_tcl_literal()) + } + } + } + + pub fn as_str(&self) -> &str { + match self { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value, + AttributeValue::Integer { .. } => "", + AttributeValue::Keyed { .. } => "", + } + } +} + +#[derive(Clone, Debug)] +pub struct Comment { + /// Comment text with the leading `#` removed; for `##` doc + /// comments, both `#`s are removed. + pub text: String, + pub span: Span, + pub is_doc: bool, +} + +#[derive(Clone, Debug)] +pub struct ParseFailure { + pub message: String, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub struct Word { + pub form: WordForm, + pub parts: Vec, + pub span: Span, + /// Populated at parse time for braced-word args that are + /// KNOWN to be script bodies: the third arg of `foreach`, the + /// fourth arg of `dict for`, both branches of `if`, etc. + /// `None` for everything else — including bare braced strings + /// and braced expressions. + /// + /// Interior spans are absolute (whole-source) coordinates, + /// same convention as [`Proc::body`]. Populated by + /// `parser::populate_procs` after the initial word parse. + /// + /// Downstream tools (validator, putr rewriter, LSP hover / + /// goto, syntax highlighter) descend into this when + /// non-`None` so behavior inside a control-flow body matches + /// behavior at the top level. + pub body: Option>, +} + +impl Word { + /// If this word is a single literal text part (no interpolation), + /// return its value. Useful for matching command names, fixed + /// keywords, and option flags without rebuilding the string. + pub fn as_text(&self) -> Option<&str> { + match self.parts.as_slice() { + [WordPart::Text { value, .. }] => Some(value), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WordForm { + Bare, + Quoted, + Braced, +} + +#[derive(Clone, Debug)] +pub enum WordPart { + Text { + value: String, + span: Span, + }, + VarRef { + name: String, + span: Span, + /// True when the source used the `${name}` bracketed form. + /// Preserved for lowering so `"${ip}_wrapper.vhd"` doesn't + /// re-emit as `"$ip_wrapper.vhd"` — Tcl's `$` reads a + /// greedy ident (`[A-Za-z0-9_:]+`) and would then try to + /// dereference the non-existent `ip_wrapper` variable. See + /// `lower_word_parts` for the emit side. + braced: bool, + }, + /// `[ cmd ... ]` command substitution. `source` is the raw interior + /// text (between the brackets) and `span` covers the whole + /// `[...]`. `body` is populated by a post-pass that recursively + /// parses the interior into statements with absolute spans, so + /// hover / goto / signature-help can descend in. + CmdSubst { + source: String, + span: Span, + body: Vec, + }, + Escape { + value: char, + span: Span, + }, +} + +impl WordPart { + pub fn span(&self) -> Span { + match self { + WordPart::Text { span, .. } + | WordPart::VarRef { span, .. } + | WordPart::CmdSubst { span, .. } + | WordPart::Escape { span, .. } => *span, + } + } +} diff --git a/vw-htcl/src/cmdline.rs b/vw-htcl/src/cmdline.rs new file mode 100644 index 0000000..cf37c2f --- /dev/null +++ b/vw-htcl/src/cmdline.rs @@ -0,0 +1,336 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lightweight analysis of the partially-typed command at the cursor. +//! +//! Completion and signature help need to know, mid-edit, which command +//! the cursor sits in and which word is being typed. The full AST is +//! unreliable here *precisely because* the text is incomplete, so we +//! scan the raw source backward to the nearest command boundary +//! (newline, `;`, or the `[` that opens a command substitution) and +//! tokenize on whitespace. This is a deliberately shallow Tcl reader — +//! good enough to drive IDE affordances, not to execute. + +use crate::span::Span; + +#[derive(Clone, Debug)] +pub struct CmdLine<'a> { + /// Whitespace-separated complete words before the cursor. The + /// first, when present, is the command name. + pub words: Vec<&'a str>, + /// The word currently under the cursor: the trailing token when + /// the prefix doesn't end in whitespace, otherwise empty. + pub partial: &'a str, + /// Span of `partial` in the source — the range a completion should + /// replace. Zero-width (an insertion point) when `partial` is + /// empty. + pub partial_span: Span, +} + +impl CmdLine<'_> { + /// The command name (first complete word). `None` while the cursor + /// is still on the first word — i.e. command-name position. + pub fn command_name(&self) -> Option<&str> { + self.words.first().copied() + } + + /// True when the cursor is in command-name position (no complete + /// words precede it). + pub fn in_command_position(&self) -> bool { + self.words.is_empty() + } + + /// Flags (`-foo`) already supplied among the complete words after + /// the command name. + pub fn used_flags(&self) -> impl Iterator { + self.words + .iter() + .skip(1) + .copied() + .filter(|w| w.starts_with('-')) + } +} + +/// Analyze the command the cursor at `offset` is editing. +pub fn analyze(source: &str, offset: u32) -> CmdLine<'_> { + let off = (offset as usize).min(source.len()); + let bytes = source.as_bytes(); + + // Walk back to the start of the current command. The boundary + // depends on the cursor's *bracket nesting*: inside a `[ … ]`, + // newlines are whitespace (matching the parser), only `;` and the + // opening `[` terminate. Outside brackets, `\n` and `;` both + // terminate at the cursor's level. + // + // We track depth as we walk backward — each `]` going back means + // we're entering a deeper region, each `[` brings us back out. If + // we hit an unmatched `[` (the opening bracket of the substitution + // the cursor sits in), that's the command boundary. Otherwise the + // closest `\n`/`;` we passed at depth 0 wins. We have to scan past + // a candidate `\n`/`;` because an enclosing `[` further back would + // override it. + let mut depth: i32 = 0; + let mut nearest_top_sep: Option = None; + let mut bracket_open: Option = None; + let mut i = off; + while i > 0 { + i -= 1; + match bytes[i] { + b']' => depth += 1, + b'[' => { + if depth > 0 { + depth -= 1; + } else { + bracket_open = Some(i + 1); + break; + } + } + b'\n' | b';' + if depth == 0 + && nearest_top_sep.is_none() + && !escaped_by_backslash(bytes, i) + && !(bytes[i] == b'\n' + && next_line_is_flag_continuation(bytes, i)) => + { + // The `!escaped_by_backslash` guard above is Tcl's + // line-continuation rule: `\` and `\;` are + // escaped literals, not separators. Without it, a + // multi-line invocation like + // + // create_clk_wizard_clkout \ + // -cell $clk \ + // -req + // + // would have its completion fall off the cliff because + // the analyzer thought line 3 was a fresh command. + // + // The `next_line_is_flag_continuation` guard mirrors + // the parser's dash-line-continuation rule so the same + // shape without backslashes also completes correctly: + // + // create_clk_wizard_clkout + // -cell $clk + // -req + nearest_top_sep = Some(i + 1); + } + _ => {} + } + } + let start = bracket_open.or(nearest_top_sep).unwrap_or(0); + let prefix = &source[start..off]; + + // The partial word is the trailing run of non-whitespace, unless + // the prefix already ends in whitespace (then we're between words). + let partial_len: usize = prefix + .chars() + .rev() + .take_while(|c| !c.is_whitespace()) + .map(char::len_utf8) + .sum(); + let split = prefix.len() - partial_len; + let head = &prefix[..split]; + let partial = &prefix[split..]; + + CmdLine { + words: head.split_whitespace().collect(), + partial, + partial_span: Span::new((start + split) as u32, off as u32), + } +} + +/// Peek past `bytes[i]` (a `\n`) and any inline whitespace on the +/// next line: does the first non-whitespace byte look like a flag +/// (`-` followed by letter/digit/`-`)? Mirrors the parser's dash- +/// line-continuation rule so the analyzer's cursor-at-end +/// completion path agrees with the parser about whether an +/// unescaped newline ends a command or extends it. +fn next_line_is_flag_continuation(bytes: &[u8], i: usize) -> bool { + let mut j = i + 1; + while j < bytes.len() { + match bytes[j] { + b' ' | b'\t' | b'\r' => j += 1, + _ => break, + } + } + if j >= bytes.len() || bytes[j] != b'-' { + return false; + } + let next = bytes.get(j + 1).copied().unwrap_or(b'\0'); + // Flag-shaped: letter/digit/underscore/`-`. See the parser's + // `next_line_is_flag_continuation` for the rationale on the + // underscore case (real Vivado flags like `-_64bit`). + next.is_ascii_alphanumeric() || next == b'-' || next == b'_' +} + +/// True when the byte at `i` is preceded by an odd-length run of +/// backslashes — Tcl's escape rule. `bytes[i]` itself is not consulted. +fn escaped_by_backslash(bytes: &[u8], i: usize) -> bool { + let mut j = i; + let mut count = 0usize; + while j > 0 && bytes[j - 1] == b'\\' { + count += 1; + j -= 1; + } + count % 2 == 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at_end(src: &str) -> CmdLine<'_> { + analyze(src, src.len() as u32) + } + + #[test] + fn command_position_with_partial() { + let line = at_end("gr"); + assert!(line.in_command_position()); + assert_eq!(line.partial, "gr"); + assert_eq!(line.partial_span, Span::new(0, 2)); + } + + #[test] + fn argument_position_after_name() { + let line = at_end("greet "); + assert!(!line.in_command_position()); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, ""); + assert_eq!(line.partial_span, Span::new(6, 6)); + } + + #[test] + fn partial_flag_after_name() { + let line = at_end("greet -na"); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, "-na"); + assert_eq!(line.partial_span.slice("greet -na"), "-na"); + } + + #[test] + fn used_flags_are_reported() { + let line = at_end("f -a 1 -b "); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-a", "-b"]); + } + + #[test] + fn resets_at_command_substitution() { + // Only the text inside the `[...]` counts as the command. + let src = "puts [greet -na"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("greet")); + assert_eq!(line.partial, "-na"); + } + + #[test] + fn resets_at_newline() { + let src = "set x 1\ngr"; + let line = analyze(src, src.len() as u32); + assert!(line.in_command_position()); + assert_eq!(line.partial, "gr"); + } + + #[test] + fn ignores_newlines_inside_brackets() { + // The cursor sits on `-cell ` in a multi-line `[ … ]`. The + // analyzer must skip the intervening newlines so it can still + // see `create_cpm5_cpm_pcie0` as the command name. + let src = "\ +set x [ + create_cpm5_cpm_pcie0 + -cell "; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_cpm5_cpm_pcie0")); + assert_eq!(line.partial, ""); + // The flag in the middle counts as already-used. + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn active_partial_flag_across_lines() { + // Partial `-max_link_` typed on a fresh line of a multi-line + // bracket should still be recognized as the partial word, and + // the command name should still be the bracket's first word. + let src = "\ +set x [ + create_cpm5_cpm_pcie0 + -cell cpm5 + -max_link_"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_cpm5_cpm_pcie0")); + assert_eq!(line.partial, "-max_link_"); + } + + /// Dash-line continuation without `\`: the completion path + /// must agree with the parser that a newline followed by a + /// `-flag` line extends the current command, so cursor-at-end + /// resolves to the command's flag position. + #[test] + fn dash_led_next_line_continuation_no_backslash() { + let src = "\ +create_clk_wizard_clkout + -cell $clk + -req"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_clk_wizard_clkout")); + assert_eq!(line.partial, "-req"); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn backslash_newline_continuation_keeps_command_alive() { + // A `\` is Tcl's line continuation — the analyzer + // must look through it so the cursor at the end of line 3 is + // still "argument position of `create_clk_wizard_clkout`," + // not a fresh top-level command. + let src = "\ +create_clk_wizard_clkout \\ + -cell $clk \\ + -req"; + let line = analyze(src, src.len() as u32); + assert_eq!(line.command_name(), Some("create_clk_wizard_clkout")); + assert_eq!(line.partial, "-req"); + let used: Vec<&str> = line.used_flags().collect(); + assert_eq!(used, vec!["-cell"]); + } + + #[test] + fn escaped_double_backslash_newline_still_separates() { + // `\\` ends with a literal backslash followed by a + // real command boundary — the second-to-last command is a + // separate statement. + let src = "set x foo\\\\\nbar"; + let line = analyze(src, src.len() as u32); + // The trailing `\\` is a literal backslash, then `\n` + // separates, so `bar` is a fresh command. + assert!(line.in_command_position(), "{:?}", line.words); + assert_eq!(line.partial, "bar"); + } + + #[test] + fn skips_balanced_inner_brackets() { + // Walking back past a complete `[…]` shouldn't fool the + // analyzer into thinking the cursor is at top level when it's + // really inside another, *outer* bracket. + let src = "\ +set x [ + [a b] + outer "; + let line = analyze(src, src.len() as u32); + // The cursor's enclosing bracket is the outer one; its first + // word is the standalone `[a b]` substitution, not a simple + // identifier — so command_name is None, but partial is empty + // (we're between words on a continuation line). The point is + // that the *outer* bracket is what we recognized, not the + // inner one. + let used: Vec<&str> = line.used_flags().collect(); + assert!(used.is_empty(), "{used:?}"); + // `outer` is the second word inside the outer bracket; the + // first word was the `[…]` substitution itself. + assert!(line.words.contains(&"outer"), "{:?}", line.words); + } +} diff --git a/vw-htcl/src/complete.rs b/vw-htcl/src/complete.rs new file mode 100644 index 0000000..e2f0a0f --- /dev/null +++ b/vw-htcl/src/complete.rs @@ -0,0 +1,858 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Code completion for htcl. +//! +//! Two contexts, both keyed off [`cmdline::analyze`]: +//! +//! - **Command position** (typing the first word) → the names of +//! `proc`s declared in the document. +//! - **Argument position** (after a known proc's name) → that proc's +//! `-flag` arguments, minus any already supplied. +//! +//! Pure analysis: returns structured [`Completion`]s referencing the +//! document; the LSP backend maps them to `CompletionItem`s and the +//! REPL will render them its own way. Vivado builtins are not offered +//! yet — that needs the UG835 command database (project-plan Phase 8). + +use std::fmt::Write; + +use crate::ast::{ + Attribute, AttributeValue, CommandKind, Document, ProcArg, ProcSignature, + Stmt, +}; +use crate::cmdline::{self, CmdLine}; +use crate::span::Span; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CompletionKind { + /// A `proc` name in command position. + Proc, + /// A `-flag` keyword argument of a known proc. + Flag, + /// A value from a flag's `@enum(...)` constraint. + EnumValue, + /// A constructor call inferred from a `Construct with [path]` doc + /// hint on the target arg — inserts a multi-line command- + /// substitution block with the cursor positioned to start typing + /// the constructor's own flags. + Constructor, +} + +#[derive(Clone, Debug)] +pub struct Completion { + /// Text shown in the list (`greet`, `-name`, or the constructor + /// path for `Constructor` kind). Also used as the inserted text + /// unless `insert_text` overrides it. + pub label: String, + pub kind: CompletionKind, + /// Short, single-line annotation shown inline next to the label. + pub detail: Option, + /// Longer markdown shown in the item's documentation popup. + pub documentation: Option, + /// Source range the inserted text replaces (the partial word, or a + /// zero-width insertion point between words). + pub replace: Span, + /// Text to insert instead of `label` — used by snippet-shaped + /// completions (`Constructor`) whose visible label is a plain + /// name but whose insertion is a multi-line block. When `None` + /// the LSP layer inserts `label` verbatim. + pub insert_text: Option, + /// If true, `insert_text` uses LSP snippet syntax (`$0`, `${1:foo}`, + /// etc.) and the LSP layer sets `InsertTextFormat::SNIPPET`. + pub snippet: bool, +} + +struct ProcInfo<'a> { + /// Qualified name as it would be called — bare proc name for + /// top-level declarations, `::` for procs declared + /// inside `namespace eval` blocks. + name: String, + doc_comments: &'a [String], + signature: Option<&'a ProcSignature>, +} + +/// Completions available at `offset`. +pub fn complete_at( + document: &Document, + source: &str, + offset: u32, +) -> Vec { + complete_at_with_extras(document, source, offset, &[]) +} + +/// Same as [`complete_at`] but also considers a workspace-side +/// document as an extra source of proc definitions. +/// +/// The LSP calls this way so it can: +/// - use the **current** local text (`document` + `source`) for +/// cmdline-scan context — so what the user *just typed* is what +/// drives `in_command_position` / `partial` / `-flag` detection; +/// - AND still pick up cross-file proc names (`gtwiz_versal::configure`, +/// etc.) from a stale but recently-committed workspace parse +/// (`workspace_document`). +/// +/// Local procs shadow workspace ones on name collision. `workspace_ +/// document` should be a document parsed from the merged workspace +/// view (local + all transitive imports); the local prefix will +/// duplicate the local file's procs, so the shadowing rule dedupes +/// them without extra bookkeeping. +/// +/// Passing `None` is exactly equivalent to `complete_at` — no +/// workspace symbols get added. +pub fn complete_at_with_extras( + document: &Document, + source: &str, + offset: u32, + workspace_documents: &[&Document], +) -> Vec { + // Inside a proc's argument-declaration braces, command/flag + // completion is meaningless (attribute completion will live here + // later). Stay quiet rather than offer nonsense. + if in_proc_args(&document.stmts, offset) { + return Vec::new(); + } + + let line = cmdline::analyze(source, offset); + let mut procs = collect_procs(document); + // Merge in workspace-provided procs. Local procs already in + // `procs` shadow workspace ones — we only append a workspace + // proc when no local proc with the same qualified name exists. + for ws in workspace_documents { + let ws_procs = collect_procs(ws); + for wp in ws_procs { + if !procs.iter().any(|p| p.name == wp.name) { + procs.push(wp); + } + } + } + + if line.in_command_position() { + return complete_proc_names(&procs, &line); + } + + // If the previous complete word is a `-flag`, the cursor is in + // value position — even if the partial is empty (user just hit + // space after the flag). Offer the flag's `@enum(...)` choices + // when it has them; otherwise stay silent so the user can type a + // free-form value (string, int, etc.) without a flag list popping + // up in front of it. + // + // If the partial *starts with* `-` we step back into flag-typing + // mode regardless — the user is clearly typing a new flag. + let last_word_is_flag = line.words.len() >= 2 + && line.words.last().is_some_and(|w| w.starts_with('-')); + if last_word_is_flag && !line.partial.starts_with('-') { + // In value position, offer only value-shaped completions — + // enum choices and `Construct with [...]` snippets. Never + // fall through to flag completion: a free-form value slot + // has no reason to pop a flag list in front of what the user + // is typing. + let mut items = complete_enum_values(&procs, &line); + items.extend(complete_constructor(&procs, &line)); + return items; + } + + complete_flags(&procs, &line) +} + +/// If the flag currently in value position has a `Construct with +/// [path]` hint in its doc comment, emit a single snippet completion +/// that inserts a multi-line command-substitution block calling that +/// constructor. The cursor lands after the constructor name so the +/// user can immediately start typing its own `-flag value` pairs. +/// +/// Empty when the flag has no such hint (so the caller can fall +/// through to normal flag/value handling). +fn complete_constructor( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + let Some(last) = line.words.last() else { + return Vec::new(); + }; + let Some(flag) = last.strip_prefix('-') else { + return Vec::new(); + }; + let Some(arg) = sig.find(flag) else { + return Vec::new(); + }; + let Some(path) = extract_constructor_hint(&arg.doc_comments) else { + return Vec::new(); + }; + // Only offer when the partial (what the user has typed after the + // flag) is either empty or a prefix of the constructor path — a + // free-form value like `$my_var` shouldn't fight the constructor + // suggestion. + if !line.partial.is_empty() && !path.starts_with(line.partial) { + return Vec::new(); + } + // Constructor invocation, multi-line for readability, `$0` sets + // the LSP cursor after the constructor name so `-flag` completion + // picks up next. + let insert = format!("[\n {path} $0\n]"); + vec![Completion { + label: path.clone(), + kind: CompletionKind::Constructor, + detail: Some(format!("constructor for -{}", arg.name)), + documentation: Some(format!( + "Insert a `\\[{path} ...\\]` command-substitution block \ + for the `-{}` slot.", + arg.name + )), + replace: line.partial_span, + insert_text: Some(insert), + snippet: true, + }] +} + +/// Scan doc comments for the `Construct with [path::name]` idiom and +/// return the bracketed path if found. Matches on any line the doc +/// author put it on; case-insensitive on the `construct with` phrase +/// so `Construct` / `construct` / `CONSTRUCT` all work. +fn extract_constructor_hint(doc_comments: &[String]) -> Option { + for line in doc_comments { + let lower = line.to_ascii_lowercase(); + let mut cursor = 0; + while let Some(rel) = lower[cursor..].find("construct with [") { + let start = cursor + rel + "construct with [".len(); + if let Some(end_rel) = line[start..].find(']') { + let path = &line[start..start + end_rel]; + if is_valid_path(path) { + return Some(path.to_string()); + } + cursor = start + end_rel + 1; + } else { + break; + } + } + } + None +} + +fn is_valid_path(s: &str) -> bool { + !s.is_empty() + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') + && s.chars().next().is_some_and(|c| !c.is_ascii_digit()) +} + +/// `@enum(…)` value completions when the cursor sits in value +/// position. Returns empty when the flag has no `@enum` (so the +/// caller can fall back to flag completion). +fn complete_enum_values( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + // The flag whose value we're completing is the last word on the + // line; if it isn't a `-flag`, the user is between options and + // there's nothing to enum-complete. + let Some(last) = line.words.last() else { + return Vec::new(); + }; + let Some(flag) = last.strip_prefix('-') else { + return Vec::new(); + }; + let Some(arg) = sig.find(flag) else { + return Vec::new(); + }; + let Some(enum_attr) = arg.attribute("enum") else { + return Vec::new(); + }; + + let needle = line.partial; + enum_attr + .values + .iter() + .filter_map(|v| { + let raw = enum_value_text(v); + // Filter by either the bare or quoted form so a user typing + // `Mas` matches the value `Master Mode` whose insert form + // is `"Master Mode"`. + if !raw.starts_with(needle) + && !quote_for_completion(&raw).starts_with(needle) + { + return None; + } + let insert = quote_for_completion(&raw); + Some(Completion { + label: insert.clone(), + kind: CompletionKind::EnumValue, + detail: Some(format!("value for -{}", arg.name)), + documentation: crate::doc::brief(&arg.doc_comments), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + }) + .collect() +} + +fn enum_value_text(v: &AttributeValue) -> String { + match v { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.clone(), + // `key=value` in an @enum(…) doesn't make semantic sense — + // enum values are positional. Render the whole thing as + // the literal `key=value` string. + AttributeValue::Keyed { .. } => v.to_tcl_literal(), + } +} + +/// Quote `s` for use as a value on a call site if it can't ride as a +/// bare word. Mirrors the rule [`crate::emit::Word::lit`] uses: bare +/// when safe, double-quoted with `\`/`"` escapes otherwise. +fn quote_for_completion(s: &str) -> String { + let needs = s.is_empty() + || s.chars().any(|c| { + c.is_whitespace() + || matches!( + c, + ';' | '"' | '\\' | '[' | ']' | '{' | '}' | '$' | '#' + ) + }); + if needs { + let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } else { + s.to_string() + } +} + +fn complete_proc_names( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + procs + .iter() + .filter(|p| p.name.starts_with(line.partial)) + .map(|p| Completion { + label: p.name.to_string(), + kind: CompletionKind::Proc, + detail: first_doc_line(p.doc_comments), + documentation: proc_documentation(p), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + .collect() +} + +fn complete_flags( + procs: &[ProcInfo<'_>], + line: &CmdLine<'_>, +) -> Vec { + let Some(name) = line.command_name() else { + return Vec::new(); + }; + let Some(proc) = procs.iter().find(|p| p.name == name) else { + return Vec::new(); + }; + let Some(sig) = proc.signature else { + return Vec::new(); + }; + + let used: Vec<&str> = line.used_flags().collect(); + let needle = line.partial; + let bare_needle = needle.trim_start_matches('-'); + + sig.args + .iter() + .filter_map(|arg| { + let label = format!("-{}", arg.name); + // Don't re-offer a flag already on the line, unless it's + // the very word being typed. + if used.iter().any(|u| *u == label) && needle != label { + return None; + } + // Match either the dashed form (`-na`) or the bare name + // (`na`); an empty needle matches everything. + if !label.starts_with(needle) && !arg.name.starts_with(bare_needle) + { + return None; + } + Some(Completion { + label, + kind: CompletionKind::Flag, + detail: flag_detail(arg), + documentation: Some(arg_documentation(arg)), + replace: line.partial_span, + insert_text: None, + snippet: false, + }) + }) + .collect() +} + +fn collect_procs(document: &Document) -> Vec> { + let mut out = Vec::new(); + collect_procs_in(&document.stmts, "", &mut out); + out +} + +fn collect_procs_in<'a>( + stmts: &'a [Stmt], + prefix: &str, + out: &mut Vec>, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + out.push(ProcInfo { + name: qualified, + doc_comments: &cmd.doc_comments, + signature: proc.signature.as_ref(), + }); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + collect_procs_in(&ns.body, &nested, out); + } + _ => {} + } + } +} + +/// True if `offset` is inside any proc's argument-declaration braces, +/// at any nesting depth. +fn in_proc_args(stmts: &[Stmt], offset: u32) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.args_span.contains(offset) { + return true; + } + if in_proc_args(&proc.body, offset) { + return true; + } + } + false +} + +fn first_doc_line(docs: &[String]) -> Option { + crate::doc::brief(docs) +} + +fn proc_documentation(p: &ProcInfo<'_>) -> Option { + // Use `extended` (body only) here because the call site populates + // `CompletionItem::detail` with the brief sentence separately — + // shipping the full reflowed text would duplicate that sentence at + // the top of every popup. + let mut out = String::new(); + if let Some(ext) = crate::doc::extended(p.doc_comments) { + out.push_str(&ext); + } + if let Some(sig) = p.signature { + if !sig.args.is_empty() { + if !out.is_empty() { + out.push_str("\n\n"); + } + for arg in &sig.args { + write!(out, "- `-{}`", arg.name).unwrap(); + if let Some(d) = crate::doc::brief(&arg.doc_comments) { + write!(out, " — {d}").unwrap(); + } + out.push('\n'); + } + } + } + (!out.is_empty()).then_some(out) +} + +fn arg_documentation(arg: &ProcArg) -> String { + // `extended` only — the brief sentence is handled by the caller's + // `detail` field; see `proc_documentation` for the rationale. + let mut out = String::new(); + if let Some(ext) = crate::doc::extended(&arg.doc_comments) { + out.push_str(&ext); + } + for attr in &arg.attributes { + if !out.is_empty() { + out.push('\n'); + } + write!(out, "- `{}`", render_attribute(attr)).unwrap(); + } + out +} + +/// Single-line label shown inline next to the flag in the completion +/// popup. We prepend any `@enum(...)` / `@default(...)` constraint — +/// rendered with its actual values rather than the bare attribute +/// name — so the user sees *what's allowed* without having to expand +/// the documentation pane. The doc-brief, if any, follows after a +/// dash. +fn flag_detail(arg: &ProcArg) -> Option { + let mut parts: Vec = Vec::new(); + for attr in &arg.attributes { + parts.push(render_attribute(attr)); + } + let brief = crate::doc::brief(&arg.doc_comments); + match (parts.is_empty(), brief) { + (true, b) => b, + (false, None) => Some(parts.join(" ")), + (false, Some(b)) => Some(format!("{} — {b}", parts.join(" "))), + } +} + +/// `@name(value, value, ...)` if the attribute carries values, +/// `@name` otherwise. Values are rendered via +/// [`AttributeValue::to_tcl_literal`] so strings get quoted and +/// integers/idents render as-is — same convention `proc_args` uses +/// when echoing back a defaulted call site. +fn render_attribute(attr: &Attribute) -> String { + if attr.values.is_empty() { + format!("@{}", attr.name) + } else { + let vals: Vec = attr + .values + .iter() + .map(AttributeValue::to_tcl_literal) + .collect(); + format!("@{}({})", attr.name, vals.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + /// Build `src` plus a cursor at the `|` marker, returning the + /// marker-free source and the byte offset of the cursor. + fn cursor(src_with_marker: &str) -> (String, u32) { + let offset = src_with_marker.find('|').expect("no cursor marker"); + let src = src_with_marker.replacen('|', "", 1); + (src, offset as u32) + } + + fn labels(src_with_marker: &str) -> Vec { + let (src, off) = cursor(src_with_marker); + let parsed = parse(&src); + complete_at(&parsed.document, &src, off) + .into_iter() + .map(|c| c.label) + .collect() + } + + #[test] + fn proc_names_in_command_position() { + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gr|\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["greet", "grumble"]); + } + + #[test] + fn proc_names_filtered_by_prefix() { + let src = "\ +proc greet {} { }\n\ +proc grumble {} { }\n\ +gree|\n"; + assert_eq!(labels(src), vec!["greet"]); + } + + #[test] + fn flags_in_argument_position() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["-depth", "-width"]); + } + + #[test] + fn flags_filtered_by_partial() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -w|\n"; + assert_eq!(labels(src), vec!["-width"]); + } + + #[test] + fn already_used_flag_is_not_reoffered() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -width 8 |\n"; + assert_eq!(labels(src), vec!["-depth"]); + } + + #[test] + fn completes_call_inside_proc_body() { + let src = "\ +proc helper {} { }\n\ +proc outer {} {\n hel|\n}\n"; + assert_eq!(labels(src), vec!["helper"]); + } + + #[test] + fn no_completion_inside_arg_decls() { + let src = "\ +proc greet {} { }\n\ +proc cfg {\n wi|\n} { }\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn enum_value_position_offers_choices() { + // `2.5_GT/s` etc. aren't valid `attribute_value_ident`s, so + // the IP generator quotes them in `@enum(…)` — the proc-args + // grammar parses them as strings. The completion labels come + // back bare here because no whitespace requires re-quoting. + let src = "\ +proc cfg {\n @enum(\"2.5_GT/s\", \"5.0_GT/s\", \"8.0_GT/s\") max_link_speed\n} { }\n\ +cfg -max_link_speed |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["2.5_GT/s", "5.0_GT/s", "8.0_GT/s"]); + } + + #[test] + fn enum_values_filter_by_partial() { + let src = "\ +proc cfg {\n @enum(\"2.5_GT/s\", \"5.0_GT/s\", \"8.0_GT/s\") max_link_speed\n} { }\n\ +cfg -max_link_speed 5|\n"; + assert_eq!(labels(src), vec!["5.0_GT/s"]); + } + + #[test] + fn enum_completion_kind_marks_items() { + let src = "\ +proc cfg {\n @enum(target, controller) kind\n} { }\n\ +cfg -kind |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + assert!(items.iter().all(|c| c.kind == CompletionKind::EnumValue)); + } + + #[test] + fn enum_value_with_spaces_gets_quoted() { + let src = "\ +proc cfg {\n @enum(\"Master Mode\", \"Slave Mode\") role\n} { }\n\ +cfg -role |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["\"Master Mode\"", "\"Slave Mode\""]); + } + + #[test] + fn flag_without_enum_offers_no_completions_at_value_position() { + // For a flag with no `@enum` the user is expected to type a + // free-form value. Popping a flag list there is wrong; it + // gets in the way of the actual value the user is typing. + let src = "\ +proc cfg {\n @default(0) width\n @default(0) depth\n} { }\n\ +cfg -width |\n"; + assert!(labels(src).is_empty(), "{:?}", labels(src)); + } + + #[test] + fn flag_completion_returns_after_value_is_typed() { + // After the value is typed, the cursor is between args again + // — show the next flags. + let src = "\ +proc cfg {\n @default(0) width\n @default(0) depth\n} { }\n\ +cfg -width 8 |\n"; + let mut got = labels(src); + got.sort(); + assert_eq!(got, vec!["-depth"]); + } + + #[test] + fn dash_partial_keeps_flag_completion() { + // Typing `-` after a complete flag should still mean "new + // flag," not "enum value." + let src = "\ +proc cfg {\n @enum(a, b) mode\n @default(0) width\n} { }\n\ +cfg -mode -|\n"; + let got = labels(src); + assert!(got.contains(&"-width".to_string()), "{got:?}"); + assert!(!got.contains(&"a".to_string()), "{got:?}"); + } + + #[test] + fn unknown_command_offers_no_flags() { + let src = "puts |\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn flag_completion_carries_doc_and_detail() { + // Multi-sentence doc: the brief sentence joins the attribute + // summary in `detail`, the rest goes in `documentation`. They + // must NOT overlap — an LSP client renders both, and a + // repeated leading sentence reads as a duplicate. + let src = "\ +proc cfg { + ## Bus width in bits. Must be a power of two. + @default(8) width +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-width").unwrap(); + assert_eq!(item.kind, CompletionKind::Flag); + assert_eq!( + item.detail.as_deref(), + Some("@default(8) — Bus width in bits.") + ); + let doc = item.documentation.as_deref().unwrap(); + assert!(doc.contains("Must be a power of two."), "{doc}"); + assert!( + !doc.contains("Bus width in bits."), + "documentation should not repeat the brief: {doc}" + ); + assert!(doc.contains("@default(8)"), "{doc}"); + } + + #[test] + fn flag_detail_renders_enum_alternatives_and_default() { + // The whole point: a user scanning the completion list sees + // exactly which values are allowed (`@enum(...)`) and what + // ships by default (`@default(...)`) without having to expand + // the doc pane. + let src = "\ +proc cfg { + @enum(LOW, HIGH, OPTIMIZED) @default(OPTIMIZED) bandwidth +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-bandwidth").unwrap(); + assert_eq!( + item.detail.as_deref(), + Some("@enum(LOW, HIGH, OPTIMIZED) @default(OPTIMIZED)") + ); + let doc = item.documentation.as_deref().unwrap(); + assert!(doc.contains("@enum(LOW, HIGH, OPTIMIZED)"), "{doc}"); + assert!(doc.contains("@default(OPTIMIZED)"), "{doc}"); + } + + #[test] + fn constructor_hint_offers_snippet_in_value_position() { + // A doc line of the form `Construct with [namespaced::path]` + // on the target arg turns into a snippet completion in value + // position — the label is the constructor path, the insert + // text is a `[\n path $0\n]` block with cursor placed + // after the constructor name. + let src = "\ +namespace eval demo {}\n\ +proc demo::child { -x: int } {}\n\ +proc parent {\n ## Construct with [demo::child].\n child: any\n} { }\n\ +parent -child |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items + .iter() + .find(|c| c.kind == CompletionKind::Constructor) + .unwrap_or_else(|| { + panic!( + "no constructor completion; got {:?}", + items + .iter() + .map(|c| (&c.label, c.kind)) + .collect::>() + ) + }); + assert_eq!(item.label, "demo::child"); + assert!(item.snippet); + let insert = item.insert_text.as_deref().unwrap(); + assert!(insert.contains("demo::child"), "insert={insert:?}"); + assert!(insert.contains("$0"), "insert={insert:?}"); + assert!(insert.starts_with('['), "insert={insert:?}"); + assert!(insert.trim_end().ends_with(']'), "insert={insert:?}"); + } + + #[test] + fn constructor_hint_absent_produces_no_extra_completion() { + // No `Construct with ...` doc → no constructor completion. + // Value position on a non-enum, non-hinted flag returns empty. + let src = "\ +proc parent {\n ## An arbitrary slot.\n child: any\n} { }\n\ +parent -child |\n"; + assert!(labels(src).is_empty()); + } + + #[test] + fn constructor_hint_case_insensitive() { + let src = "\ +proc parent {\n ## construct with [foo::bar]\n slot: any\n} { }\n\ +parent -slot |\n"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + assert!(items + .iter() + .any(|c| c.kind == CompletionKind::Constructor + && c.label == "foo::bar")); + } + + #[test] + fn flag_detail_quotes_string_enum_values() { + // Values that started life as a quoted string in the source + // must stay quoted in the detail line — `"Master Mode"` not + // `Master Mode` — so the displayed text is what the user + // would type back as the value. + let src = "\ +proc cfg { + @enum(\"Master Mode\", \"Slave Mode\") role +} { } +cfg | +"; + let (s, off) = cursor(src); + let parsed = parse(&s); + let items = complete_at(&parsed.document, &s, off); + let item = items.iter().find(|c| c.label == "-role").unwrap(); + assert_eq!( + item.detail.as_deref(), + Some("@enum(\"Master Mode\", \"Slave Mode\")") + ); + } +} diff --git a/vw-htcl/src/doc.rs b/vw-htcl/src/doc.rs new file mode 100644 index 0000000..4e7f749 --- /dev/null +++ b/vw-htcl/src/doc.rs @@ -0,0 +1,358 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Render helpers for doc-comment blocks. +//! +//! `##` doc comments in source are typically wrapped at a comfortable +//! editing column (~80 chars). When a display surface — an LSP hover, +//! signature help, completion documentation — joins those lines +//! verbatim, the source wrap survives into the rendered markdown. +//! Most LSP clients then treat the first wrapped fragment as a "brief +//! summary," which is almost always a mid-sentence truncation. +//! +//! [`reflow_doc_comments`] converts a slice of doc-comment lines into +//! markdown-clean text: consecutive non-empty lines collapse into one +//! paragraph (joined with a single space), and a blank line becomes a +//! paragraph break (`\n\n`). The first paragraph then reads as a +//! complete unit — usually one or more whole sentences — instead of +//! the editor-wrap fragment that surfaces today. + +/// One-line summary suitable for an inline annotation (LSP +/// `CompletionItem::detail`, a parameter list's `— brief` suffix, +/// etc.). Takes the first reflowed paragraph and trims to its first +/// sentence — the convention rustdoc, godoc, and most doc generators +/// follow for "short description vs full body." +/// +/// Returns `None` when `lines` has no non-blank content. Falls back +/// to the whole first paragraph when no sentence terminator (`.`, +/// `!`, `?` followed by whitespace or end-of-string) is found. +pub fn brief(lines: &[String]) -> Option { + let reflowed = reflow_doc_comments(lines); + if reflowed.is_empty() { + return None; + } + let first_paragraph = reflowed.split("\n\n").next().unwrap(); + let bytes = first_paragraph.as_bytes(); + for (i, &b) in bytes.iter().enumerate() { + if !matches!(b, b'.' | b'!' | b'?') { + continue; + } + let next = bytes.get(i + 1).copied(); + if next.is_none() || matches!(next, Some(b' ' | b'\t' | b'\n')) { + return Some(first_paragraph[..=i].to_string()); + } + } + Some(first_paragraph.to_string()) +} + +/// Extended description — everything **after** the first sentence, +/// reflowed into markdown. +/// +/// Pairs with [`brief`]: an LSP-facing renderer puts `brief` in +/// `CompletionItem::detail` (the inline summary next to the label) +/// and `extended` in `documentation` (the body popup). Splitting +/// this way avoids the duplication that occurs when both fields +/// start with the same sentence. +/// +/// Returns `None` when there is no content after the first sentence +/// — e.g. when the doc is a single-sentence summary with no body. +pub fn extended(lines: &[String]) -> Option { + let reflowed = reflow_doc_comments(lines); + if reflowed.is_empty() { + return None; + } + let bytes = reflowed.as_bytes(); + let mut split_at = None; + for (i, &b) in bytes.iter().enumerate() { + if !matches!(b, b'.' | b'!' | b'?') { + continue; + } + let next = bytes.get(i + 1).copied(); + if next.is_none() || matches!(next, Some(b' ' | b'\t' | b'\n')) { + split_at = Some(i + 1); + break; + } + } + // No sentence terminator means the whole reflow IS the brief — + // nothing to put in the body. + let after = reflowed[split_at?..].trim_start(); + (!after.is_empty()).then(|| after.to_string()) +} + +/// Word-wrap `text` into lines no wider than `width` chars. Used by +/// doc-comment generators that want source files with paragraphs +/// re-flowed to a comfortable editing width (the LSP reflows again +/// for display, but a wrapped source is easier for humans to read +/// and diff). +/// +/// A single word longer than `width` is left on a line by itself +/// rather than truncated. +pub fn wrap_paragraph(text: &str, width: usize) -> Vec { + let mut out: Vec = Vec::new(); + let mut current = String::new(); + for word in text.split_whitespace() { + if current.is_empty() { + current.push_str(word); + } else if current.len() + 1 + word.len() <= width { + current.push(' '); + current.push_str(word); + } else { + out.push(std::mem::take(&mut current)); + current.push_str(word); + } + } + if !current.is_empty() { + out.push(current); + } + out +} + +/// Reflow doc-comment lines into a markdown string. See module docs. +pub fn reflow_doc_comments(lines: &[String]) -> String { + let mut out = String::new(); + let mut paragraph = String::new(); + let flush = |paragraph: &mut String, out: &mut String| { + if paragraph.is_empty() { + return; + } + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(paragraph); + paragraph.clear(); + }; + for line in lines { + let trimmed = line.trim(); + if trimmed.is_empty() { + flush(&mut paragraph, &mut out); + } else { + if !paragraph.is_empty() { + paragraph.push(' '); + } + paragraph.push_str(&render_refs(trimmed)); + } + } + flush(&mut paragraph, &mut out); + out +} + +/// Rewrite `[NAME]` tokens as `` `NAME` `` so hover-popup markdown +/// renders them as inline code — visually distinct from prose, and +/// (in editors that honor code-span click handlers) discoverable as +/// something a reader can act on. The analyzer's goto/hover paths +/// already resolve the cursor to the same reference; this is the +/// display side of the same feature. +/// +/// Interior chars accepted: letters, digits, `_`, and `:` (for +/// namespace qualification). Anything else is left as-is — a +/// prose sentence like "see [1]" or "[TODO: refactor]" isn't a +/// reference. +fn render_refs(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'[' { + let content_start = i + 1; + // First char of a Tcl-style ident must be a letter or + // underscore; digits and `:`-prefixed forms don't count + // (they're prose or footnote-style refs, not identifiers). + if content_start < bytes.len() + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < bytes.len() && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < bytes.len() && bytes[j] == b']' && j > content_start { + out.push('`'); + out.push_str(&s[content_start..j]); + out.push('`'); + i = j + 1; + continue; + } + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(arr: [&str; N]) -> Vec { + arr.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn single_wrapped_paragraph_becomes_one_line() { + let out = reflow_doc_comments(&lines([ + "Create an external port in the current block design and connect that to the", + "selected block pin.", + ])); + assert_eq!( + out, + "Create an external port in the current block design and connect that to the selected block pin." + ); + } + + #[test] + fn blank_line_becomes_paragraph_break() { + let out = reflow_doc_comments(&lines([ + "Summary line one.", + "", + "Body line two,", + "wrapped.", + ])); + assert_eq!(out, "Summary line one.\n\nBody line two, wrapped."); + } + + #[test] + fn leading_and_trailing_blanks_are_dropped() { + let out = reflow_doc_comments(&lines(["", "Hello.", "", ""])); + assert_eq!(out, "Hello."); + } + + #[test] + fn empty_input_returns_empty_string() { + assert_eq!(reflow_doc_comments(&[]), ""); + } + + #[test] + fn brief_extracts_first_sentence_from_wrapped_lines() { + let out = brief(&lines([ + "Create an external port in the current block design and connect that to the", + "selected block pin. If a bd_cell is specified, all pins are made external.", + ])); + assert_eq!( + out.as_deref(), + Some( + "Create an external port in the current block design and connect that to the selected block pin." + ) + ); + } + + #[test] + fn brief_handles_single_sentence_proc() { + let out = brief(&lines(["Width of the data bus in bits."])); + assert_eq!(out.as_deref(), Some("Width of the data bus in bits.")); + } + + #[test] + fn brief_falls_back_to_paragraph_when_no_terminator() { + let out = brief(&lines(["just a phrase", "with no period"])); + assert_eq!(out.as_deref(), Some("just a phrase with no period")); + } + + #[test] + fn brief_returns_none_for_empty_input() { + assert!(brief(&[]).is_none()); + assert!(brief(&lines([""])).is_none()); + } + + #[test] + fn extended_skips_the_summary_sentence() { + let out = extended(&lines([ + "Summary. Body sentence in same paragraph.", + "", + "Second paragraph here.", + ])); + assert_eq!( + out.as_deref(), + Some("Body sentence in same paragraph.\n\nSecond paragraph here.") + ); + } + + #[test] + fn extended_returns_none_for_single_sentence_doc() { + assert!(extended(&lines(["Width of the data bus in bits."])).is_none()); + } + + #[test] + fn extended_brief_round_trip_covers_full_text() { + // Together, `brief` and `extended` should reproduce every + // visible character of the reflowed input (modulo a single + // separator between them). + let input = lines([ + "First sentence.", + "Continued first paragraph.", + "", + "Second paragraph.", + ]); + let b = brief(&input).unwrap(); + let e = extended(&input).unwrap(); + let full = reflow_doc_comments(&input); + // The recombined text should equal the reflow (with a space + // between b and e since the brief is part of paragraph 1). + assert!(full.starts_with(&b)); + assert!(full.ends_with(&e)); + } + + #[test] + fn brief_does_not_trip_on_decimal_or_versal_dots() { + // `3.4` shouldn't end the sentence — terminator must be + // followed by whitespace or end-of-string. + let out = + brief(&lines(["Source IP-XACT: xilinx.com:ip:versal_cips:3.4"])); + assert_eq!( + out.as_deref(), + Some("Source IP-XACT: xilinx.com:ip:versal_cips:3.4") + ); + } + + #[test] + fn wrap_paragraph_breaks_at_word_boundaries() { + let out = wrap_paragraph("one two three four five six", 12); + assert_eq!(out, vec!["one two", "three four", "five six"]); + } + + #[test] + fn wrap_paragraph_keeps_oversize_words_on_their_own_line() { + let out = wrap_paragraph("short superlongword end", 8); + assert_eq!(out, vec!["short", "superlongword", "end"]); + } + + #[test] + fn single_leading_space_is_trimmed_per_line() { + // `##` doc comments may include a leading space after the + // `##` marker that gets preserved in the parsed string; we + // trim each line so the leading space doesn't become a + // double-space inside the joined paragraph. + let out = reflow_doc_comments(&lines([" word one", " word two"])); + assert_eq!(out, "word one word two"); + } + + /// `[NAME]` refs in doc-comment text render as inline code + /// spans so hover popups distinguish them from prose. + #[test] + fn ref_tokens_render_as_inline_code() { + let out = reflow_doc_comments(&lines([ + "Construct with [dcmac::mac_port] before calling [dcmac::create].", + ])); + assert_eq!( + out, + "Construct with `dcmac::mac_port` before calling `dcmac::create`." + ); + } + + /// Not every `[…]` in prose is a reference. Only accept alnum + + /// `_` + `:` interiors; anything else stays as-is. + #[test] + fn non_ref_brackets_left_alone() { + let out = reflow_doc_comments(&lines([ + "See [1] and [TODO: refactor] for details.", + ])); + assert_eq!(out, "See [1] and [TODO: refactor] for details."); + } +} diff --git a/vw-htcl/src/emit.rs b/vw-htcl/src/emit.rs new file mode 100644 index 0000000..c21e264 --- /dev/null +++ b/vw-htcl/src/emit.rs @@ -0,0 +1,553 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Build and emit htcl source code. +//! +//! Distinct from [`crate::ast`], which is the parser's CST and carries +//! spans, doc comments as raw text, and a structure optimized for +//! analysis. `emit` is the dual: for code generation. No spans, +//! ergonomic constructors, and a [`Display`](std::fmt::Display) impl +//! that produces well-formed, indented htcl text. +//! +//! The model is small on purpose. The [`Word`] variants line up with +//! the parser's [`crate::ast::WordForm`] / [`crate::ast::WordPart`] +//! distinctions (bare / quoted / braced / `$var` / `[cmd]`), and +//! [`Word::lit`] picks the safest word form for a runtime string. The +//! [`ToHtcl`] trait is the interpolation interface used by `vw-quote`'s +//! `quote_htcl!` macro and by hand-written generators. + +use std::fmt; + +/// A complete htcl document being built. +#[derive(Clone, Debug, Default)] +pub struct Doc { + pub items: Vec, +} + +impl Doc { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, item: impl Into) -> &mut Self { + self.items.push(item.into()); + self + } + + pub fn cmd(&mut self, cmd: Command) -> &mut Self { + self.items.push(Item::Command(cmd)); + self + } + + pub fn comment(&mut self, text: impl Into) -> &mut Self { + self.items.push(Item::Comment(text.into())); + self + } + + pub fn doc(&mut self, text: impl Into) -> &mut Self { + self.items.push(Item::DocComment(text.into())); + self + } + + pub fn blank(&mut self) -> &mut Self { + self.items.push(Item::Blank); + self + } +} + +#[derive(Clone, Debug)] +pub enum Item { + Command(Command), + /// Regular `# ...` comment (one line, no leading `#`). + Comment(String), + /// Doc `## ...` comment (one line, no leading `##`). Doc comments + /// attached to a specific command live on [`Command::doc_comments`]. + DocComment(String), + /// Emit a blank line. + Blank, +} + +impl From for Item { + fn from(c: Command) -> Self { + Item::Command(c) + } +} + +/// A single htcl command (one logical line, possibly with a body +/// block). +#[derive(Clone, Debug, Default)] +pub struct Command { + /// `##` doc comments emitted immediately above the command. + pub doc_comments: Vec, + /// The command name and its arguments, in order. + pub words: Vec, + /// Optional braced body emitted as `{ … }` after the words, with + /// its contents indented. Used by `proc`, `if`, `while`, etc. + pub body: Option, +} + +impl Command { + /// `name arg1 arg2 …` with no body. Most generic command shape. + pub fn call(name: impl Into, args: I) -> Self + where + I: IntoIterator, + W: Into, + { + let mut words = vec![name.into()]; + words.extend(args.into_iter().map(Into::into)); + Self { + words, + ..Self::default() + } + } + + pub fn with_doc(mut self, doc: impl Into) -> Self { + self.doc_comments.push(doc.into()); + self + } + + pub fn with_body(mut self, body: Doc) -> Self { + self.body = Some(body); + self + } +} + +/// One word of an htcl command. +/// +/// The variants correspond to the parser's word forms. Prefer +/// [`Word::lit`] when you have a runtime string and want the safest +/// form chosen for you; the named constructors are for when you know +/// the form (e.g. you're producing a `$var` reference deliberately). +#[derive(Clone, Debug)] +pub enum Word { + /// A bare unquoted word. Caller is responsible for ensuring `s` + /// contains no whitespace or shell-special characters; prefer + /// [`Word::lit`] when in doubt. + Bare(String), + /// A double-quoted word (`"…"`). Tcl substitution applies inside; + /// the content is escaped during emit so embedded `"` and `\` + /// are safe. + Quoted(String), + /// A braced word (`{…}`). No substitution; embedded `{`/`}` are + /// the caller's responsibility (typically rare). + Braced(String), + /// A `$name` variable reference. + Var(String), + /// A `[ cmd ]` command substitution; `s` is the interior text, + /// emitted verbatim. + CmdSubst(String), + /// Pre-formatted text inserted as-is. Caller is responsible for + /// it being a valid single word. Useful when composing fragments + /// produced elsewhere. + Raw(String), +} + +impl Word { + /// Choose the smallest safe word form for `s`: bare when it + /// contains only word-safe ASCII characters, double-quoted with + /// escapes otherwise. Empty strings become `""`. + pub fn lit(s: impl Into) -> Word { + let s = s.into(); + if needs_quoting(&s) { + Word::Quoted(s) + } else { + Word::Bare(s) + } + } + + /// `$name` reference. The name is not validated. + pub fn var(name: impl Into) -> Word { + Word::Var(name.into()) + } +} + +fn needs_quoting(s: &str) -> bool { + if s.is_empty() { + return true; + } + s.chars().any(|c| { + c.is_whitespace() + || matches!(c, ';' | '"' | '\\' | '[' | ']' | '{' | '}' | '$' | '#') + }) +} + +impl From<&str> for Word { + fn from(s: &str) -> Self { + Word::lit(s) + } +} + +impl From for Word { + fn from(s: String) -> Self { + Word::lit(s) + } +} + +// --------------------------------------------------------------------------- +// ToHtcl — the interpolation interface for `quote_htcl!`. +// --------------------------------------------------------------------------- + +/// Produce a [`Word`] for interpolation into emitted htcl. +/// +/// Implemented for the common Rust value types. Pass any `T: ToHtcl` +/// to `#expr` slots in `quote_htcl!`; the macro calls +/// `(&expr).to_htcl()` to get the inserted word. +pub trait ToHtcl { + fn to_htcl(&self) -> Word; +} + +impl ToHtcl for Word { + fn to_htcl(&self) -> Word { + self.clone() + } +} +impl ToHtcl for str { + fn to_htcl(&self) -> Word { + Word::lit(self) + } +} +impl ToHtcl for String { + fn to_htcl(&self) -> Word { + Word::lit(self.clone()) + } +} +impl ToHtcl for &T { + fn to_htcl(&self) -> Word { + (*self).to_htcl() + } +} +impl ToHtcl for bool { + fn to_htcl(&self) -> Word { + Word::Bare(if *self { "1".into() } else { "0".into() }) + } +} + +macro_rules! impl_to_htcl_display { + ($($t:ty),* $(,)?) => { + $( + impl ToHtcl for $t { + fn to_htcl(&self) -> Word { + Word::Bare(self.to_string()) + } + } + )* + }; +} +impl_to_htcl_display!(i8, i16, i32, i64, i128, isize); +impl_to_htcl_display!(u8, u16, u32, u64, u128, usize); +impl_to_htcl_display!(f32, f64); + +// --------------------------------------------------------------------------- +// ToTcl — interpolation interface for `quote_tcl!`. +// --------------------------------------------------------------------------- + +/// Produce a [`Word`] for interpolation into emitted *pure Tcl*. +/// +/// Distinct from [`ToHtcl`] so that compiler intrinsics (the `repr` +/// codegen module, `kwargs` shim helpers, future ones) which emit +/// Tcl bodies — not htcl — can carry an independent vocabulary if +/// they grow it. For now the surface is intentionally identical: +/// the same Rust value types yield the same [`Word`] under both +/// traits. The split exists so future Tcl-only forms (typed +/// `Tcl_Obj` handle quoting, namespaced-proc-name formatting, +/// etc.) can land on `ToTcl` without changing `ToHtcl`'s contract. +pub trait ToTcl { + fn to_tcl(&self) -> Word; +} + +impl ToTcl for Word { + fn to_tcl(&self) -> Word { + self.clone() + } +} +impl ToTcl for str { + fn to_tcl(&self) -> Word { + Word::lit(self) + } +} +impl ToTcl for String { + fn to_tcl(&self) -> Word { + Word::lit(self.clone()) + } +} +impl ToTcl for &T { + fn to_tcl(&self) -> Word { + (*self).to_tcl() + } +} +impl ToTcl for bool { + fn to_tcl(&self) -> Word { + Word::Bare(if *self { "1".into() } else { "0".into() }) + } +} + +macro_rules! impl_to_tcl_display { + ($($t:ty),* $(,)?) => { + $( + impl ToTcl for $t { + fn to_tcl(&self) -> Word { + Word::Bare(self.to_string()) + } + } + )* + }; +} +impl_to_tcl_display!(i8, i16, i32, i64, i128, isize); +impl_to_tcl_display!(u8, u16, u32, u64, u128, usize); +impl_to_tcl_display!(f32, f64); + +// --------------------------------------------------------------------------- +// Emit — Display impls produce well-formed htcl text. +// --------------------------------------------------------------------------- + +impl fmt::Display for Doc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_doc(f, self, 0) + } +} + +impl fmt::Display for Item { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_item(f, self, 0) + } +} + +impl fmt::Display for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_command(f, self, 0) + } +} + +impl fmt::Display for Word { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + emit_word(f, self) + } +} + +const INDENT: &str = " "; + +fn emit_indent(f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { + for _ in 0..level { + f.write_str(INDENT)?; + } + Ok(()) +} + +fn emit_doc( + f: &mut fmt::Formatter<'_>, + doc: &Doc, + level: usize, +) -> fmt::Result { + for item in &doc.items { + emit_item(f, item, level)?; + } + Ok(()) +} + +fn emit_item( + f: &mut fmt::Formatter<'_>, + item: &Item, + level: usize, +) -> fmt::Result { + match item { + Item::Command(c) => emit_command(f, c, level), + Item::Comment(text) => { + emit_indent(f, level)?; + writeln!(f, "# {text}") + } + Item::DocComment(text) => { + emit_indent(f, level)?; + writeln!(f, "## {text}") + } + Item::Blank => writeln!(f), + } +} + +fn emit_command( + f: &mut fmt::Formatter<'_>, + cmd: &Command, + level: usize, +) -> fmt::Result { + for doc in &cmd.doc_comments { + emit_indent(f, level)?; + writeln!(f, "## {doc}")?; + } + emit_indent(f, level)?; + let mut first = true; + for w in &cmd.words { + if !first { + f.write_str(" ")?; + } + emit_word(f, w)?; + first = false; + } + if let Some(body) = &cmd.body { + if body.items.is_empty() { + f.write_str(" {}\n")?; + } else { + f.write_str(" {\n")?; + emit_doc(f, body, level + 1)?; + emit_indent(f, level)?; + f.write_str("}\n")?; + } + } else { + f.write_str("\n")?; + } + Ok(()) +} + +fn emit_word(f: &mut fmt::Formatter<'_>, w: &Word) -> fmt::Result { + match w { + Word::Bare(s) => f.write_str(s), + Word::Quoted(s) => { + f.write_str("\"")?; + for c in s.chars() { + match c { + '\\' => f.write_str("\\\\")?, + '"' => f.write_str("\\\"")?, + '$' => f.write_str("\\$")?, + '[' => f.write_str("\\[")?, + ']' => f.write_str("\\]")?, + other => f.write_fmt(format_args!("{other}"))?, + } + } + f.write_str("\"") + } + Word::Braced(s) => { + f.write_str("{")?; + f.write_str(s)?; + f.write_str("}") + } + Word::Var(name) => { + f.write_str("$")?; + f.write_str(name) + } + Word::CmdSubst(s) => { + f.write_str("[")?; + f.write_str(s)?; + f.write_str("]") + } + Word::Raw(s) => f.write_str(s), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn word_lit_picks_bare_when_safe() { + assert!( + matches!(Word::lit("hello"), Word::Bare(ref s) if s == "hello") + ); + assert!(matches!(Word::lit("32.0_GT/s"), Word::Bare(_))); + } + + #[test] + fn word_lit_quotes_when_special() { + let cases = ["with space", "has\"quote", "has$dollar", "has;semi", ""]; + for c in cases { + assert!( + matches!(Word::lit(c), Word::Quoted(_)), + "expected quoted for {c:?}" + ); + } + } + + #[test] + fn emit_command_simple() { + let cmd = Command::call("puts", ["hi"]); + assert_eq!(format!("{cmd}"), "puts hi\n"); + } + + #[test] + fn emit_command_quotes_when_needed() { + let cmd = Command::call("puts", ["hello world"]); + assert_eq!(format!("{cmd}"), "puts \"hello world\"\n"); + } + + #[test] + fn emit_doc_full_proc() { + // proc greet {name} { puts "hi $name" } + let inner = Command::call("puts", [Word::Quoted("hi $name".into())]); + let body = { + let mut d = Doc::new(); + d.cmd(inner); + d + }; + let proc = Command { + doc_comments: vec!["Say hi.".into()], + words: vec![ + Word::Bare("proc".into()), + Word::Bare("greet".into()), + Word::Braced("name".into()), + ], + body: Some(body), + }; + let mut doc = Doc::new(); + doc.cmd(proc); + let out = format!("{doc}"); + let expected = "\ +## Say hi. +proc greet {name} { + puts \"hi \\$name\" +} +"; + assert_eq!(out, expected); + } + + #[test] + fn empty_body_emits_braces() { + let cmd = Command { + words: vec![ + Word::Bare("proc".into()), + Word::Bare("f".into()), + Word::Braced("".into()), + ], + body: Some(Doc::new()), + ..Default::default() + }; + assert_eq!(format!("{cmd}"), "proc f {} {}\n"); + } + + #[test] + fn to_htcl_basic_types() { + assert!(matches!("hi".to_htcl(), Word::Bare(ref s) if s == "hi")); + assert!(matches!(42i64.to_htcl(), Word::Bare(ref s) if s == "42")); + assert!(matches!(true.to_htcl(), Word::Bare(ref s) if s == "1")); + } + + #[test] + fn emitted_output_round_trips_through_parser() { + // Build a doc, emit it, re-parse, and check we get a structurally + // similar document — proves the emitter is producing well-formed + // htcl that the parser accepts. + use crate::parser::parse; + let mut body = Doc::new(); + body.cmd(Command::call("puts", [Word::Quoted("hi $name".into())])); + let proc = Command { + words: vec![ + Word::Bare("proc".into()), + Word::Bare("greet".into()), + Word::Braced("name".into()), + ], + body: Some(body), + ..Default::default() + }; + let mut doc = Doc::new(); + doc.cmd(proc); + let text = doc.to_string(); + let parsed = parse(&text); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + // First (and only) statement should be the proc. + let stmt = &parsed.document.stmts[0]; + let crate::ast::Stmt::Command(cmd) = stmt else { + panic!("expected command, got {stmt:?}"); + }; + let crate::ast::CommandKind::Proc(p) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(p.name.as_deref(), Some("greet")); + } +} diff --git a/vw-htcl/src/enum_parse.rs b/vw-htcl/src/enum_parse.rs new file mode 100644 index 0000000..5938c04 --- /dev/null +++ b/vw-htcl/src/enum_parse.rs @@ -0,0 +1,318 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Mini-parser for the variants block of an `enum NAME = { ... }` +//! declaration. +//! +//! Grammar (operates on the text INSIDE the body braces, not +//! including the braces themselves): +//! +//! ```text +//! Variants ::= Sep* (Variant Sep+ Variant)* Sep* +//! Variant ::= Ident (':' Type)? +//! Sep ::= '\n' | comment | doc_comment | whitespace +//! ``` +//! +//! Variants are newline-separated (mirroring `proc {a; b}` arg-list +//! style); blank lines and `##` doc comments are ignored. The payload +//! type, when present, is parsed via [`crate::type_parse`] verbatim +//! — so anything that grammar accepts (primitives, newtypes, +//! generics, qualified) is legal here too. A future tightening could +//! reject `Qualified` payloads as nonsensical; v1 keeps it permissive +//! and lets the validator decide. + +use crate::ast::EnumVariant; +use crate::span::Span; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnumParseError { + pub message: String, + pub span: Span, +} + +/// Parse the body of an enum declaration. `text` is the contents +/// INSIDE the body braces (not including the braces). `base_offset` +/// is the absolute byte position of `text[0]` in the original +/// source so returned spans land in the right place. +pub fn parse( + text: &str, + base_offset: u32, +) -> Result, EnumParseError> { + let mut p = Parser::new(text, base_offset); + let mut variants = Vec::new(); + loop { + p.skip_separators(); + if p.eof() { + break; + } + variants.push(p.parse_variant()?); + } + Ok(variants) +} + +struct Parser<'a> { + text: &'a str, + bytes: &'a [u8], + pos: usize, + base: u32, +} + +impl<'a> Parser<'a> { + fn new(text: &'a str, base: u32) -> Self { + Self { + text, + bytes: text.as_bytes(), + pos: 0, + base, + } + } + + fn eof(&self) -> bool { + self.pos >= self.bytes.len() + } + + fn here(&self) -> u32 { + self.base + self.pos as u32 + } + + fn here_span(&self) -> Span { + let h = self.here(); + Span::new(h, h) + } + + fn span_from(&self, start: usize) -> Span { + Span::new(self.base + start as u32, self.base + self.pos as u32) + } + + fn skip_horizontal_ws(&mut self) { + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c == b' ' || c == b'\t' || c == b'\r' { + self.pos += 1; + } else { + break; + } + } + } + + /// Skip newlines, whitespace, regular `#` comments, and `##` + /// doc comments. Variants are separated by at least one newline + /// (or are at the start of the body). + fn skip_separators(&mut self) { + loop { + // Any whitespace, including newlines. + while self.pos < self.bytes.len() + && self.bytes[self.pos].is_ascii_whitespace() + { + self.pos += 1; + } + if self.eof() { + break; + } + // Comment line — consume to next newline. `##` doc + // comments are dropped here; if a future revision needs + // to attach docs to variants, this is the spot. + if self.bytes[self.pos] == b'#' { + while self.pos < self.bytes.len() + && self.bytes[self.pos] != b'\n' + { + self.pos += 1; + } + continue; + } + break; + } + } + + /// Variant := IDENT (':' TYPE)? + fn parse_variant(&mut self) -> Result { + let start = self.pos; + let (name, name_span) = self.parse_ident()?; + self.skip_horizontal_ws(); + let payload_pos = self.pos; + let (payload, payload_span) = if self.pos < self.bytes.len() + && self.bytes[self.pos] == b':' + { + self.pos += 1; // ':' + self.skip_horizontal_ws(); + let type_start = self.pos; + // Consume up to end-of-line or end-of-input. The + // type-text-extraction window stops at newline so a + // bad payload doesn't bleed into the next variant. + while self.pos < self.bytes.len() && self.bytes[self.pos] != b'\n' { + self.pos += 1; + } + // Trim trailing horizontal whitespace from the + // payload text so spans are tight. + let mut end = self.pos; + while end > type_start + && matches!(self.bytes[end - 1], b' ' | b'\t' | b'\r') + { + end -= 1; + } + let payload_text = &self.text[type_start..end]; + let span = Span::new( + self.base + type_start as u32, + self.base + end as u32, + ); + let ty = crate::type_parse::parse( + payload_text, + self.base + type_start as u32, + ) + .map_err(|e| EnumParseError { + message: e.message, + span: e.span, + })?; + (Some(ty), span) + } else { + // Empty-payload variant. Span is a zero-width point + // right after the name. + let here = Span::new( + self.base + payload_pos as u32, + self.base + payload_pos as u32, + ); + (None, here) + }; + Ok(EnumVariant { + name, + name_span, + payload, + payload_span, + span: self.span_from(start), + }) + } + + fn parse_ident(&mut self) -> Result<(String, Span), EnumParseError> { + self.skip_horizontal_ws(); + let start = self.pos; + if self.eof() { + return Err(EnumParseError { + message: "expected variant name, found end of body".into(), + span: self.here_span(), + }); + } + let first = self.bytes[self.pos]; + if !(first.is_ascii_alphabetic() || first == b'_') { + return Err(EnumParseError { + message: format!( + "expected variant name, found `{}`", + first as char + ), + span: self.here_span(), + }); + } + self.pos += 1; + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + let name = self.text[start..self.pos].to_string(); + Ok((name, self.span_from(start))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::TypeExpr; + + fn p(s: &str) -> Vec { + parse(s, 0).unwrap_or_else(|e| panic!("parse failed: {e:?}")) + } + + #[test] + fn empty_body() { + let v = p(""); + assert!(v.is_empty()); + let v = p(" \n\n "); + assert!(v.is_empty()); + } + + #[test] + fn single_variant_with_payload() { + let v = p("Scalar: string"); + assert_eq!(v.len(), 1); + assert_eq!(v[0].name, "Scalar"); + let ty = v[0].payload.as_ref().unwrap(); + assert_eq!(ty.name(), "string"); + } + + #[test] + fn single_empty_payload_variant() { + let v = p("North"); + assert_eq!(v.len(), 1); + assert_eq!(v[0].name, "North"); + assert!(v[0].payload.is_none()); + } + + #[test] + fn mixed_payload_and_empty() { + let v = p("\n North\n South: int\n East\n West\n"); + assert_eq!(v.len(), 4); + assert_eq!(v[0].name, "North"); + assert!(v[0].payload.is_none()); + assert_eq!(v[1].name, "South"); + assert_eq!(v[1].payload.as_ref().unwrap().name(), "int"); + assert_eq!(v[2].name, "East"); + assert!(v[2].payload.is_none()); + assert_eq!(v[3].name, "West"); + assert!(v[3].payload.is_none()); + } + + #[test] + fn generic_payload() { + let v = p("\n Scalar: string\n Nested: dict\n"); + assert_eq!(v.len(), 2); + let TypeExpr::Generic { name, args, .. } = + v[1].payload.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "Property"); + } + + #[test] + fn comments_skipped_between_variants() { + let v = p( + "\n# leading comment\n## doc comment\nScalar: string\n# trailing\nNested: int\n", + ); + assert_eq!(v.len(), 2); + assert_eq!(v[0].name, "Scalar"); + assert_eq!(v[1].name, "Nested"); + } + + #[test] + fn err_invalid_first_char() { + let e = parse("123Foo: int", 0).unwrap_err(); + assert!(e.message.contains("variant name")); + } + + #[test] + fn err_bad_payload_type() { + let e = parse("Scalar: Option { + let table = signature_table(document); + // Try a doc-comment `[NAME]` reference first — it's cheap and + // rules out any structural resolution when the cursor is inside + // a `##` block. Otherwise fall through to the structural paths. + if let Some(span) = + definition_in_doc_comment(document, source, offset, &table) + { + return Some(span); + } + definition_in_stmts(&document.stmts, None, document, &table, source, offset) + // Fallback: a `$var` the structured tree keeps opaque — inside + // a command substitution or an `if`/`while` condition. Found by + // scanning the source and resolving against the enclosing + // proc's scope. + .or_else(|| definition_of_scanned_var(document, source, offset)) + // Fallback: cursor on a type-name annotation (arg type, + // return type, `type … = TYPE` underlying, generic arg). + .or_else(|| definition_of_type(document, offset)) +} + +/// Cursor on a type name in a proc signature or a `type` decl's +/// underlying → return the matching type declaration's name span. +/// Handles qualified names (`dcmac::MacPortProps`) via the parser's +/// `Qualified` variant plus the type-table lookup helper. +fn definition_of_type(document: &Document, offset: u32) -> Option { + let ty = type_expr_at(document, offset)?; + let name = type_expr_lookup_name(ty); + let decl = find_type_decl(document, &name)?; + Some(decl.name_span) +} + +/// Resolve a `[NAME]` reference embedded in a `##` doc-comment +/// block. Returns `None` when the cursor isn't inside any command's +/// or arg's doc-comment span, or when the `[…]` at the cursor +/// doesn't name a proc declared in this document. +/// +/// Cross-file references (e.g. `[Properties::from]` where +/// `Properties::from` lives in `types.htcl`) resolve when the +/// document has already sourced that file; unresolved names simply +/// return `None`, which the LSP client renders as "no definition +/// available" without disrupting the fallback paths. +fn definition_in_doc_comment( + document: &Document, + source: &str, + offset: u32, + _table: &SignatureTable<'_>, +) -> Option { + let block = enclosing_doc_block(&document.stmts, offset)?; + let name = extract_ref_at(source, block, offset)?; + let proc = find_proc_decl(document, &name)?; + Some(proc.name_span) +} + +/// Return the doc-comment span that contains `offset`, if any. Walks +/// commands, proc signatures, nested proc bodies, and namespace-eval +/// bodies. +fn enclosing_doc_block(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let Some(span) = cmd.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + // ProcArg doc-comment blocks sit inside the proc's args span. + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(span) = arg.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + } + } + if let Some(span) = enclosing_doc_block(&proc.body, offset) { + return Some(span); + } + } + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = enclosing_doc_block(&ns.body, offset) { + return Some(span); + } + } + } + None +} + +/// Given a doc-comment block-span and a cursor offset inside it, +/// find a `[NAME]` reference whose interior span contains `offset` +/// and return `NAME`. Interior must be a valid ident (letters, +/// digits, `_`, `:` for namespace qualification). `[` and `]` are +/// the disambiguators from surrounding prose. +fn extract_ref_at(source: &str, block: Span, offset: u32) -> Option { + let bytes = source.as_bytes(); + let start = block.start as usize; + let end = (block.end as usize).min(bytes.len()); + let off = offset as usize; + if off < start || off > end { + return None; + } + // Scan for `[…]` pairs. `[` and `]` are cheap to find; the + // interior is validated once we have both delimiters. + let mut i = start; + while i < end { + if bytes[i] == b'[' { + let content_start = i + 1; + // Ident-start rule: first char must be a letter or `_`. + // Same as `render_refs` in doc.rs — keeps the analyzer + // and the renderer in agreement on what counts as a ref. + if content_start < end + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < end && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < end && bytes[j] == b']' && j > content_start { + // Reference spans `[NAME]` inclusive. The cursor + // hits a ref if it's on any of `[`, `NAME`, or `]`. + if off >= i && off <= j { + let name = + std::str::from_utf8(&bytes[content_start..j]) + .ok()? + .to_string(); + return Some(name); + } + i = j + 1; + continue; + } + } + } + i += 1; + } + None +} + +fn definition_of_scanned_var( + document: &Document, + source: &str, + offset: u32, +) -> Option { + let (name, _) = scan_var_ref(source, offset)?; + let (stmts, enclosing) = innermost_scope(document, offset); + resolve_var_def(&name, stmts, enclosing, offset).map(|d| d.def_span()) +} + +/// Resolve the definition at `offset` within `stmts`, descending into +/// proc bodies. `enclosing` is the proc whose body `stmts` belongs to +/// (`None` at the top level), used to resolve variables to parameters. +/// `document` is the whole document so call sites — at any nesting +/// depth — can find their declaring proc, which always lives at the +/// top level. +fn definition_in_stmts<'a>( + stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + + // Inside a proc declaration, attribute ident values can + // reference sibling args by name. Resolve those to the arg's + // declaration site. + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(span) = definition_in_proc_decl(proc, offset) { + return Some(span); + } + // Cursor on the proc's own name — "goto def" of the def + // itself is the same span. Not super useful but + // consistent. + if proc.name_span.contains(offset) { + return Some(proc.name_span); + } + // Otherwise the cursor is somewhere in the body: recurse, + // making this proc the enclosing scope. + return definition_in_stmts( + &proc.body, + Some(proc), + document, + table, + source, + offset, + ); + } + + // `namespace eval { … }` — descend into the populated + // body directly. Without this arm we'd fall through to the + // brace-body reparse path, which works but discards the + // pre-populated proc bodies and re-triggers a full parse. + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = definition_in_stmts( + &ns.body, enclosing, document, table, source, offset, + ) { + return Some(span); + } + } + + // Cursor on a `$var` reference → its definition in scope. + if let Some(span) = definition_of_var(cmd, stmts, enclosing, offset) { + return Some(span); + } + + // Generic call site. Two flavors: + // 1. Cursor on the call name → proc declaration. + // 2. Cursor on a `-flag` arg → that arg's decl in the proc. + if let Some(span) = definition_in_call(cmd, document, table, offset) { + return Some(span); + } + + // Cursor inside a `[ … ]` command substitution → recurse into + // its parsed body so goto works on calls written inline. + if let Some(span) = + definition_in_cmd_substs(cmd, document, table, source, offset) + { + return Some(span); + } + + // Cursor inside a `{ … }` control-flow body (the second word + // of `if`, the third of `while`, the body of `foreach`, + // etc.). The parser leaves those as opaque `Braced` words — + // they're semantically Tcl scripts, but there's no + // eager-parse pass like there is for `[ … ]`. So we reparse + // on demand when goto lands the cursor inside one. + if let Some(span) = + definition_in_braced_bodies(cmd, document, table, source, offset) + { + return Some(span); + } + } + None +} + +/// When the cursor sits inside a `{ … }` word of a control-flow +/// command, reparse that word's interior as a htcl fragment, shift +/// all spans back into whole-source coordinates, and recurse into +/// [`definition_in_stmts`]. Without this, `if { … … }`, +/// `while { … … }`, `foreach x $xs { … … }`, and +/// friends never resolve their body calls — goto-def returns +/// "no definition found" from inside every generated +/// `if {[llength …]} { }` scaffold. +/// +/// Only `Braced` words are considered; the enclosing command's +/// head has to be one of a small list of body-hosts so we don't +/// waste a parse on `list {a b c}`-style data braces (their content +/// is a Tcl list, not a script). +fn definition_in_braced_bodies<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + let head = cmd.words.first().and_then(|w| w.as_text())?; + if !is_body_host(head) { + return None; + } + // The interior text lives in the source we're parsing against + // — we don't have that here directly, but we do have span + // access to the whole-source Command tree. Each `Braced` word's + // interior is `span.start+1 .. span.end-1` in the outer + // source. We recover it via the WordPart::Text that the parser + // populates for braced words. + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + // First-part Text carries the interior string with a span + // starting at word.span.start + 1 (the byte after `{`). + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + // Reparse the body as a bracket-body-mode fragment so + // newlines are whitespace and multi-line control flow + // parses cleanly. + // Reparse the brace-body against `source` — the WHOLE + // outer document. That lets us pass `source` back into + // `populate_procs` below so nested `[ … ]` CmdSubst + // bodies get filled in against the same coordinate space + // the outer AST uses. Without the populate pass, a call + // like `set cell [vivado_cmd::create_bd_cell …]` inside + // `if {$bd} { … }` still has an empty CmdSubst.body and + // the recursion bottoms out at `set`. + // + // Order matters: shift first (spans become absolute), then + // populate — the shifted CmdSubst.span carries the + // absolute offset `populate_cmd_subst_parts` needs to shift + // its reparsed interior into place. + // Braced bodies are Tcl scripts — `\n` terminates a + // command. Sibling comment in `hover_in_braced_bodies` + // (hover.rs) has the full story; the historical + // `Mode::BracketBody` glued all lines into one command + // and made every non-head call invisible to goto. + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::Toplevel, + ); + let delta = text_span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs(&mut stmts, source, &mut errs); + // Recurse into the reparsed stmts. Passing `None` as the + // enclosing proc mirrors what `definition_in_cmd_substs` + // does — variable resolution across a control-flow body + // is out of scope for this fix; the call-site → proc-decl + // path is what matters. + return definition_in_stmts( + &stmts, None, document, table, source, offset, + ); + } + None +} + +/// Command names whose brace-args hold Tcl scripts rather than +/// data. Restricting the lookup to these keeps us from waking the +/// parser on data braces (`list {a b c}`, `dict {k v}`, etc.). +fn is_body_host(head: &str) -> bool { + matches!( + head, + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + | "try" + | "finally" + | "eval" + | "uplevel" + | "namespace" + | "on" + | "apply" + // `dict for` — same rationale as `hover::is_body_host`. + | "dict" + ) +} + +fn definition_in_cmd_substs<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option { + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let crate::ast::WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return definition_in_stmts( + body, None, document, table, source, offset, + ); + } + } + } + } + None +} + +/// If the cursor is on a `$var` reference (a real [`WordPart::VarRef`]) +/// in `cmd`, resolve it to its definition within `scope_stmts` or a +/// parameter of `enclosing`. +fn definition_of_var<'a>( + cmd: &'a Command, + scope_stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + offset: u32, +) -> Option { + let name = var_ref_at(cmd, offset)?; + resolve_var_def(name, scope_stmts, enclosing, offset).map(|d| d.def_span()) +} + +/// The name of the `$var` reference under the cursor, if any. Walks +/// word parts so it also fires inside quoted words (`"hi $name"`) and +/// array syntax (`$arr($idx)`). +fn var_ref_at(cmd: &Command, offset: u32) -> Option<&str> { + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let WordPart::VarRef { name, span, .. } = part { + if span.contains(offset) { + return Some(name.as_str()); + } + } + } + } + None +} + +fn definition_in_call<'a>( + cmd: &'a Command, + document: &'a Document, + table: &SignatureTable<'a>, + offset: u32, +) -> Option { + let first = cmd.words.first()?; + let name = first.as_text()?; + + // Cursor on the call name. + if first.span.contains(offset) { + let proc = find_proc_decl(document, name)?; + return Some(proc.name_span); + } + + // Cursor on one of the `-flag` words. Look the flag up in the + // called proc's signature and return that arg's name_span. + let sig = *table.get(name)?; + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(arg.name_span); + } + + None +} + +/// Find the `proc` declaration that registers under `name` in the +/// document's signature table. Walks `namespace eval` bodies +/// recursively so a call to `project::set_target_language` resolves +/// to the inner `proc set_target_language` inside +/// `namespace eval project { … }`. +fn find_proc_decl<'a>(document: &'a Document, name: &str) -> Option<&'a Proc> { + find_proc_decl_in(&document.stmts, "", name) +} + +fn find_proc_decl_in<'a>( + stmts: &'a [Stmt], + prefix: &str, + name: &str, +) -> Option<&'a Proc> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(decl_name) = proc.name.as_deref() else { + continue; + }; + let qualified = if prefix.is_empty() { + decl_name.to_string() + } else { + format!("{prefix}::{decl_name}") + }; + if qualified == name { + return Some(proc); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(ns_name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + ns_name.to_string() + } else { + format!("{prefix}::{ns_name}") + }; + if let Some(found) = find_proc_decl_in(&ns.body, &nested, name) + { + return Some(found); + } + } + _ => {} + } + } + None +} + +fn definition_in_proc_decl(proc: &Proc, offset: u32) -> Option { + let sig = proc.signature.as_ref()?; + for arg in &sig.args { + for attr in &arg.attributes { + for value in &attr.values { + let AttributeValue::Ident { value: name, span } = value else { + continue; + }; + if !span.contains(offset) { + continue; + } + if let Some(target) = find_sibling_arg(sig, name) { + return Some(target.name_span); + } + // Ident value naming an unknown arg — no definition. + return None; + } + } + } + None +} + +fn find_sibling_arg<'a>( + sig: &'a ProcSignature, + name: &str, +) -> Option<&'a ProcArg> { + sig.args.iter().find(|a| a.name == name) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn first(src: &str, needle: &str) -> u32 { + src.find(needle).expect("needle not found") as u32 + } + + fn nth(src: &str, needle: &str, n: usize) -> u32 { + let mut start = 0; + for i in 0..=n { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found enough times"); + if i == n { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + #[test] + fn call_to_proc_decl() { + let src = "\ +proc greet {\n name\n} { puts hi }\n\ +greet -name there\n"; + let parsed = parse(src); + // Cursor on the `g` of the call-site `greet`. + let pos = first(src, "greet -"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Should point at the `greet` in the proc declaration (first + // occurrence after `proc `). + let decl_span = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("greet") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, decl_span); + } + + #[test] + fn attribute_ident_to_sibling_arg() { + let src = "\ +proc f {\n has_a\n @requires(has_a) has_b\n} { }\n"; + let parsed = parse(src); + // Cursor on `has_a` inside `@requires(has_a)`. + // First occurrence is the declaration; second is the + // attribute argument. + let pos = nth(src, "has_a", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + let decl_pos = first(src, "has_a"); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_to_unknown_proc_returns_none() { + let src = "puts hello\n"; + let parsed = parse(src); + assert!( + definition_at(&parsed.document, src, first(src, "puts")).is_none() + ); + } + + #[test] + fn attribute_ident_to_unknown_arg_returns_none() { + let src = "proc f {\n @requires(typo) only\n} { }\n"; + let parsed = parse(src); + let pos = first(src, "typo"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn call_flag_to_arg_decl() { + let src = "\ +proc show {\n flag_a\n width\n} { }\n\ +show -width 16\n"; + let parsed = parse(src); + // Cursor on `-width` at the call site. + let pos = first(src, "-width"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Decl `width` arg name is the second `width` in the source. + let decl_pos = nth(src, "width", 0); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_to_namespaced_proc_resolves_to_inner_decl() { + // `project::set_target_language` at the call site should + // resolve to `proc set_target_language` declared inside the + // matching `namespace eval project { ... }` block. + let src = "\ +namespace eval project { + proc set_target_language { + proj + language + } { } +} +project::set_target_language -proj p -language VHDL +"; + let parsed = parse(src); + let pos = first(src, "project::set_target_language"); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // The decl's name span covers just `set_target_language` + // (without the namespace prefix), which appears as the + // first occurrence of that bare token in the source. + let decl_pos = first(src, "set_target_language"); + assert_eq!(target.start, decl_pos); + } + + #[test] + fn call_inside_proc_body_to_proc_decl() { + // Mirrors interface.htcl: a call to a top-level proc from + // inside another proc's body. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis {\n width\n} {\n if_tport\n}\n"; + let parsed = parse(src); + // Cursor on the `if_tport` call inside `axis`'s body — the + // second occurrence of `if_tport`. + let pos = nth(src, "if_tport", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Resolves to the `if_tport` name in the declaration (first + // occurrence). + assert_eq!(target.start, first(src, "if_tport")); + } + + #[test] + fn var_ref_to_set_in_same_body() { + // Mirrors interface.htcl: `$mode` resolves to `set mode ...`. + let src = "\ +proc axis_if {\n kind\n} {\n\ + set mode hello\n\ + use_it $mode\n}\n"; + let parsed = parse(src); + let pos = first(src, "$mode") + 1; // on the `m` of `$mode` + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Should point at the `mode` in `set mode`. + assert_eq!(target.start, first(src, "mode")); + } + + #[test] + fn var_ref_to_proc_parameter() { + // `$name` has no `set`, so it resolves to the proc parameter. + let src = "\ +proc axis_if {\n kind\n name\n} {\n\ + use_it $name\n}\n"; + let parsed = parse(src); + let pos = first(src, "$name") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Parameter `name` decl is the first occurrence of `name`. + assert_eq!(target.start, first(src, "name")); + } + + #[test] + fn var_ref_to_variable_declaration() { + let src = "\ +proc p {} {\n\ + variable vlnv\n\ + use_it $vlnv\n}\n"; + let parsed = parse(src); + let pos = first(src, "$vlnv") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "vlnv")); + } + + #[test] + fn var_ref_inside_dict_for_body_goes_to_kv_binder() { + let src = "\ +set deps [some_proc] +dict for {lib srcs} $deps { + puts $lib +} +"; + let parsed = parse(src); + let pos = first(src, "$lib\n") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + // Target should be the `{lib srcs}` braced word. + assert_eq!(target.start, first(src, "{lib srcs}")); + } + + #[test] + fn var_ref_inside_foreach_body_goes_to_binder() { + let src = "foreach x {a b c} {\n puts $x\n}\n"; + let parsed = parse(src); + let pos = first(src, "$x\n") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "x {a")); + } + + #[test] + fn unknown_var_ref_returns_none() { + let src = "proc p {} {\n use_it $nope\n}\n"; + let parsed = parse(src); + let pos = first(src, "$nope") + 1; + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn var_ref_inside_opaque_condition_resolves_to_param() { + // `$kind` lives inside an `if` condition that sits inside a + // command substitution — both opaque to the structured tree. + // The source-scan fallback still resolves it to the parameter. + let src = "\ +proc axis_if {\n kind\n} {\n\ + set mode [\n\ + if {$kind == controller} { Master }\n\ + ]\n}\n"; + let parsed = parse(src); + let pos = nth(src, "$kind", 0) + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "kind")); + } + + #[test] + fn call_inside_command_substitution_resolves_to_decl() { + let src = "\ +proc create_cpm5 {\n name\n} { puts hi }\n\ +set cell [create_cpm5 -name x]\n"; + let parsed = parse(src); + // The second occurrence of `create_cpm5` (the call inside `[…]`). + let pos = nth(src, "create_cpm5", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + assert_eq!(target.start, first(src, "create_cpm5")); + } + + #[test] + fn call_flag_to_unknown_arg_returns_none() { + let src = "\ +proc show {\n width\n} { }\n\ +show -widthz 16\n"; + let parsed = parse(src); + let pos = first(src, "-widthz"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// `[NAME]` in a proc's leading `##` doc block resolves to that + /// proc's declaration when NAME is defined in the same document. + #[test] + fn doc_ref_in_proc_block_resolves_to_proc() { + let src = "\ +proc target {} { puts hi } +## See [target] for related config. +proc other {} { return 1 } +"; + let parsed = parse(src); + // Cursor on the `t` inside `[target]`. + let pos = first(src, "[target]") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("target") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// `[NAME]` in a proc arg's `##` block resolves too — this is + /// exactly the shape the generator emits for `-port0`-style args. + #[test] + fn doc_ref_in_proc_arg_block_resolves() { + let src = "\ +proc mac_port {} { puts ok } +proc create { + ## Configuration for MAC port 0. Construct with [mac_port]. + port0 +} { return 1 } +"; + let parsed = parse(src); + let pos = first(src, "[mac_port]") + 1; + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::Proc(p) + if p.name.as_deref() == Some("mac_port") => + { + Some(p.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// Unresolved `[NAME]` — target doesn't exist — falls through to + /// None without touching the structural paths. + #[test] + fn doc_ref_to_unknown_returns_none() { + let src = "\ +## See [nonexistent] for details. +proc foo {} { puts hi } +"; + let parsed = parse(src); + let pos = first(src, "[nonexistent]") + 1; + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// Cursor on prose inside a doc comment (not on a `[…]` token) + /// falls through to structural resolution. Sanity check that the + /// doc-ref path doesn't intercept every cursor position in a `##`. + #[test] + fn doc_prose_falls_through() { + let src = "\ +## Just prose, no refs here at all. +proc target {} { puts hi } +"; + let parsed = parse(src); + // Cursor on the `p` of "prose" — inside the block but not + // inside a `[…]`. + let pos = first(src, "prose"); + assert!(definition_at(&parsed.document, src, pos).is_none()); + } + + /// Goto on a return-type annotation resolves to the `type` decl. + #[test] + fn goto_on_return_type_finds_type_decl() { + let src = "\ +type MyProps = string +proc use {name} MyProps { return $name } +"; + let parsed = parse(src); + // Cursor on `MyProps` in the return-type slot (2nd occurrence). + let pos = nth(src, "MyProps", 1); + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some("MyProps") => + { + Some(td.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } + + /// Goto on a qualified type name (`dcmac::MacPortProps`) resolves + /// to the declaration when the type_table key matches. + #[test] + fn goto_on_qualified_type_finds_decl() { + let src = "\ +namespace eval dcmac {} +namespace eval dcmac::T {} +type dcmac::T = string +proc dcmac::T::from {v: string} dcmac::T { return $v } +proc dcmac::T::to {v: dcmac::T} string { return $v } +proc dcmac::T::repr {v: dcmac::T} string { return $v } +proc use {port0: dcmac::T} string { return $port0 } +"; + let parsed = parse(src); + // Cursor on `T` inside `port0: dcmac::T` (the last occurrence). + let pos = src.rfind("dcmac::T").unwrap() as u32 + 7; // land on `T` + let target = definition_at(&parsed.document, src, pos).unwrap(); + let expected = parsed + .document + .stmts + .iter() + .find_map(|s| match s { + Stmt::Command(c) => match &c.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some("dcmac::T") => + { + Some(td.name_span) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + assert_eq!(target, expected); + } +} diff --git a/vw-htcl/src/hover.rs b/vw-htcl/src/hover.rs new file mode 100644 index 0000000..2bf186d --- /dev/null +++ b/vw-htcl/src/hover.rs @@ -0,0 +1,1284 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Find the htcl construct at a given byte offset. +//! +//! Used by `vw analyzer` for `textDocument/hover` and (later) by the +//! REPL for inline hover-style popups. Pure analysis — returns a +//! structured [`HoverTarget`] referencing into the document; the +//! caller formats it (markdown for LSP, a ratatui widget for the +//! REPL, etc). + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcArg, ProcSignature, Stmt, Word, + WordForm, WordPart, +}; +use crate::lower::{signature_table, SignatureTable}; +use crate::scope::{ + find_type_decl, innermost_scope, resolve_var_def, scan_var_ref, + type_expr_at, type_expr_lookup_name, VarDef, +}; +use crate::span::Span; + +/// A construct the cursor is on, plus the data needed to render +/// hover content. Lifetime-tied to the [`Document`] passed into +/// [`hover_at`]. +#[derive(Clone, Debug)] +pub enum HoverTarget<'a> { + /// Cursor is on the name of a `proc` declaration. The proc's own + /// signature contains the docs. + ProcDef { proc: &'a Proc, span: Span }, + /// Cursor is on the name of an argument inside a `proc` + /// declaration's args braces. + ProcArgDef { + proc_name: String, + arg: &'a ProcArg, + span: Span, + }, + /// Cursor is on the first word of a command that resolves to a + /// known structured proc — i.e. a call to a documented proc. + CallSite { + proc_name: String, + signature: &'a ProcSignature, + span: Span, + }, + /// Cursor is on a `-flag` word in a call to a known proc. + CallArg { + proc_name: String, + arg: &'a ProcArg, + span: Span, + }, + /// Cursor is on a `$var` reference that resolves to a local + /// (`set`/`variable`) rather than a parameter. The span is the + /// reference itself. `ty` carries the type inferred from the + /// binding's RHS when the shape is knowable (a `[typed_proc ...]` + /// call substitution, a `$other_typed_var` copy, or a + /// `true`/`false` literal); `None` when the RHS is opaque. + LocalVar { + name: String, + span: Span, + ty: Option, + }, + /// Cursor is on the name of an `enum` declaration. Shows the + /// variants block as a hover popup. + EnumDef { + decl: &'a crate::ast::EnumDecl, + span: Span, + }, + /// Cursor is on a type name in a proc signature or type-decl + /// underlying — e.g. `MyNewtype` in `proc f {} MyNewtype { … }` + /// or `dcmac::MacPortProps` in `-port0: dcmac::MacPortProps`. + /// Resolves to a declared `type` in the document. + TypeDef { + decl: &'a crate::ast::TypeDecl, + span: Span, + }, +} + +impl HoverTarget<'_> { + pub fn span(&self) -> Span { + match self { + HoverTarget::ProcDef { span, .. } + | HoverTarget::ProcArgDef { span, .. } + | HoverTarget::CallSite { span, .. } + | HoverTarget::CallArg { span, .. } + | HoverTarget::LocalVar { span, .. } + | HoverTarget::EnumDef { span, .. } + | HoverTarget::TypeDef { span, .. } => *span, + } + } +} + +pub fn hover_at<'a>( + document: &'a Document, + source: &'a str, + offset: u32, +) -> Option> { + let table = signature_table(document); + // Doc-comment `[NAME]` reference — cheap to detect and rules + // out any structural resolution when the cursor is inside a + // `##` block. Renders the target proc's signature as the hover + // content, same as if the cursor were on a call site. + if let Some(t) = hover_in_doc_comment(document, source, offset, &table) { + return Some(t); + } + // Cursor on the name-word of a `set VAR X` binding → treat it + // as the local's definition site, showing the same + // name-and-inferred-type hover that a `$VAR` reference would. + // Runs before the general stmt walker because that walker + // resolves `set` as a generic call and returns nothing useful + // for the name-word position. + if let Some(t) = hover_in_set_binding(document, source, offset, &table) { + return Some(t); + } + // Cursor on a binder-name inside a control-flow varname-list + // (`foreach {a b}`, `dict for {k v}`, `catch BODY resvar`). + // Same LocalVar hover shape as the `set NAME` binding site so + // the two look identical to the reader. + if let Some(t) = hover_in_control_flow_binding(document, source, offset) { + return Some(t); + } + hover_in_stmts(&document.stmts, &table, source, offset) + // Fallback: a `$var` reference — including one buried in opaque + // text (a command substitution or `if`/`while` condition). + .or_else(|| hover_scanned_var(document, source, offset, &table)) + // Fallback: cursor on a type-name annotation (arg type, + // return type, `type … = TYPE` underlying, generic arg). + .or_else(|| hover_of_type(document, offset)) +} + +/// Cursor on the name-word of a `set NAME VALUE` command (the +/// binding site, no `$` prefix). Reuses the same `infer_local_type` +/// walker as the `$var` fallback so the hover shows `$NAME: T` in +/// both places consistently. +fn hover_in_set_binding<'a>( + document: &'a Document, + _source: &'a str, + offset: u32, + sig_table: &crate::lower::SignatureTable<'a>, +) -> Option> { + use crate::ast::CommandKind; + let (stmts, enclosing) = innermost_scope(document, offset); + let cmd = find_set_command_at(stmts, offset)?; + // Skip the containing proc when scanning nested control-flow — + // for now `find_set_command_at` only looks at top-level stmts of + // the innermost scope, which covers the common case. + let CommandKind::Set = cmd.kind else { + return None; + }; + let name_word = cmd.words.get(1)?; + if !name_word.span.contains(offset) { + return None; + } + let name = name_word.as_text()?.to_string(); + let ty = infer_local_type( + stmts, + enclosing, + sig_table, + document, + &name, + name_word.span, + ); + Some(HoverTarget::LocalVar { + name, + span: name_word.span, + ty, + }) +} + +/// Cursor sits on a bare identifier inside a control-flow varname +/// list — the `lib` / `srcs` inside `dict for {lib srcs} …`, an +/// `a` / `b` inside `foreach {a b} …`, or the result-var of +/// `catch { … } err`. Returns a `LocalVar` target pointing at the +/// whole braced word (or the bare word), so hover shows the same +/// `$name` shape it would show on a `$name` reference lower in +/// the body. +/// +/// Sub-token spans inside the braced list aren't currently +/// tracked by the parser, so the returned span covers the whole +/// braced group — the click target still lands on `lib` (or +/// wherever the cursor is), just with a slightly wider highlight. +fn hover_in_control_flow_binding<'a>( + document: &'a Document, + source: &'a str, + offset: u32, +) -> Option> { + use crate::ast::{CommandKind, WordForm, WordPart}; + let (stmts, _enclosing) = innermost_scope(document, offset); + // Walk statements looking for a Generic-command whose head is + // a body-host and whose varname arg contains `offset`. + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if !matches!(cmd.kind, CommandKind::Generic) { + continue; + } + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + continue; + }; + // Collect (word-idx, is-varname-list) tuples per head. + let varname_word_indices: Vec = match head { + "foreach" => { + // Word 1, 3, 5, … up to body_idx (last word). + let body_idx = cmd.words.len().saturating_sub(1); + (1..body_idx).step_by(2).collect() + } + "dict" => { + if cmd.words.get(1).and_then(|w| w.as_text()) == Some("for") { + vec![2] + } else { + continue; + } + } + "catch" => vec![2, 3], + _ => continue, + }; + for idx in varname_word_indices { + let Some(word) = cmd.words.get(idx) else { + continue; + }; + if !word.span.contains(offset) { + continue; + } + // Figure out which sub-name the cursor is on. + let target_name = match word.form { + WordForm::Bare => word.as_text()?.to_string(), + WordForm::Braced => { + let WordPart::Text { + value, + span: text_span, + } = word.parts.first()? + else { + continue; + }; + // Find the whitespace-delimited token at the + // cursor offset within the braced interior. + let rel = offset.saturating_sub(text_span.start) as usize; + let bytes = value.as_bytes(); + let mut start = rel.min(bytes.len()); + while start > 0 + && !bytes[start - 1].is_ascii_whitespace() + && bytes[start - 1] != b'{' + { + start -= 1; + } + let mut end = rel.min(bytes.len()); + while end < bytes.len() + && !bytes[end].is_ascii_whitespace() + && bytes[end] != b'}' + { + end += 1; + } + if start >= end { + continue; + } + value[start..end].to_string() + } + _ => continue, + }; + let _ = source; // reserved for future sub-token spans + return Some(HoverTarget::LocalVar { + name: target_name, + span: word.span, + ty: None, + }); + } + } + None +} + +fn find_set_command_at(stmts: &[Stmt], offset: u32) -> Option<&Command> { + use crate::ast::{CommandKind, Stmt}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if matches!(cmd.kind, CommandKind::Set) { + return Some(cmd); + } + } + None +} + +/// Cursor on a type name → return a `TypeDef` hover target for the +/// matching declaration. Mirror of [`crate::goto::definition_of_type`]. +fn hover_of_type(document: &Document, offset: u32) -> Option> { + let ty = type_expr_at(document, offset)?; + let name = type_expr_lookup_name(ty); + let decl = find_type_decl(document, &name)?; + Some(HoverTarget::TypeDef { + decl, + span: ty.span(), + }) +} + +/// Hover for a `[NAME]` reference inside a `##` doc-comment block. +/// The result mirrors `HoverTarget::CallSite` for the referenced +/// proc, so the LSP formatter renders the target's signature with +/// its own docs — exactly what a reader following the reference +/// wants to see. +fn hover_in_doc_comment<'a>( + document: &'a Document, + source: &str, + offset: u32, + table: &SignatureTable<'a>, +) -> Option> { + let block = enclosing_doc_block(&document.stmts, offset)?; + let name = extract_ref_at(source, block, offset)?; + // Anchor the hover span on the `[NAME]` reference itself so the + // editor highlights just that token. + let ref_span = ref_span_at(source, block, offset)?; + let sig = *table.get(&name)?; + Some(HoverTarget::CallSite { + proc_name: name, + signature: sig, + span: ref_span, + }) +} + +/// Return the doc-comment span containing `offset`. Mirror of +/// [`crate::goto::enclosing_doc_block`] — kept crate-local because +/// both consumers want the same rule. +fn enclosing_doc_block(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let Some(span) = cmd.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(span) = arg.doc_comments_span { + if span.contains(offset) { + return Some(span); + } + } + } + } + if let Some(span) = enclosing_doc_block(&proc.body, offset) { + return Some(span); + } + } + if let CommandKind::NamespaceEval(ns) = &cmd.kind { + if let Some(span) = enclosing_doc_block(&ns.body, offset) { + return Some(span); + } + } + } + None +} + +/// Extract the identifier inside a `[NAME]` reference at `offset`. +/// See goto.rs's `extract_ref_at` for the disambiguation rules. +fn extract_ref_at(source: &str, block: Span, offset: u32) -> Option { + let (_, name) = find_ref(source, block, offset)?; + Some(name) +} + +/// Return the inclusive span of the `[NAME]` token at `offset` in +/// the block, so the hover popup anchors on the reference (not the +/// entire doc block). +fn ref_span_at(source: &str, block: Span, offset: u32) -> Option { + let (span, _) = find_ref(source, block, offset)?; + Some(span) +} + +fn find_ref(source: &str, block: Span, offset: u32) -> Option<(Span, String)> { + let bytes = source.as_bytes(); + let start = block.start as usize; + let end = (block.end as usize).min(bytes.len()); + let off = offset as usize; + if off < start || off > end { + return None; + } + let mut i = start; + while i < end { + if bytes[i] == b'[' { + let content_start = i + 1; + // Ident-start rule mirrors doc.rs / goto.rs. + if content_start < end + && (bytes[content_start].is_ascii_alphabetic() + || bytes[content_start] == b'_') + { + let mut j = content_start; + while j < end && bytes[j] != b']' { + let b = bytes[j]; + let ok = + b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + if !ok { + break; + } + j += 1; + } + if j < end && bytes[j] == b']' && j > content_start { + if off >= i && off <= j { + let name = + std::str::from_utf8(&bytes[content_start..j]) + .ok()? + .to_string(); + return Some(( + Span::new(i as u32, (j + 1) as u32), + name, + )); + } + i = j + 1; + continue; + } + } + } + i += 1; + } + None +} + +/// Hover for a `$var` reference found by scanning the source. Resolves +/// to a parameter (rendered like an arg) or a local (`set`/`variable`). +fn hover_scanned_var<'a>( + document: &'a Document, + source: &str, + offset: u32, + sig_table: &crate::lower::SignatureTable<'a>, +) -> Option> { + let (name, span) = scan_var_ref(source, offset)?; + let (stmts, enclosing) = innermost_scope(document, offset); + match resolve_var_def(&name, stmts, enclosing, offset)? { + VarDef::Param(arg) => Some(HoverTarget::ProcArgDef { + proc_name: enclosing + .and_then(|p| p.name.clone()) + .unwrap_or_default(), + arg, + // Anchor the hover on the reference, not the declaration. + span, + }), + VarDef::Local(def_span) => { + let ty = infer_local_type( + stmts, enclosing, sig_table, document, &name, def_span, + ); + Some(HoverTarget::LocalVar { name, span, ty }) + } + } +} + +/// Infer the type of the local variable `name` whose defining `set` +/// command's name-word lives at `def_span`. Walks `scope_stmts` in +/// order to seed `VarTypeTable` with any typed `set`s that come +/// before the target — so a chain like `set a [typed]; set b $a` +/// still types `b`. Seeds parameter types too, in case `set b $arg` +/// forwards a parameter through a local. Returns `None` when the +/// RHS is opaque (the same policy the validator uses). +fn infer_local_type<'a>( + scope_stmts: &'a [Stmt], + enclosing: Option<&'a crate::ast::Proc>, + sig_table: &crate::lower::SignatureTable<'a>, + document: &'a Document, + name: &str, + def_span: Span, +) -> Option { + use crate::ast::{CommandKind, Stmt}; + let mut var_table = crate::validate::VarTypeTable::new(); + // Parameter types are visible to `set` RHS inference — a forward + // like `set out $arg` propagates the arg's type through. + if let Some(proc) = enclosing { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + var_table.insert(arg.name.clone(), ty.clone()); + } + } + } + } + // Full document-wide proc table so `[user_proc]` return-type + // inference kicks in for user procs without an annotated + // return type — same walker the REPL / putr chain uses. + let proc_table = crate::validate::build_proc_table(document); + for stmt in scope_stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !matches!(cmd.kind, CommandKind::Set) { + continue; + } + let Some(name_word) = cmd.words.get(1) else { + continue; + }; + let Some(value_word) = cmd.words.get(2) else { + continue; + }; + let Some(binding_name) = name_word.as_text() else { + continue; + }; + // Type the RHS in the pre-target var-table (chain support). + let ty = crate::validate::value_type_with_procs( + value_word, + sig_table, + &var_table, + Some(&proc_table), + ); + if let Some(ref t) = ty { + var_table.insert(binding_name.to_string(), t.clone()); + } + // Return the type of the specific binding the hover is on — + // matched by name-word span so shadowed bindings before it + // don't overwrite the answer. + if name_word.span == def_span && binding_name == name { + return ty; + } + } + None +} + +/// Find the hover target at `offset` within `stmts`, descending into +/// proc bodies. The signature table is the document-wide (top-level) +/// one, so a call inside a body still resolves to the proc it names. +fn hover_in_stmts<'a>( + stmts: &'a [Stmt], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + if let Some(target) = hover_in_command(cmd, table, source, offset) { + return Some(target); + } + } + None +} + +fn hover_in_command<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + let primary = match &cmd.kind { + CommandKind::Proc(proc) => hover_in_proc_decl(proc, offset) + // Cursor isn't on the proc's name or an arg — look inside + // the body. + .or_else(|| hover_in_stmts(&proc.body, table, source, offset)), + CommandKind::NamespaceEval(ns) => { + // `namespace eval { … }` — descend into the + // populated body. The parser's post-pass already + // reparsed the block into `ns.body`, so we walk the + // structured AST rather than re-triggering the on-demand + // brace-body reparse. + hover_in_stmts(&ns.body, table, source, offset) + } + CommandKind::EnumDecl(decl) => { + // Cursor on the enum's name → show the variants. + if decl.name_span.contains(offset) { + Some(HoverTarget::EnumDef { + decl, + span: decl.name_span, + }) + } else { + None + } + } + _ => hover_in_call(cmd, table, offset), + }; + primary + .or_else(|| hover_in_cmd_substs(&cmd.words, table, source, offset)) + .or_else(|| hover_in_braced_bodies(cmd, table, source, offset)) +} + +/// Cursor inside a `{ … }` control-flow body (the second word of +/// `if`, the third of `while`, the body of `foreach`, etc.). The +/// parser leaves those as opaque `Braced` words — semantically +/// Tcl scripts, but no eager-parse pass like `[ … ]` gets. So we +/// reparse the body on demand when hover lands the cursor inside +/// one. Mirrors `goto::definition_in_braced_bodies` (same fix, same +/// motivating case — IP-wrapper `set_property` calls sit inside +/// `if {[llength $_vw_d] > 0} { … }` scaffolds). +fn hover_in_braced_bodies<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + let head = cmd.words.first().and_then(|w| w.as_text())?; + if !is_body_host(head) { + return None; + } + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + // Reparse against the fragment text; shift spans up to + // whole-source coordinates; THEN run `populate_procs` so + // nested `[ … ]` CmdSubst bodies inside the brace-body + // get their own recursive parse. Without the populate + // pass, a call like + // `set cell [vivado_cmd::create_bd_cell …]` inside an + // `if {$bd} { … }` branch has an empty CmdSubst body and + // the transient walker can't reach the call. + // + // Braced bodies are Tcl scripts — `\n` terminates a + // command. That's `Mode::Toplevel`. Historically this + // used `Mode::BracketBody` (for `[ … ]` substitutions + // where the whole content is ONE command), which + // glued every line of the body into a single flat + // command and made every non-head word (i.e. every + // call after the first) invisible to hover. + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::Toplevel, + ); + let delta = text_span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs(&mut stmts, source, &mut errs); + // NOTE: the reparsed stmts are owned by this call; but + // `HoverTarget` variants only borrow from `Command` / + // `Proc` / `ProcArg` / `ProcSignature` values that we've + // been threading via `&'a` from the outer document. + // Anything produced from the *reparsed* fragment would need + // its own owned storage — and there's nowhere to put it in + // the current HoverTarget shape. Rather than restructure + // that lifetime, we recurse only to look up calls that + // resolve through `table` (which lives at the top level + // and is already `'a`). + return hover_in_stmts_transient(&stmts, table, source, offset); + } + None +} + +/// Sig-table-only pass over transient (locally-owned) stmts: +/// only the `hover_in_call` branch that resolves through the +/// document-level `SignatureTable<'a>` is reachable. See the +/// comment in [`hover_in_braced_bodies`] for the lifetime story. +fn hover_in_stmts_transient<'a>( + stmts: &[Stmt], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + // Cursor on a call name → hover its proc via `table`. + if let Some(target) = hover_call_via_table(cmd, table, offset) { + return Some(target); + } + // Nested `[ … ]` inside the reparsed body — recurse via + // the same transient walker so an `if { if { … [call] } }` + // chain works. + for word in &cmd.words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return hover_in_stmts_transient( + body, table, source, offset, + ); + } + } + } + } + // Nested braced body inside the reparsed body — same idea. + let head = cmd.words.first().and_then(|w| w.as_text()); + if let Some(head) = head { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if !matches!(word.form, WordForm::Braced) { + continue; + } + if !word.span.contains(offset) { + continue; + } + let Some(WordPart::Text { + value, + span: text_span, + }) = word.parts.first() + else { + continue; + }; + // Nested braced body — same "script mode" + // reasoning as the outer reparse above. + let (mut inner, mut inner_errs) = + crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::Toplevel, + ); + let delta = text_span.start; + for s in &mut inner { + crate::parser::shift_stmt(s, delta); + } + crate::parser::populate_procs( + &mut inner, + source, + &mut inner_errs, + ); + return hover_in_stmts_transient( + &inner, table, source, offset, + ); + } + } + } + } + None +} + +/// Sig-table-only variant of [`hover_in_call`] that avoids taking +/// any borrow from `cmd` — returned data references only `'a` +/// values that live in the outer document via the sig table. +fn hover_call_via_table<'a>( + cmd: &Command, + table: &SignatureTable<'a>, + offset: u32, +) -> Option> { + let first = cmd.words.first()?; + let name_text = first.as_text()?; + let sig = *table.get(name_text)?; + // Cursor on the call name → the callee's signature. + if first.span.contains(offset) { + return Some(HoverTarget::CallSite { + proc_name: name_text.to_string(), + signature: sig, + span: first.span, + }); + } + // Cursor on a `-flag` word → the corresponding arg's docs. + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(HoverTarget::CallArg { + proc_name: name_text.to_string(), + arg, + span: word.span, + }); + } + None +} + +/// Command names whose brace-args hold Tcl scripts rather than +/// data. Same list as [`crate::goto`]'s counterpart. Exposed to +/// the crate so the unused-var pass (`crate::unused`) can reuse +/// it without a third copy. +pub(crate) fn is_body_host(head: &str) -> bool { + matches!( + head, + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + | "try" + | "finally" + | "eval" + | "uplevel" + | "namespace" + | "on" + | "apply" + // `dict for` — head word alone doesn't disambiguate + // (`dict get`/`dict set`/… have no script body); we + // include `dict` here and let the per-word-form loop + // skip non-braced args. False positives cost a + // reparse but never produce spurious hover results. + | "dict" + ) +} + +/// Descend into any `[ … ]` command substitutions on this command's +/// words so hover works on calls written inline, e.g. +/// `set cell [create_cpm5 -name x]`. +fn hover_in_cmd_substs<'a>( + words: &'a [Word], + table: &SignatureTable<'a>, + source: &'a str, + offset: u32, +) -> Option> { + for word in words { + if !word.span.contains(offset) { + continue; + } + for part in &word.parts { + if let crate::ast::WordPart::CmdSubst { span, body, .. } = part { + if span.contains(offset) { + return hover_in_stmts(body, table, source, offset); + } + } + } + } + None +} + +fn hover_in_proc_decl<'a>( + proc: &'a Proc, + offset: u32, +) -> Option> { + if proc.name_span.contains(offset) { + return Some(HoverTarget::ProcDef { + proc, + span: proc.name_span, + }); + } + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + if arg.name_span.contains(offset) { + let proc_name = proc.name.clone().unwrap_or_default(); + return Some(HoverTarget::ProcArgDef { + proc_name, + arg, + span: arg.name_span, + }); + } + } + } + None +} + +fn hover_in_call<'a>( + cmd: &'a Command, + table: &SignatureTable<'a>, + offset: u32, +) -> Option> { + let first = cmd.words.first()?; + let name = first.as_text()?; + let sig = *table.get(name)?; + + if first.span.contains(offset) { + return Some(HoverTarget::CallSite { + proc_name: name.to_string(), + signature: sig, + span: first.span, + }); + } + + // Walk remaining words looking for the `-flag` under the cursor. + // Value words (the token after a flag) don't trigger hover — + // they could be anything from a literal to a [cmd subst], and + // there's no general definition to point at. + for word in cmd.words.iter().skip(1) { + if !word.span.contains(offset) { + continue; + } + let text = word.as_text()?; + let flag = text.strip_prefix('-')?; + let arg = sig.find(flag)?; + return Some(HoverTarget::CallArg { + proc_name: name.to_string(), + arg, + span: word.span, + }); + } + None +} + +// Helpers retained for symmetric use from formatters that want to +// pretty-print attributes etc. without re-walking from raw AST. +#[allow(dead_code)] +fn _word_text(word: &Word) -> Option<&str> { + word.as_text() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn at(src: &str, needle: &str, occurrence: usize) -> u32 { + let mut start = 0; + for i in 0..=occurrence { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found"); + if i == occurrence { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + fn first(src: &str, needle: &str) -> u32 { + at(src, needle, 0) + } + + #[test] + fn hover_on_call_name() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let target = + hover_at(&parsed.document, src, first(src, "greet -")).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "greet"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_call_arg_flag() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let pos = first(src, "-name there"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallArg { arg, proc_name, .. } => { + assert_eq!(proc_name, "greet"); + assert_eq!(arg.name, "name"); + } + other => panic!("expected CallArg, got {other:?}"), + } + } + + #[test] + fn hover_on_value_word_returns_none() { + let src = "\ +proc greet {\n @default(\"world\") name\n} { puts $name }\n\ +greet -name there\n"; + let parsed = parse(src); + let pos = first(src, "there"); + assert!(hover_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn hover_on_proc_decl_name() { + let src = "proc greet {\n name\n} { puts $name }\n"; + let parsed = parse(src); + let pos = first(src, "greet"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + assert!(matches!(target, HoverTarget::ProcDef { .. })); + } + + #[test] + fn hover_on_proc_arg_decl() { + let src = "proc greet {\n @default(\"x\") name\n} { puts hi }\n"; + let parsed = parse(src); + let pos = first(src, "name"); // first "name" is in args + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::ProcArgDef { arg, .. } => { + assert_eq!(arg.name, "name"); + assert_eq!(arg.attributes[0].name, "default"); + } + other => panic!("expected ProcArgDef, got {other:?}"), + } + } + + #[test] + fn hover_on_call_inside_proc_body() { + // A call to a documented proc from within another proc's body + // should hover, just like a top-level call. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis {\n width\n} {\n if_tport\n}\n"; + let parsed = parse(src); + let pos = at(src, "if_tport", 1); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "if_tport"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_call_inside_command_substitution() { + // The interior of `[ … ]` is now parsed; hover on the inner + // call's name should report the proc the same way it does at + // the top level. + let src = "\ +proc create_cpm5 {\n @default(0) name\n} { puts hi }\n\ +set cell [create_cpm5 -name x]\n"; + let parsed = parse(src); + let pos = at(src, "create_cpm5", 1); // the call inside [ ] + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "create_cpm5"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + #[test] + fn hover_on_unknown_call_returns_none() { + let src = "puts hello\n"; + let parsed = parse(src); + let pos = first(src, "puts"); + assert!(hover_at(&parsed.document, src, pos).is_none()); + } + + #[test] + fn hover_on_var_in_condition_shows_param() { + // `$kind` inside an opaque `if` condition resolves, via the + // source scan, to the proc parameter — rendered like an arg. + let src = "\ +proc axis_if {\n @enum(target, controller) kind\n} {\n\ + set m [ if {$kind == controller} { a } ]\n}\n"; + let parsed = parse(src); + let pos = first(src, "$kind") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::ProcArgDef { arg, .. } => { + assert_eq!(arg.name, "kind"); + assert_eq!(arg.attributes[0].name, "enum"); + } + other => panic!("expected ProcArgDef, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_reports_local() { + let src = "\ +proc p {} {\n set count 0\n use $count\n}\n"; + let parsed = parse(src); + let pos = first(src, "$count") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "count"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_var_ref_inside_dict_for_body_resolves_to_kv_binder() { + // `dict for {lib srcs} $deps { … $lib … }` — hovering the + // `$lib` reference in the body should resolve to the + // binder at the `{lib srcs}` list. + let src = "\ +set deps [some_proc] +dict for {lib srcs} $deps { + puts $lib +} +"; + let parsed = parse(src); + let pos = first(src, "$lib\n") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "lib"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_bare_name_inside_dict_for_kv_list_resolves() { + // Cursor on `lib` INSIDE `{lib srcs}` — the binding site + // itself. Same LocalVar hover shape as if the cursor were + // on `$lib` later in the body. + let src = "dict for {lib srcs} $deps { puts $lib }\n"; + let parsed = parse(src); + let pos = first(src, "lib srcs"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "lib"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_foreach_kv_list_binder_resolves() { + // Same story for `foreach {a b} $pairs { … }`. + let src = "foreach {a b} $pairs { puts $a }\n"; + let parsed = parse(src); + let pos = first(src, "a b}"); + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, .. } => assert_eq!(name, "a"), + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_set_binding_name_infers_type() { + // Cursor on the LHS of `set x [typed]` (no `$`) — should + // resolve as a LocalVar with the RHS's type, same as + // hovering `$x` later would. + let src = "\ +proc make_it {} string { return hi } +proc p {} { + set x [make_it] +} +"; + let parsed = parse(src); + // Cursor lands on the `x` of `set x` (not `$x`). + let pos = first(src, "set x ") + "set ".len() as u32; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, ty, .. } => { + assert_eq!(name, "x"); + let ty = ty.expect("expected inferred type"); + assert!( + matches!( + ty, + crate::ast::TypeExpr::Named { ref name, .. } + if name == "string" + ), + "got {ty:?}", + ); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_infers_type_from_typed_call() { + // `set x [typed_proc ...]` where `typed_proc` has an + // annotated return type → hovering `$x` should carry that + // type. + let src = "\ +proc make_it {} string { return hi } +proc p {} { + set x [make_it] + use $x +} +"; + let parsed = parse(src); + let pos = first(src, "$x") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { name, ty, .. } => { + assert_eq!(name, "x"); + let ty = ty.expect("expected inferred type"); + assert!( + matches!( + ty, + crate::ast::TypeExpr::Named { ref name, .. } + if name == "string" + ), + "got {ty:?}", + ); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + #[test] + fn hover_on_local_var_untyped_rhs_reports_no_type() { + let src = "\ +proc p {} { + set x [some_untyped] + use $x +} +"; + let parsed = parse(src); + let pos = first(src, "$x") + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::LocalVar { ty, .. } => { + assert!(ty.is_none(), "expected no type, got {ty:?}"); + } + other => panic!("expected LocalVar, got {other:?}"), + } + } + + /// Repro for a bug reported against `~/src/htcl/vw/module.htcl`: + /// hover on `log::info` sitting inside `if { … } { body }` + /// after a `\`-continued `vivado_cmd::set_msg_config` returned + /// None. Root cause was `hover_in_braced_bodies` reparsing the + /// braced body in `Mode::BracketBody` — that mode treats `\n` + /// as whitespace (correct for `[ … ]` substitution, wrong for + /// script bodies), so every line of the body glued together + /// into one command whose head resolved fine but whose + /// subsequent calls (`log::info`, `synth_ip`) became word + /// arguments and were invisible to hover. Fixed by switching + /// the reparse to `Mode::Toplevel`. Uses qualified names to + /// mirror the exact shape of the reporting file. + #[test] + fn hover_on_qualified_call_inside_if_body_after_backslash_continued_prev() { + let src = "\ +namespace eval log { + proc info {} { return 0 } +} +namespace eval vivado_cmd { + proc set_msg_config {} { return 0 } + proc synth_ip {} { return 0 } +} +proc caller {} { + if {1 > 0} { + vivado_cmd::set_msg_config \\ + -id \"abc\" \\ + -suppress true + log::info -id 1 \\ + -msg \"hi\" + vivado_cmd::synth_ip -objects x + } +} +"; + let parsed = crate::parser::parse(src); + let pos = src.find("log::info -id 1").unwrap() as u32 + 2; + let target = hover_at(&parsed.document, src, pos) + .expect("expected hover to resolve log::info inside if-body"); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "log::info"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + /// Sibling of the if-body test above: same qualified-call + + /// backslash-continuation shape but WITHOUT the enclosing + /// `if { … }`. Confirms the pattern already worked at top + /// level (a proc body parses in `Mode::Toplevel`) — the + /// regression was specific to reparsed braced bodies. + #[test] + fn hover_on_qualified_call_after_backslash_continued_prev_command() { + let src = "\ +namespace eval log { + proc info {} { return 0 } +} +namespace eval vivado_cmd { + proc set_msg_config {} { return 0 } + proc synth_ip {} { return 0 } +} +proc caller {} { + vivado_cmd::set_msg_config \\ + -id \"abc\" \\ + -suppress true + log::info -id 1 \\ + -msg \"hi\" + vivado_cmd::synth_ip -objects x +} +"; + let parsed = crate::parser::parse(src); + let pos = src.find("log::info -id 1").unwrap() as u32 + 2; + let target = hover_at(&parsed.document, src, pos) + .expect("expected hover to resolve log::info call"); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "log::info"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } + + /// `[NAME]` inside a `##` block renders as a CallSite hover + /// on the referenced proc — same as if the cursor were on a + /// live call. Lets the reader hover the reference and see + /// the target's signature. + #[test] + fn doc_ref_hovers_as_target_call_site() { + let src = "\ +## Documented target proc. +proc target {} { puts hi } +## See [target] for more. +proc caller {} { return 1 } +"; + let parsed = crate::parser::parse(src); + let pos = src.find("[target]").unwrap() as u32 + 1; + let target = hover_at(&parsed.document, src, pos).unwrap(); + match target { + HoverTarget::CallSite { proc_name, .. } => { + assert_eq!(proc_name, "target"); + } + other => panic!("expected CallSite, got {other:?}"), + } + } +} diff --git a/vw-htcl/src/lib.rs b/vw-htcl/src/lib.rs new file mode 100644 index 0000000..2df00bb --- /dev/null +++ b/vw-htcl/src/lib.rs @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl language layer. +//! +//! Provides the parser, concrete syntax tree, and analysis passes that +//! every htcl-consuming subcommand of `vw` shares. The same code drives +//! `vw run`, `vw check`, the LSP (`vw analyzer`), and (eventually) the +//! REPL (`vw repl`). Keeping a single source of truth for parsing and +//! analysis is the durable fix for the "compiler vs. IDE drift" failure +//! mode of language tooling. +//! +//! This v0 covers the Phase 0 subset from the project plan: literals, +//! variables, command substitution, `set`, `proc` (vanilla form), +//! generic command invocations, and comments. Control flow, structured +//! proc grammar, modules, and dependency-aware imports come in later +//! phases. + +// `quote_tcl!` (and `quote_htcl!`) generate code that names this +// crate as `::vw_htcl::…`. Within the crate itself that path +// doesn't resolve by default; this directive aliases the current +// crate as `vw_htcl` so the macros work uniformly inside and +// outside of vw-htcl. Standard Rust idiom for self-targeting +// proc-macros. +extern crate self as vw_htcl; + +pub mod ast; +pub mod cmdline; +pub mod complete; +pub mod doc; +pub mod emit; +pub mod enum_parse; +pub mod goto; +pub mod hover; +pub mod line_index; +pub mod loader; +pub mod lower; +pub mod overload; +pub mod parser; +pub mod proc_args; +pub mod putr; +pub mod references; +pub mod rename; +pub mod repr; +pub mod scope; +pub mod signature_help; +pub mod span; +pub mod src_path; +pub mod type_parse; +pub mod undefined; +pub use undefined::{top_level_var_names, top_level_var_types}; +pub mod unused; +pub mod validate; + +pub use complete::{ + complete_at, complete_at_with_extras, Completion, CompletionKind, +}; +pub use goto::definition_at; +pub use hover::{hover_at, HoverTarget}; +pub use loader::{ + load as load_program, load_source as load_program_source, + load_with_observer as load_program_with_observer, + load_with_preloaded as load_program_with_preloaded, ImportEdge, LoadError, + LoadObserver, LoadedFile, LoadedProgram, SourceRegion, +}; +pub use lower::{ + extern_rename_prelude, is_extern_call, lower_command, + lower_command_with_putr, lower_command_with_putr_and_index, + lower_proc_decl_with_name, lower_proc_decl_with_name_and_index, + rewrite_externs, signature_table, ExternRewrite, SignatureTable, + EXTERN_PREFIX, +}; +pub use overload::emit_dispatcher; +pub use references::{find_references_in, identify_at, ReferenceTarget}; +pub use rename::{rename_at, RenameEdit}; +pub use repr::{ + emit_enum_prelude, emit_primitive_prelude, emit_repr, emit_repr_with_types, +}; +pub use signature_help::{signature_help_at, SignatureHelp}; +pub use src_path::{ + classify as classify_src_path, PathKind, ResolveError, Resolver, +}; +pub use validate::{ + build_enum_decl_table, build_signature_table_with_overloads, + build_type_decl_table, mangle_specialization, validate, + validate_with_all_extras, validate_with_all_extras_and_vars, + validate_with_extras, validate_with_signatures, + Diagnostic as ValidatorDiagnostic, OverloadTable, Severity, +}; + +pub use ast::{ + Attribute, AttributeValue, Command, CommandKind, Document, EnumDecl, + EnumVariant, OverloadInfo, OverloadVariant, Proc, ProcArg, ProcSignature, + SrcImport, Stmt, TypeDecl, TypeExpr, Word, WordPart, +}; +pub use line_index::{LineCol, LineIndex}; +pub use parser::{parse, ParseError, ParseOutput}; +pub use span::Span; diff --git a/vw-htcl/src/line_index.rs b/vw-htcl/src/line_index.rs new file mode 100644 index 0000000..4375eca --- /dev/null +++ b/vw-htcl/src/line_index.rs @@ -0,0 +1,163 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Byte-offset ↔ line/column conversion. +//! +//! LSP positions are 0-indexed `(line, character)` where `character` +//! counts UTF-16 code units, not bytes. We honor that here so the +//! editor's cursor lands where the user expects on non-ASCII source. + +use crate::span::Span; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LineCol { + pub line: u32, + pub character: u32, +} + +#[derive(Clone, Debug)] +pub struct LineIndex { + /// Byte offset of the start of each line. `line_starts[0] == 0`. + line_starts: Vec, + /// Full source text; needed for the UTF-8 → UTF-16 column + /// conversion. + text: String, +} + +impl LineIndex { + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0u32]; + for (i, b) in text.bytes().enumerate() { + if b == b'\n' { + line_starts.push((i + 1) as u32); + } + } + Self { + line_starts, + text: text.to_string(), + } + } + + pub fn position(&self, byte_offset: u32) -> LineCol { + let offset = byte_offset.min(self.text.len() as u32); + let line_idx = match self.line_starts.binary_search(&offset) { + Ok(i) => i, + Err(i) => i - 1, + }; + let line_start = self.line_starts[line_idx]; + let line_text = &self.text[line_start as usize..offset as usize]; + let character = line_text.encode_utf16().count() as u32; + LineCol { + line: line_idx as u32, + character, + } + } + + pub fn range(&self, span: Span) -> (LineCol, LineCol) { + (self.position(span.start), self.position(span.end)) + } + + /// Convert a UTF-16 line/character position back to a byte + /// offset. Inverse of [`position`](Self::position); used to map + /// LSP positions from clients (which speak UTF-16) into byte + /// offsets the rest of the analysis uses. + /// + /// Clamps gracefully: a line past EOF returns the source length; + /// a character past EOL returns the offset of the line ending. + pub fn offset_of(&self, lc: LineCol) -> u32 { + let Some(&line_start) = self.line_starts.get(lc.line as usize) else { + return self.text.len() as u32; + }; + let line_end = self + .line_starts + .get(lc.line as usize + 1) + .copied() + .map(|n| n.saturating_sub(1)) + .unwrap_or(self.text.len() as u32); + let line_text = &self.text[line_start as usize..line_end as usize]; + let mut byte_offset = line_start; + let mut char_count: u32 = 0; + for ch in line_text.chars() { + if char_count >= lc.character { + break; + } + char_count += ch.len_utf16() as u32; + byte_offset += ch.len_utf8() as u32; + } + byte_offset + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_positions() { + let idx = LineIndex::new("abc\nde\nf"); + assert_eq!( + idx.position(0), + LineCol { + line: 0, + character: 0 + } + ); + assert_eq!( + idx.position(3), + LineCol { + line: 0, + character: 3 + } + ); + assert_eq!( + idx.position(4), + LineCol { + line: 1, + character: 0 + } + ); + assert_eq!( + idx.position(7), + LineCol { + line: 2, + character: 0 + } + ); + } + + #[test] + fn utf16_character_count() { + // `é` is one UTF-16 code unit, two UTF-8 bytes. + let idx = LineIndex::new("é\nx"); + let pos = idx.position(2); // byte after `é` + assert_eq!( + pos, + LineCol { + line: 0, + character: 1 + } + ); + } + + #[test] + fn offset_of_round_trips() { + let idx = LineIndex::new("abc\nde\nf"); + for &b in &[0u32, 1, 3, 4, 6, 7] { + assert_eq!(idx.offset_of(idx.position(b)), b); + } + } + + #[test] + fn offset_of_clamps_past_line_end() { + let idx = LineIndex::new("abc\nde"); + // line 0, character 100 → end of line 0 (byte 3) + assert_eq!( + idx.offset_of(LineCol { + line: 0, + character: 100 + }), + 3 + ); + } +} diff --git a/vw-htcl/src/loader.rs b/vw-htcl/src/loader.rs new file mode 100644 index 0000000..50468d4 --- /dev/null +++ b/vw-htcl/src/loader.rs @@ -0,0 +1,738 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Recursive `src` import resolution. +//! +//! Reads an entry-point .htcl file, parses it, resolves every `src` +//! statement via [`crate::src_path::Resolver`], and recursively pulls +//! in each imported module's contents. Idempotent on canonical +//! (realpath'd) file paths — a file imported by N callers loads +//! exactly once. +//! +//! The output is a single flat [`LoadedProgram`] carrying: +//! +//! - the concatenated source text (imports first, in topological +//! order, then the entry file's non-`src` content), which downstream +//! stages (lower, the analyzer, `vw run`) consume as if it were one +//! document; +//! - the set of canonical paths that were loaded, for cache +//! invalidation and tooling. + +use std::collections::HashSet; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::ast::{CommandKind, Stmt}; +use crate::parser::parse; +use crate::src_path::{ResolveError, Resolver}; + +#[derive(Debug, thiserror::Error)] +pub enum LoadError { + #[error("reading {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("resolving `src {raw}` from {importer}: {source}")] + Resolve { + importer: PathBuf, + raw: String, + #[source] + source: ResolveError, + }, + #[error( + "`src` import at {importer}:{line} has a non-literal path (it \ + contains `$var` or `[cmd]` substitution); module paths must \ + be a plain string" + )] + DynamicPath { importer: PathBuf, line: u32 }, + #[error("parse errors in {path}")] + Parse { + path: PathBuf, + errors: Vec, + }, +} + +/// Hooks called as the loader makes progress. Lets the CLI surface +/// real-time `Sourcing …` / `Checking …` lines without baking display +/// concerns into the loader. +/// +/// Events fire in dependency-first order, which matches Cargo's +/// "compile deps before the top crate" convention: for each `src` +/// import we hit, [`on_source`](Self::on_source) fires immediately, +/// the import is loaded (recursing through *its* dependencies first), +/// and only then does [`on_parsed`](Self::on_parsed) fire for that +/// file. The entry file's `on_parsed` fires last. +pub trait LoadObserver { + /// A `src ` statement is about to be resolved and loaded. + /// `resolved` is the on-disk path the loader is about to read, + /// so the observer can render dep-relative labels like + /// `@cpm/cpm_pcie1_axibar2pcie` even when `raw` itself is a + /// relative path (`./cpm_pcie1_axibar2pcie`) inside a dep. + fn on_source(&mut self, _raw: &str, _resolved: &Path) {} + /// `file` finished parsing. `raw` is the original `src` text when + /// this file was reached through an import (so callers can render + /// `amd-htcl/cpm5` rather than the full filesystem path); `None` + /// for the entry file. + fn on_parsed(&mut self, _file: &Path, _raw: Option<&str>) {} +} + +struct NoopObserver; +impl LoadObserver for NoopObserver {} + +#[derive(Debug, Default)] +pub struct LoadedProgram { + /// Flattened htcl source — every loaded file's non-`src` content, + /// concatenated. Downstream stages (lower, the analyzer in CLI + /// mode, `vw run`) consume this as if it were one document. + pub source: String, + /// Files seen, in the order [`load_file`] first visits them + /// (importer-first, depth-first). Each entry carries the file's + /// canonical path and its original on-disk text so callers can + /// map a span in [`source`](Self::source) back to a line/column + /// in the file it actually came from. + pub files: Vec, + /// Per-region map from byte ranges in [`source`](Self::source) + /// to `(file_index, file_offset)`. Regions are emitted in order + /// as content is concatenated, so the slice is sorted by + /// `flat_start` and non-overlapping — `locate` does a binary + /// search. + pub regions: Vec, +} + +#[derive(Debug, Clone)] +pub struct LoadedFile { + pub path: PathBuf, + pub source: String, + /// The `src` import that pulled this file in, or `None` for + /// the entry file the loader was started against. Captured at + /// load time so analyzers and the REPL can render call chains + /// like `failing.htcl:12 ← importer.htcl:4 (src @dep/foo) + /// ← entry.htcl:1 (src ip/cips)` — which is the htcl-level + /// equivalent of a stack trace. + pub imported_via: Option, + /// Modification time of the file at read time. Populated when + /// the fs metadata call succeeds; `None` when it doesn't + /// (unlikely on a file we just read, but the API allows it). + /// Used by the REPL's cross-batch loader cache: if the + /// current mtime differs from this stored one, the file + /// gets re-read on the next `src` — this is what makes + /// live-editing an already-sourced `.htcl` file work + /// interactively without a full restart. + pub mtime: Option, +} + +/// An edge in the import graph: this file was loaded because the +/// file at index [`Self::importer_file`] executed a `src` statement +/// covering [`Self::src_span`] in the importer's source. +#[derive(Debug, Clone, Copy)] +pub struct ImportEdge { + pub importer_file: usize, + /// Span of the `src` statement in the **importer's file-local + /// source** (i.e. an offset into the importer's + /// [`LoadedFile::source`], *not* into the flattened + /// [`LoadedProgram::source`]). + pub src_span: crate::span::Span, +} + +#[derive(Debug, Clone, Copy)] +pub struct SourceRegion { + /// Inclusive byte start in the flattened source. + pub flat_start: u32, + /// Exclusive byte end in the flattened source. + pub flat_end: u32, + /// Index into [`LoadedProgram::files`]. + pub file_index: u32, + /// Byte offset of the start of this region in the originating + /// file's source. + pub file_offset: u32, +} + +impl LoadedProgram { + /// Map a byte offset in [`source`](Self::source) back to its + /// originating file's index and the byte offset within that file. + pub fn locate(&self, offset: u32) -> Option<(usize, u32)> { + // `regions` is sorted by `flat_start`; find the last region + // whose start is at or before `offset` and verify the offset + // falls inside it. + let idx = self.regions.partition_point(|r| r.flat_start <= offset); + if idx == 0 { + return None; + } + let region = &self.regions[idx - 1]; + if offset >= region.flat_end { + return None; + } + Some(( + region.file_index as usize, + region.file_offset + (offset - region.flat_start), + )) + } + + /// Map a span in the flattened source to `(file_index, + /// file_local_span)`. Assumes the span lies within a single + /// originating file's contribution — true for diagnostics emitted + /// against a single word/command, which is the use case we care + /// about. + pub fn locate_span( + &self, + span: crate::span::Span, + ) -> Option<(usize, crate::span::Span)> { + let (file_index, file_start) = self.locate(span.start)?; + let length = span.end.saturating_sub(span.start); + Some(( + file_index, + crate::span::Span::new(file_start, file_start + length), + )) + } + + /// Walk the import chain from `file_index` toward the entry, + /// yielding each [`ImportEdge`] in order (nearest first). The + /// entry file has no edge and so produces no items. + pub fn ancestry( + &self, + file_index: usize, + ) -> impl Iterator + '_ { + let mut cur = self.files.get(file_index).and_then(|f| f.imported_via); + std::iter::from_fn(move || { + let edge = cur?; + cur = self + .files + .get(edge.importer_file) + .and_then(|f| f.imported_via); + Some(edge) + }) + } +} + +/// Read `entry` and recursively resolve its imports. Each file is +/// loaded at most once; circular imports (a → b → a) short-circuit on +/// the second visit. +pub fn load( + entry: &Path, + resolver: &Resolver, +) -> Result { + let mut noop = NoopObserver; + load_with_observer(entry, resolver, &mut noop) +} + +/// Like [`load`], but reports progress through `observer` so the CLI +/// can print `Sourcing …` and `Checking …` lines. +pub fn load_with_observer( + entry: &Path, + resolver: &Resolver, + observer: &mut dyn LoadObserver, +) -> Result { + load_with_preloaded( + entry, + resolver, + observer, + &std::collections::HashMap::new(), + ) +} + +/// Same as [`load_with_observer`] but seeds the "already loaded" +/// map with paths → mtime-at-load-time. +/// +/// The REPL uses this to skip re-parsing files a prior batch has +/// already sourced. Without it, `src ip/gtm` in a REPL session +/// that just `--load prime.htcl`-ed the same file re-parses +/// **every** transitive import from scratch — the loader's own +/// `self.loaded` cache is per-load-call, so cross-batch +/// redundancy explodes into O(minutes) for a large tree. +/// +/// A preloaded path short-circuits ONLY when the file's current +/// on-disk mtime matches the stored one. That's what lets a user +/// edit `ip/gtm.htcl`, then re-run `src ip/gtm` at the REPL and +/// actually see the change — the loader stats the target, sees +/// the mtime bump, and re-reads + re-parses instead of taking +/// the cached-empty path. A stat per hit is negligible next to +/// what the re-parse would cost. +/// +/// A missing mtime (`None`) on either side downgrades to +/// "unknown → always reload" — safer than silently reusing +/// possibly-stale content. +pub fn load_with_preloaded( + entry: &Path, + resolver: &Resolver, + observer: &mut dyn LoadObserver, + preloaded: &std::collections::HashMap, +) -> Result { + let entry = entry.canonicalize().unwrap_or_else(|_| entry.to_path_buf()); + let mut state = State { + program: LoadedProgram::default(), + // Seed `preloaded_mtimes` — the actual short-circuit + // check is inside `load_file` and gates on the mtime + // being unchanged since the prior batch stored it. The + // entry path itself is intentionally omitted even if the + // caller included it; that path is the batch's own + // scratch or the user's typed input, which the caller + // wants walked regardless. + preloaded_mtimes: preloaded + .iter() + .filter(|(p, _)| *p != &entry) + .map(|(p, t)| (p.clone(), *t)) + .collect(), + loaded: HashSet::new(), + in_progress: HashSet::new(), + resolver, + observer, + entry_override: None, + }; + state.load_file(&entry, None, None)?; + Ok(state.program) +} + +/// Like [`load`], but the entry's content is supplied in memory rather +/// than read from disk. `entry_path` is a synthetic path — it need not +/// exist on disk; it only anchors the workspace/resolver (its parent +/// dir is used for relative `src ./…` imports, and it feeds +/// `LoadedProgram::files[0]`). `@name` imports still resolve through +/// the resolver's dependency map and are read from disk normally. +/// +/// This is what lets `vw check` build and run an in-memory +/// `src @vw` + `vw::configure_ip` program without writing a temp file. +pub fn load_source( + entry_source: &str, + entry_path: &Path, + resolver: &Resolver, +) -> Result { + let mut noop = NoopObserver; + let mut state = State { + program: LoadedProgram::default(), + preloaded_mtimes: std::collections::HashMap::new(), + loaded: HashSet::new(), + in_progress: HashSet::new(), + resolver, + observer: &mut noop, + entry_override: Some(( + entry_path.to_path_buf(), + entry_source.to_string(), + )), + }; + state.load_file(entry_path, None, None)?; + Ok(state.program) +} + +struct State<'r, 'o> { + program: LoadedProgram, + /// In-memory content for the entry file, when the load was + /// started via [`load_source`]. When the loader visits this exact + /// path it uses this string instead of reading from disk; every + /// other (`src @dep/…`) file is read normally. `None` for + /// disk-backed loads. + entry_override: Option<(PathBuf, String)>, + /// Files the loader has already read this call (its own dedup + /// tracking). Grows as `load_file` recurses. + loaded: HashSet, + /// Cross-call preloaded set: path → mtime-when-prior-batch- + /// loaded-it. `load_file` short-circuits only when the entry + /// is here AND the current mtime matches. Never mutated + /// during a load call. + preloaded_mtimes: std::collections::HashMap, + in_progress: HashSet, + resolver: &'r Resolver, + observer: &'o mut dyn LoadObserver, +} + +impl State<'_, '_> { + fn load_file( + &mut self, + path: &Path, + reached_via: Option<&str>, + imported_via: Option, + ) -> Result<(), LoadError> { + if self.loaded.contains(path) || self.in_progress.contains(path) { + return Ok(()); + } + // Cross-batch cache: skip re-parse only when this file + // was preloaded by a prior batch AND its on-disk mtime + // hasn't budged. Any mismatch (unknown current mtime, + // different mtime, no stored mtime) forces a fresh + // read so live edits show up on the next `src`. + if let Some(stored_mtime) = self.preloaded_mtimes.get(path) { + let current_mtime = + fs::metadata(path).ok().and_then(|m| m.modified().ok()); + if current_mtime == Some(*stored_mtime) { + return Ok(()); + } + } + self.in_progress.insert(path.to_path_buf()); + + // A `load_source` entry is supplied in memory — use it instead + // of touching the disk (the synthetic path may not exist). + // Its mtime is `None` (nothing to stat), which the + // preloaded-cache path treats as "unknown → always reload". + let entry_override = match &self.entry_override { + Some((op, os)) if op.as_path() == path => Some(os.clone()), + _ => None, + }; + let (source, mtime) = match entry_override { + Some(src) => (src, None), + None => { + let source = + fs::read_to_string(path).map_err(|e| LoadError::Io { + path: path.to_path_buf(), + source: e, + })?; + // Stat right after the read so the recorded mtime + // matches the source we just captured. Failing to + // stat (fs permissions, deleted between read and + // stat, etc.) downgrades to `None`, which the + // preloaded-cache path treats as "unknown → always + // reload". + let mtime = + fs::metadata(path).ok().and_then(|m| m.modified().ok()); + (source, mtime) + } + }; + let parsed = parse(&source); + if !parsed.errors.is_empty() { + return Err(LoadError::Parse { + path: path.to_path_buf(), + errors: parsed.errors, + }); + } + + // Register the file up front so we have a stable index for + // every chunk we emit on its behalf. + let file_index = self.program.files.len() as u32; + self.program.files.push(LoadedFile { + path: path.to_path_buf(), + source: source.clone(), + imported_via, + mtime, + }); + + // Walk the parsed document, copying text in span order. Any + // `src` statement triggers a recursion so the imported content + // lands in the flat source before we continue the importer's + // remaining text. Each pushed slice gets a `SourceRegion` + // entry so locations in the flat source can be mapped back. + let mut cursor = 0usize; + let parent_dir = path.parent().unwrap_or_else(|| Path::new(".")); + for stmt in &parsed.document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(import) = &cmd.kind else { + continue; + }; + self.emit_chunk( + &source, + cursor, + cmd.span.start as usize, + file_index, + ); + cursor = cmd.span.end as usize; + // Skip the trailing newline that terminated the `src` + // command so we don't leave a stray blank line behind. + if source.as_bytes().get(cursor) == Some(&b'\n') { + cursor += 1; + } + + let Some(raw) = import.path.as_deref() else { + let line = line_of(&source, cmd.span.start) + 1; + return Err(LoadError::DynamicPath { + importer: path.to_path_buf(), + line, + }); + }; + let resolved = + self.resolver.resolve(parent_dir, raw).map_err(|source| { + LoadError::Resolve { + importer: path.to_path_buf(), + raw: raw.to_string(), + source, + } + })?; + if !self.loaded.contains(&resolved) + && !self.in_progress.contains(&resolved) + { + self.observer.on_source(raw, &resolved); + } + self.load_file( + &resolved, + Some(raw), + Some(ImportEdge { + importer_file: file_index as usize, + src_span: cmd.span, + }), + )?; + } + // Tail after the last `src`. + self.emit_chunk(&source, cursor, source.len(), file_index); + if !self.program.source.ends_with('\n') { + // Synthetic newline so subsequent files don't run on; no + // region for it — it didn't come from any input file. + self.program.source.push('\n'); + } + + self.in_progress.remove(path); + self.loaded.insert(path.to_path_buf()); + self.observer.on_parsed(path, reached_via); + Ok(()) + } + + /// Push `source[start..end]` onto the flat source and record a + /// region mapping that byte range back to the file it came from. + fn emit_chunk( + &mut self, + source: &str, + start: usize, + end: usize, + file_index: u32, + ) { + if start >= end { + return; + } + let flat_start = self.program.source.len() as u32; + self.program.source.push_str(&source[start..end]); + let flat_end = self.program.source.len() as u32; + self.program.regions.push(SourceRegion { + flat_start, + flat_end, + file_index, + file_offset: start as u32, + }); + } +} + +fn line_of(source: &str, byte: u32) -> u32 { + source[..(byte as usize).min(source.len())] + .bytes() + .filter(|b| *b == b'\n') + .count() as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn workspace() -> tempfile::TempDir { + tempfile::tempdir().unwrap() + } + + #[test] + fn loads_a_single_file_unchanged() { + let dir = workspace(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "puts hi\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + assert_eq!(prog.source.trim(), "puts hi"); + assert_eq!(prog.files.len(), 1); + } + + #[test] + fn load_source_splices_in_memory_entry_and_resolves_named_dep() { + // The dep lives on disk under a cache root; the entry is + // supplied in memory and never written — mirroring the + // `src @vw` + `vw::configure_ip` pre-pass. + let dep = workspace(); + fs::write(dep.path().join("module.htcl"), "proc vw::go {} {}\n") + .unwrap(); + let resolver = Resolver::new().with_dep("vw", dep.path().to_path_buf()); + // Synthetic entry path inside a (real) workspace dir, but with + // no file behind it. + let ws = workspace(); + let entry = ws.path().join(".vw-configure-ip.htcl"); + let prog = load_source("src @vw\nvw::go\n", &entry, &resolver).unwrap(); + // Dep content spliced first, entry's own command last. + assert_eq!(prog.source, "proc vw::go {} {}\nvw::go\n"); + // Entry (in-memory) + the dep file. + assert_eq!(prog.files.len(), 2); + assert_eq!(prog.files[0].path, entry); + assert_eq!(prog.files[0].mtime, None, "synthetic entry has no mtime"); + } + + #[test] + fn imports_local_file_and_drops_src_statement() { + let dir = workspace(); + fs::write(dir.path().join("lib.htcl"), "proc f {} { puts hi }\n") + .unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src lib\nf\n").unwrap(); + + let prog = load(&entry, &Resolver::new()).unwrap(); + // Imported content first, no `src` statement, then the importer. + assert_eq!( + prog.source, "proc f {} { puts hi }\nf\n", + "actual: {:?}", + prog.source + ); + assert_eq!(prog.files.len(), 2); + } + + #[test] + fn idempotent_across_diamond_imports() { + // main → a, b ; a → c ; b → c — c must load exactly once. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src c\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + let prog = load(&entry, &Resolver::new()).unwrap(); + let occurrences = prog.source.matches("proc c {}").count(); + assert_eq!(occurrences, 1, "c loaded multiple times: {}", prog.source); + } + + #[test] + fn cycle_does_not_loop_forever() { + let dir = workspace(); + fs::write(dir.path().join("a.htcl"), "src b\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src a\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + assert!(prog.source.contains("proc a")); + assert!(prog.source.contains("proc b")); + } + + #[test] + fn named_dependency_resolves_through_the_cache() { + let dir = workspace(); + let dep_root = dir.path().join("cache").join("xilinx-ip-deadbeef"); + fs::create_dir_all(&dep_root).unwrap(); + fs::write(dep_root.join("cpm5.htcl"), "proc create_cpm5 {} {}\n") + .unwrap(); + let resolver = Resolver::new().with_dep("xilinx-ip", dep_root); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src @xilinx-ip/cpm5\ncreate_cpm5\n").unwrap(); + let prog = load(&entry, &resolver).unwrap(); + assert!(prog.source.contains("proc create_cpm5")); + assert!(prog.source.contains("\ncreate_cpm5\n")); + } + + #[test] + fn observer_fires_in_dependency_order() { + // entry → a → c ; entry → b + // Expect: source a, parse a (after source c, parse c), + // source b, parse b, parse entry. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "proc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + #[derive(Default)] + struct Recorder { + events: Vec, + } + impl LoadObserver for Recorder { + fn on_source(&mut self, raw: &str, _resolved: &Path) { + self.events.push(format!("source {raw}")); + } + fn on_parsed(&mut self, file: &Path, raw: Option<&str>) { + let label = match raw { + Some(r) => r.to_string(), + None => file + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("?") + .to_string(), + }; + self.events.push(format!("parse {label}")); + } + } + + let mut rec = Recorder::default(); + load_with_observer(&entry, &Resolver::new(), &mut rec).unwrap(); + assert_eq!( + rec.events, + vec![ + "source a", + "source c", + "parse c", + "parse a", + "source b", + "parse b", + "parse main", + ] + ); + } + + #[test] + fn observer_suppresses_source_for_already_loaded_imports() { + // Diamond: main → a → c ; main → b → c. `c` is encountered + // twice via `src` but only loaded once, so "Sourcing c" + // should fire exactly once. + let dir = workspace(); + fs::write(dir.path().join("c.htcl"), "proc c {} {}\n").unwrap(); + fs::write(dir.path().join("a.htcl"), "src c\nproc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "src c\nproc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nsrc b\n").unwrap(); + + #[derive(Default)] + struct Counter { + source_c: usize, + parse_c: usize, + } + impl LoadObserver for Counter { + fn on_source(&mut self, raw: &str, _resolved: &Path) { + if raw == "c" { + self.source_c += 1; + } + } + fn on_parsed(&mut self, file: &Path, _raw: Option<&str>) { + if file.file_stem().and_then(|s| s.to_str()) == Some("c") { + self.parse_c += 1; + } + } + } + let mut counter = Counter::default(); + load_with_observer(&entry, &Resolver::new(), &mut counter).unwrap(); + assert_eq!(counter.source_c, 1); + assert_eq!(counter.parse_c, 1); + } + + #[test] + fn regions_map_each_byte_back_to_its_originating_file() { + // entry uses `set` from one local file and `puts` from another. + let dir = workspace(); + fs::write(dir.path().join("a.htcl"), "proc a {} {}\n").unwrap(); + fs::write(dir.path().join("b.htcl"), "proc b {} {}\n").unwrap(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src a\nputs hello\nsrc b\nputs done\n").unwrap(); + let prog = load(&entry, &Resolver::new()).unwrap(); + + // Pick a byte in the middle of `puts hello` — should map back + // to the entry file (main.htcl). + let puts_hello_at_flat = + prog.source.find("puts hello").expect("puts hello in flat") as u32; + let (idx, file_offset) = + prog.locate(puts_hello_at_flat).expect("locate puts hello"); + assert_eq!( + prog.files[idx].path.file_name().and_then(|s| s.to_str()), + Some("main.htcl") + ); + // In main.htcl the line `puts hello` sits right after `src a\n`, + // so file_offset is at byte 6 (`s`=0,1,2,r=3,c=4,a=5,\n=6). + // Actually 'src a\n' = 6 bytes (s,r,c,space,a,\n), so puts starts at 6. + assert_eq!(file_offset, 6); + + // Pick a byte in the middle of `proc a` — should map to a.htcl. + let proc_a_at_flat = + prog.source.find("proc a").expect("proc a in flat") as u32; + let (idx_a, _) = prog.locate(proc_a_at_flat).expect("locate proc a"); + assert_eq!( + prog.files[idx_a].path.file_name().and_then(|s| s.to_str()), + Some("a.htcl") + ); + } + + #[test] + fn unknown_dep_surfaces_helpful_error() { + let dir = workspace(); + let entry = dir.path().join("main.htcl"); + fs::write(&entry, "src @nope/cpm5\n").unwrap(); + let err = load(&entry, &Resolver::new()).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("unknown dependency"), "{msg}"); + } +} diff --git a/vw-htcl/src/lower.rs b/vw-htcl/src/lower.rs new file mode 100644 index 0000000..3d7edf4 --- /dev/null +++ b/vw-htcl/src/lower.rs @@ -0,0 +1,929 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lower htcl to plain Tcl for the EDA backend. +//! +//! Phase 2 lowering: +//! +//! - Structured `proc` declarations emit `proc name {arg1 arg2 ...} +//! body`, where the arg list is the declared canonical order with +//! no attributes (Vivado's Tcl doesn't understand `@default` etc.). +//! - Call sites to a known structured proc rewrite their `-flag +//! value` form to a positional list in the canonical order, with +//! defaults filled in for omitted args. +//! - Everything else (comments, unknown commands, calls to commands +//! without a structured signature) passes through verbatim. +//! +//! Limitation: only top-level proc declarations and top-level call +//! sites are lowered. Calls *inside* a proc body are not rewritten — +//! the body text is shipped as-is. Phase 3+ will recursively lower +//! nested commands once we have static analysis of proc bodies. + +use std::collections::HashMap; + +use crate::ast::{ + Command, CommandKind, Document, NamespaceEval, Proc, ProcSignature, Stmt, + Word, WordForm, WordPart, +}; +use crate::line_index::LineIndex; + +pub type SignatureTable<'a> = HashMap; + +/// Empty putr rewrite map used as the default when a caller +/// doesn't have one. See [`lower_command_with_putr`] for the +/// keyed-lookup semantics. +fn empty_putr_map() -> &'static crate::putr::RewriteMap { + use std::sync::OnceLock; + static EMPTY: OnceLock = OnceLock::new(); + EMPTY.get_or_init(crate::putr::RewriteMap::new) +} + +/// Walk `doc` and collect every proc's signature — top-level and +/// nested inside `namespace eval` blocks. Namespaced procs register +/// under their qualified name (`::`), matching the +/// signature table the validator builds so call-site lowering works +/// uniformly for both shapes. +pub fn signature_table(doc: &Document) -> SignatureTable<'_> { + let mut table = HashMap::new(); + collect_into(&doc.stmts, "", &mut table); + table +} + +fn collect_into<'a>( + stmts: &'a [Stmt], + prefix: &str, + table: &mut SignatureTable<'a>, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + table.insert(qualified, sig); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + collect_into(&ns.body, &nested, table); + } + _ => {} + } + } +} + +/// Lower one top-level command into its Tcl equivalent for the EDA +/// backend. See [`lower_command_with_putr`] for the variant that +/// takes a `putr` rewrite map; callers with no putr calls (or who +/// don't care) can invoke this simpler form. +/// +/// Builds a fresh [`LineIndex`] every call — fine for one-off +/// uses (tests, single-statement lowering) but pathological when +/// looped over a document with thousands of top-level procs. Bulk +/// callers should build the index once and use +/// [`lower_command_with_putr_and_index`] to skip the per-call +/// rebuild — a 19MB flat document has 1000× the newline-scan +/// cost of a single-statement fragment. +pub fn lower_command( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, +) -> String { + let line_index = LineIndex::new(source); + lower_command_with_putr_and_index( + cmd, + source, + table, + empty_putr_map(), + &line_index, + ) +} + +/// Lower one top-level command, consulting `putr_map` first: when +/// `cmd.span` is a key in the map, the map's replacement Tcl is +/// used verbatim in place of the standard lowering. This is how +/// `putr $x` becomes `puts [T::repr -v $x]` at emit time without +/// mutating the source string. +/// +/// Recurses into proc bodies / namespace-eval bodies / cmd-subst +/// bodies with the same `putr_map`, so `putr` calls buried inside +/// any of those still get the rewrite. +pub fn lower_command_with_putr( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, +) -> String { + let line_index = LineIndex::new(source); + lower_command_with_putr_and_index(cmd, source, table, putr_map, &line_index) +} + +/// Bulk-friendly variant of [`lower_command_with_putr`] that +/// accepts a pre-built [`LineIndex`] instead of constructing one +/// per call. Callers looping over thousands of top-level +/// statements MUST use this form — every `lower_proc_decl` +/// consult needs line-of-offset lookups, and rebuilding the +/// index over a 19MB flat source per proc was the O(procs × +/// source_len) accidental quadratic that made auto-loading a +/// large `.htcl` take minutes. +pub fn lower_command_with_putr_and_index( + cmd: &Command, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + // Fast path: if this command IS a putr rewrite target, emit + // the replacement verbatim. No further descent needed — the + // replacement is a complete Tcl expression. + if let Some(replacement) = putr_map.get(&cmd.span) { + return replacement.clone(); + } + match &cmd.kind { + CommandKind::Proc(proc) => { + lower_proc_decl(proc, source, table, putr_map, line_index) + } + CommandKind::NamespaceEval(ns) => { + lower_namespace_eval(ns, source, table, putr_map, line_index) + } + // `src` is a module import; by the time we lower we expect the + // [`crate::loader`] flatten pass to have already inlined every + // import's contents and dropped the `src` statements. Anything + // that slips through here we render as a no-op comment so the + // emitted Tcl is still well-formed. + CommandKind::Src(import) => { + let path = import.path.as_deref().unwrap_or(""); + format!("# vw: unresolved `src {path}` — loader bypass") + } + // Newtype declarations are compile-time only — they feed the + // analyzer / printer machinery but ship nothing to Vivado. + // Drop entirely (empty Tcl, no whitespace). + CommandKind::TypeDecl(_) => String::new(), + // Enum declarations are also compile-time-only at this + // layer — the codegen path (vw-htcl/src/repr.rs) emits the + // auto-generated `namespace eval ` block separately + // through the same wrap-with-repr pipeline used for the + // primitive prelude. The decl itself ships nothing. + CommandKind::EnumDecl(_) => String::new(), + _ => { + // Verbatim, but reconstructed word-by-word so that any + // `[ … ]` substitution inside the command gets its own + // commands lowered through the same pipeline (extern + // rewrites, multi-line bracket flattening). + // + // No keyword→positional rewrite here: htcl is keyword- + // only at the call site, and the rewrite from `-flag + // value` pairs to local variables happens at runtime + // in the wrapper's `::vw::kwargs $args { ... }` prelude + // (emitted by `lower_proc_decl`). That lets call sites + // anywhere — top-level, inside a proc body, inside a + // `[ ... ]`, inside an `eval` — work uniformly without + // the lowerer needing to see every call site. + lower_words(&cmd.words, source, table, putr_map, line_index) + } + } +} + +/// Lower a `namespace eval` block: recurse on each inner statement +/// (so inner proc declarations get their htcl attributes stripped +/// and gain the `::vw::kwargs` runtime prelude) and wrap the +/// result in `namespace eval { ... }`. Output is a single +/// Tcl-valid string the EDA backend can `eval` directly. +fn lower_namespace_eval( + ns: &NamespaceEval, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let name = ns.name.as_deref().unwrap_or(""); + let mut body = String::new(); + for stmt in &ns.body { + let Stmt::Command(cmd) = stmt else { continue }; + let line = lower_command_with_putr_and_index( + cmd, source, table, putr_map, line_index, + ); + if !line.is_empty() { + body.push_str(&line); + body.push('\n'); + } + } + format!("namespace eval {name} {{\n{body}}}") +} + +/// Lower a `proc` declaration into a Tcl proc whose runtime +/// signature is `args` (variadic). The first line of the body is a +/// generated `::vw::kwargs $args { name default ... }` call that +/// parses the caller's `-flag value` pairs into local variables +/// matching the declared parameter names — defaults applied where +/// the caller didn't supply a flag. After the prelude the original +/// body runs unchanged, using `$name`, `$dir`, etc. just as if +/// they were standard Tcl parameters. +/// +/// Why this shape: htcl is keyword-only at the call site. Doing +/// the parse at runtime (in the wrapper) means every call site +/// works the same — top-level, inside a proc body, inside a +/// `[ ... ]` substitution, inside an `eval`. The previous +/// architecture rewrote `-flag value` → positional at compile +/// time, but only for top-level calls the lowerer could see; calls +/// inside proc bodies stayed verbatim and broke at runtime against +/// a positional-only wrapper proc. +/// +/// Procs without a parsed signature (parser couldn't extract one +/// from the args list, e.g. mid-edit syntax error) pass through as +/// plain Tcl: `proc name { } { }`. The +/// `::vw::kwargs` prelude is only emitted when we know what +/// parameters to declare. +fn lower_proc_decl( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + lower_proc_decl_with_name_and_index( + proc, source, table, None, putr_map, line_index, + ) +} + +/// Like [`lower_proc_decl`] but uses `name_override` as the emitted +/// proc name instead of `proc.name`. Used by the REPL when lowering +/// an enum-overload specialization under its mangled name +/// (`____`) — the source name on the parsed proc +/// is the user-visible public name (`handle_prop`), but the +/// dispatcher needs the specialization to live under its mangled +/// alias so the runtime switch can find it. +pub fn lower_proc_decl_with_name( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + name_override: Option<&str>, + putr_map: &crate::putr::RewriteMap, +) -> String { + let line_index = LineIndex::new(source); + lower_proc_decl_with_name_and_index( + proc, + source, + table, + name_override, + putr_map, + &line_index, + ) +} + +/// Bulk-friendly variant: takes a pre-built [`LineIndex`] instead +/// of constructing one over the entire source per call. The old +/// unindexed form was the O(procs × source_len) hotspot that made +/// `prepare` for a 19MB flat document take ~85s. +pub fn lower_proc_decl_with_name_and_index( + proc: &Proc, + source: &str, + table: &SignatureTable<'_>, + name_override: Option<&str>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let name = name_override.or(proc.name.as_deref()).unwrap_or(""); + // Re-emit the body by walking its parsed statements rather + // than slicing raw text. This is what gives htcl's "newlines + // inside `[ … ]` are whitespace" semantics inside proc bodies + // too — verbatim slicing leaves Tcl to interpret the + // newlines as command separators, which silently splits a + // multi-line `set x [ foo \n -a 1 \n -b 2 \n]` into four + // separate calls and drops every flag arg. + // + // Critically, we pad the emitted body with blank lines so + // each lowered statement lands on the SAME line it occupied + // in the source. Tcl's `info frame` reports body lines + // relative to the script text it was given — without padding, + // collapsing a 5-line `[ ... ]` to one line shifts every + // subsequent statement upward and the stack trace's + // "line N in proc X" ends up pointing at unrelated source + // lines. With padding, Tcl's body line N == source body + // line N == `body_start_file_line + N - 1`, which is what + // the REPL's `ProcLocation::resolve_body_line` already + // assumes. + let body_open_line = line_index.position(proc.body_span.start).line; // 0-based + let body = if proc.body.is_empty() { + proc.body_span.slice(source).to_string() + } else { + let mut out = String::new(); + // First emitted body line corresponds to one line after + // the line containing `{`. We track 0-based file lines + // throughout. + let mut cur_line = body_open_line + 1; + for stmt in &proc.body { + let Stmt::Command(cmd) = stmt else { continue }; + let stmt_line = line_index.position(cmd.span.start).line; + while cur_line < stmt_line { + out.push('\n'); + cur_line += 1; + } + let line = lower_command_with_putr_and_index( + cmd, source, table, putr_map, line_index, + ); + if line.is_empty() { + continue; + } + out.push_str(&line); + out.push('\n'); + cur_line += 1 + line.matches('\n').count() as u32; + } + out + }; + let Some(sig) = proc.signature.as_ref() else { + // Couldn't parse a structured signature — emit the proc + // verbatim. Tcl will accept it if the raw arg text is + // valid Tcl; otherwise the user already has a parse-error + // diagnostic from the upstream parser. + let args_list = proc.args_span.slice(source); + return format!("proc {name} {{{args_list}}} {{{body}}}"); + }; + let sig_dict = build_kwargs_sig_dict(sig); + // Put `::vw::kwargs` on the SAME line as the opening `{` so + // it doesn't eat the first source line of the body and shift + // subsequent statements. Tcl treats "the line containing `{`" + // as body line 1 — putting the kwargs preamble there means + // body line 2 onward maps 1:1 to source lines, matching + // what the padding loop above produced. + format!( + "proc {name} {{args}} {{ ::vw::kwargs $args {{{sig_dict}}}\n{body}}}" + ) +} + +/// Render the parameter list as a flat `name default name default +/// ...` Tcl dict for [`::vw::kwargs`] to consume. The default for +/// an arg without `@default` is the empty string `""` — at which +/// point the validator has already complained at compile time +/// about missing required args. Quote each default through +/// [`AttributeValue::to_tcl_literal`] so integers, idents, and +/// strings all round-trip correctly. +fn build_kwargs_sig_dict(sig: &ProcSignature) -> String { + let mut out = String::new(); + for (i, arg) in sig.args.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(&arg.name); + out.push(' '); + let default = arg + .attribute("default") + .and_then(|attr| attr.values.first()) + .map(|v| v.to_tcl_literal()) + .unwrap_or_else(|| "\"\"".to_string()); + out.push_str(&default); + } + out +} + +/// The syntactic prefix that marks a call to a runtime-Tcl proc +/// (an "extern") rather than an htcl-defined proc. Anywhere in +/// lowered text, `extern::name` rewrites to a mangled Tcl symbol +/// the lowering's prelude has aliased to the underlying proc. +pub const EXTERN_PREFIX: &str = "extern::"; + +/// Result of [`rewrite_externs`]: the lowered text with every +/// `extern::name` reference replaced by its mangled Tcl form, plus +/// the deduplicated set of external names that were referenced. +/// Callers feed `names` to [`extern_rename_prelude`] to build the +/// one-time setup that exposes each extern at its mangled name. +#[derive(Clone, Debug)] +pub struct ExternRewrite { + pub text: String, + pub names: Vec, +} + +/// Rewrite every `extern::` in `text` to `::` — the +/// Tcl-absolute form that anchors the lookup at the global +/// namespace. Returns the rewritten text plus the unique, sorted +/// set of names seen. +/// +/// Anchoring at `::` matters because htcl wrappers live inside +/// `namespace eval vivado { … }`. Inside that namespace a bare +/// `create_project` resolution searches the *current* namespace +/// first and finds `vivado::create_project` (the wrapper itself!) — +/// infinite recursion. The leading `::` skips the current- +/// namespace search and goes straight to the global, where the +/// unshadowed Vivado native lives. +/// +/// The rewrite is text-level, not AST-level. Proc bodies lower +/// as raw text, and a textual pass cleanly catches calls at any +/// nesting depth — inside `[ … ]`, inside multi-arm +/// `if {…} { … extern::foo … } else { … }`, etc. Word-boundary +/// detection on the leading side prevents `not_extern::foo` from +/// triggering; the trailing identifier is parsed greedily so +/// `extern::a::b::c` rewrites as one unit (→ `::a::b::c`). +pub fn rewrite_externs(text: &str) -> ExternRewrite { + let mut out = String::with_capacity(text.len()); + let mut names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if i + EXTERN_PREFIX.len() <= bytes.len() + && &bytes[i..i + EXTERN_PREFIX.len()] == EXTERN_PREFIX.as_bytes() + && (i == 0 || !is_extern_ident_byte(bytes[i - 1])) + { + let name_start = i + EXTERN_PREFIX.len(); + let name_end = scan_extern_name_end(bytes, name_start); + if name_end > name_start { + let name = &text[name_start..name_end]; + // Leading `::` makes the lookup absolute (global + // namespace) — necessary inside `namespace eval + // vivado { … }` so the wrapper body doesn't + // recurse on itself. + out.push_str("::"); + out.push_str(name); + names.insert(name.to_string()); + i = name_end; + continue; + } + } + let ch_end = next_char_boundary(text, i); + out.push_str(&text[i..ch_end]); + i = ch_end; + } + ExternRewrite { + text: out, + names: names.into_iter().collect(), + } +} + +fn is_extern_ident_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' +} + +fn scan_extern_name_end(bytes: &[u8], start: usize) -> usize { + let mut i = start; + while i < bytes.len() { + if is_extern_ident_byte(bytes[i]) { + i += 1; + } else if bytes[i] == b':' + && bytes.get(i + 1).copied() == Some(b':') + && bytes.get(i + 2).copied().is_some_and(is_extern_ident_byte) + { + i += 2; + } else { + break; + } + } + i +} + +fn next_char_boundary(s: &str, start: usize) -> usize { + let mut end = start + 1; + while end < s.len() && !s.is_char_boundary(end) { + end += 1; + } + end +} + +/// Historically emitted a rename prelude that aliased each Vivado +/// native to a mangled name so wrappers could forward to the +/// underlying proc without recursing on themselves. With wrappers +/// now living in the `vivado::` namespace and no longer shadowing +/// the globals they wrap, no rename is needed — `extern::foo` +/// just rewrites to bare `foo`, which Tcl resolves to the global +/// native. Kept as a public symbol so callers don't have to track +/// the layering change; returns the empty string. +pub fn extern_rename_prelude(_names: &[String]) -> String { + String::new() +} + +/// True when `call_name` is the explicit `extern::…` form — used +/// by the validator to skip the unknown-call check for these +/// deliberately-external invocations. +pub fn is_extern_call(call_name: &str) -> bool { + call_name.starts_with(EXTERN_PREFIX) +} + +/// Reconstruct a command's words as lowered Tcl text. Splits the +/// problem along the AST's natural boundaries so each piece is +/// handled by the right rules: +/// +/// - Bare and quoted words are rebuilt part-by-part. Plain text, +/// `$var` references, and `\x` escapes go through verbatim; +/// `[ … ]` substitutions recurse into the lowering pipeline so +/// keyword → positional rewriting applies to calls *inside* a +/// `set proj [ create_project … ]`, and multi-line bracket +/// bodies collapse to one Tcl statement by construction. +/// - Braced words are literal text — Tcl never substitutes inside +/// `{ … }`, so the parser doesn't even surface `CmdSubst` parts +/// for them; we ship them as raw source. +fn lower_words( + words: &[Word], + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + // Preserve source-level adjacency between consecutive words. + // The parser splits `{*}$var` into two AST words ({*} as a + // braced "*", $var as a bare word) but their source spans + // touch — Tcl reads them as the expand-prefix operator. + // Joining with a literal space would force `{*} $var`, which + // Tcl reinterprets as a literal-`*`-arg followed by `$var`. + // Checking adjacency keeps the no-space form for `{*}$var` + // while still spacing genuinely-whitespace-separated words. + let mut out = String::new(); + for (i, w) in words.iter().enumerate() { + if i > 0 { + let prev_end = words[i - 1].span.end; + if w.span.start > prev_end { + out.push(' '); + } + } + out.push_str(&lower_word(w, source, table, putr_map, line_index)); + } + out +} + +fn lower_word( + word: &Word, + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + match word.form { + WordForm::Bare => { + lower_word_parts(&word.parts, source, table, putr_map, line_index) + } + WordForm::Quoted => { + let inner = lower_word_parts( + &word.parts, + source, + table, + putr_map, + line_index, + ); + format!("\"{inner}\"") + } + WordForm::Braced => word.span.slice(source).to_string(), + } +} + +fn lower_word_parts( + parts: &[WordPart], + source: &str, + table: &SignatureTable<'_>, + putr_map: &crate::putr::RewriteMap, + line_index: &LineIndex, +) -> String { + let mut out = String::new(); + for part in parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::VarRef { name, braced, .. } => { + // Preserve the source's braced form. Emitting a + // plain `$name` where the source had `${name}` + // breaks interpolations like `"${ip}_wrapper.vhd"`: + // Tcl reads `$ip_wrapper` as one greedy ident + // and errors with "no such variable ip_wrapper". + // Preserving the braces also happens to be a + // no-op for typical `$var` refs — we only wrap + // when the source did. + if *braced { + out.push_str("${"); + out.push_str(name); + out.push('}'); + } else { + out.push('$'); + out.push_str(name); + } + } + WordPart::Escape { value, .. } => { + out.push('\\'); + out.push(*value); + } + WordPart::CmdSubst { body, .. } => { + let lowered: Vec = body + .iter() + .filter_map(|s| match s { + Stmt::Command(c) => { + Some(lower_command_with_putr_and_index( + c, source, table, putr_map, line_index, + )) + } + _ => None, + }) + .filter(|s| !s.trim().is_empty()) + .collect(); + out.push('['); + out.push_str(&lowered.join("; ")); + out.push(']'); + } + } + } + out +} + +/// Helper retained for symmetry with future analyzers that want to +/// inspect a word's literal form without re-walking its parts. +#[allow(dead_code)] +fn word_text(word: &Word) -> Option { + let mut out = String::new(); + for part in &word.parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::Escape { value, .. } => out.push(*value), + _ => return None, + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn lowered(src: &str) -> Vec { + let parsed = parse(src); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let table = signature_table(&parsed.document); + parsed + .document + .stmts + .iter() + .filter_map(|s| match s { + Stmt::Command(c) => Some(lower_command(c, src, &table)), + _ => None, + }) + .collect() + } + + #[test] + fn proc_decl_emits_kwargs_prelude() { + // Every htcl proc lowers to `proc name {args} { ::vw::kwargs + // ... ; body }` — the runtime helper parses the caller's + // `-flag value` pairs into local variables matching the + // declared param names, with defaults applied where the + // caller didn't supply a flag. Body text passes through + // unchanged. + let src = "proc f {\n @default(0) a\n @default(1) b\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].starts_with("proc f {args} {"), + "wrong arg-list form: {}", + out[0] + ); + assert!( + out[0].contains("::vw::kwargs $args {a 0 b 1}"), + "missing or wrong kwargs prelude: {}", + out[0] + ); + assert!(out[0].contains("puts hi"), "lost body: {}", out[0]); + } + + #[test] + fn call_with_flags_ships_verbatim() { + // No more compile-time keyword→positional rewrite. The call + // ships as the user typed it; the wrapper proc parses the + // keywords at runtime via its kwargs prelude. + let src = "proc f {\n a\n b\n} { puts hi }\nf -b 22 -a 11\n"; + let out = lowered(src); + assert_eq!(out[1], "f -b 22 -a 11"); + } + + #[test] + fn call_with_omitted_arg_ships_verbatim() { + // The wrapper's default is wired in at runtime by + // ::vw::kwargs; the call site doesn't need to fill it. + let src = "proc f {\n @default(7) a\n b\n} { puts hi }\nf -b 22\n"; + let out = lowered(src); + assert_eq!(out[1], "f -b 22"); + } + + #[test] + fn inner_call_inside_brackets_ships_verbatim() { + // What this test used to assert (`[make xc foo]` — + // keyword→positional rewrite for the inner call) is no + // longer the architecture. The inner call ships verbatim; + // the wrapper parses `-part`/`-name` at runtime. The only + // transformation we still apply is multi-line bracket + // flattening. + let src = "proc make { + @default(\"\") part + name +} { puts ok } +set proj [ + make + -part xc + -name foo +] +"; + let out = lowered(src); + assert_eq!(out.len(), 2, "{:?}", out); + let set_line = &out[1]; + // Inner call stays keyword-form: `make -part xc -name foo`. + assert!( + set_line.contains("[make -part xc -name foo]"), + "inner call should ship verbatim; got: {set_line}" + ); + // Multi-line bracket body still collapses to one line. + assert!( + !set_line.contains('\n'), + "expected single line; got: {set_line:?}" + ); + } + + #[test] + fn call_inside_proc_body_ships_verbatim() { + // Regression guard for the create_bd_design bug: a + // keyword-form call to a known wrapper, nested inside + // another proc's body, must NOT be rewritten. In the old + // architecture the lowerer only saw top-level call sites + // and silently failed to translate this one, so at runtime + // Tcl handed `-name cips` to a positional-only wrapper + // proc and errored "wrong # args". Now the wrapper parses + // keywords at runtime, so we just ship the call as-is. + let src = "proc create_bd_design { @default(\"\") name } { puts ok }\n\ + proc configure_cips {} {\n \ + create_bd_design -name cips\n\ + }\n"; + let out = lowered(src); + // The configure_cips proc decl is the second statement. + // Its body should still contain the keyword-form call — + // we don't touch it at compile time. + assert!( + out[1].contains("create_bd_design -name cips"), + "call inside proc body should ship verbatim; got:\n{}", + out[1] + ); + } + + #[test] + fn proc_with_no_default_emits_empty_string_default() { + // An htcl arg without `@default` is implicitly required — + // the validator catches a missing-flag call at compile + // time. At runtime we still need a placeholder default so + // `::vw::kwargs` doesn't blow up when the variable is + // referenced before the (missing) `-flag` would have set + // it; we use `""` (empty string). + let src = "proc f {\n required_arg\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].contains("::vw::kwargs $args {required_arg \"\"}"), + "wrong default for required arg: {}", + out[0] + ); + } + + #[test] + fn multiline_bracket_substitution_collapses_to_one_line() { + // The exact shape that broke the REPL: an outer call whose + // sole arg is a `[ … ]` substitution spanning multiple + // source lines. Tcl would parse the bracket body as N + // separate commands; we have to flatten the newlines. + let src = "set proj [\n create_project\n -in_memory 1\n -name foo\n]\n"; + let out = lowered(src); + assert_eq!(out.len(), 1); + // No literal newline inside the brackets after lowering. + let between = out[0] + .split_once('[') + .and_then(|(_, rest)| rest.rsplit_once(']')) + .map(|(inner, _)| inner) + .unwrap(); + assert!(!between.contains('\n'), "lowered: {:?}", out[0]); + // The full call must still parse as `set proj [ ... ]`. + assert!(out[0].starts_with("set proj [")); + assert!(out[0].trim_end().ends_with(']')); + } + + #[test] + fn nested_multiline_brackets_all_collapse() { + // `[outer [inner ...] ...]` — newlines inside both layers + // become spaces; the parser sees nested CmdSubst so the + // recursive collection covers both. + let src = + "set x [\n foo\n -a [\n bar\n -b 1\n ]\n]\n"; + let out = lowered(src); + assert!(!out[0].contains('\n'), "lowered: {:?}", out[0]); + } + + #[test] + fn newlines_inside_braced_groups_stay_intact() { + // Inside `{ … }` the brackets are literal, not a + // substitution. The parser doesn't emit a `CmdSubst` for + // them so we must not strip newlines from braced bodies. + let src = "proc f {} {\n puts a\n puts b\n}\n"; + let out = lowered(src); + // The proc-decl lowering builds its own output (not the + // verbatim path), so it preserves body newlines. + assert!(out[0].contains('\n'), "lowered: {:?}", out[0]); + } + + #[test] + fn rewrite_externs_anchors_at_global_namespace() { + let r = rewrite_externs( + "set cmd [list extern::set_property]\n\ + extern::create_project -name foo\n", + ); + // Leading `::` anchors the lookup at Tcl's global + // namespace, which is where unshadowed Vivado natives + // live — necessary so wrapper bodies inside `namespace + // eval vivado { … }` don't recurse on themselves. + assert!(r.text.contains("[list ::set_property]"), "{}", r.text); + assert!(r.text.contains("::create_project -name foo"), "{}", r.text); + assert!(!r.text.contains("extern::"), "{}", r.text); + assert_eq!(r.names, vec!["create_project", "set_property"]); + } + + #[test] + fn rewrite_externs_preserves_namespaced_names() { + let r = rewrite_externs("extern::common::send_msg_id A B C\n"); + // Same anchoring for multi-segment names — + // `::common::send_msg_id` resolves the leading namespace + // search from the global root. + assert!(r.text.contains("::common::send_msg_id A B C"), "{}", r.text); + assert_eq!(r.names, vec!["common::send_msg_id"]); + } + + #[test] + fn rewrite_externs_respects_word_boundary() { + let r = rewrite_externs("set x not_extern::foo\n"); + assert_eq!(r.text, "set x not_extern::foo\n"); + assert!(r.names.is_empty()); + } + + #[test] + fn extern_rename_prelude_is_empty() { + // Wrappers no longer shadow globals (they live in the + // `vivado::` namespace), so the historical rename plumbing + // is unnecessary. The helper still exists for API stability + // but always returns empty. + let p = extern_rename_prelude(&["set_property".to_string()]); + assert!(p.is_empty(), "{p}"); + } + + #[test] + fn is_extern_call_recognizes_prefix() { + assert!(is_extern_call("extern::set_property")); + assert!(is_extern_call("extern::common::send_msg_id")); + assert!(!is_extern_call("set_property")); + assert!(!is_extern_call("not_extern::foo")); + } + + #[test] + fn braced_var_ref_preserves_braces_in_output() { + // Regression: `"${ip}_wrapper.vhd"` in htcl source used to + // lower to `"$ip_wrapper.vhd"`, which Tcl reads as + // `$ip_wrapper` (one greedy identifier) — dereferencing a + // non-existent variable. Preserving the braces is the fix. + let src = "puts \"${ip}_wrapper.vhd\"\n"; + let out = lowered(src); + assert_eq!( + out[0], "puts \"${ip}_wrapper.vhd\"", + "braced form must round-trip", + ); + } + + #[test] + fn bare_var_ref_still_uses_bare_form() { + // The braces are a source-level distinction; unadorned + // `$var` should still emit as `$var`, not `${var}` (Tcl + // handles both but the bare form is the idiomatic one). + let src = "puts \"$ip.bd\"\n"; + let out = lowered(src); + assert_eq!(out[0], "puts \"$ip.bd\""); + } + + #[test] + fn unknown_command_passes_through() { + let src = "puts \"hello $x\"\n"; + let out = lowered(src); + assert_eq!(out[0], "puts \"hello $x\""); + } + + #[test] + fn string_default_quotes_correctly_in_kwargs_sig() { + // Defaults are stamped into the proc's kwargs-prelude sig + // dict, not into the call site. A `@default("hi")` becomes + // the literal `"hi"` (quoted) in the dict — `::vw::kwargs` + // sets `$greeting` to it when the caller omits the flag. + let src = "proc f {\n @default(\"hi\") greeting\n} { puts hi }\n"; + let out = lowered(src); + assert!( + out[0].contains("::vw::kwargs $args {greeting \"hi\"}"), + "default should appear quoted in the sig dict: {}", + out[0] + ); + } +} diff --git a/vw-htcl/src/overload.rs b/vw-htcl/src/overload.rs new file mode 100644 index 0000000..9f5027a --- /dev/null +++ b/vw-htcl/src/overload.rs @@ -0,0 +1,162 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Codegen for the enum-overload dispatcher. +//! +//! When the validator classifies a set of procs sharing a name as +//! a valid enum-overload (see +//! [`crate::validate::build_signature_table_with_overloads`]), the +//! lowerer rewrites each specialization under its mangled name +//! (`____`) and emits a single public dispatcher +//! proc that switches on the tagged value's variant tag and calls +//! the right specialization with the unwrapped payload. +//! +//! Runtime model: +//! +//! ```tcl +//! proc handle_prop {v args} { +//! switch -- [lindex $v 0] { +//! Scalar { return [__handle_prop__Scalar [lindex $v 1] {*}$args] } +//! Nested { return [__handle_prop__Nested [lindex $v 1] {*}$args] } +//! default { error "handle_prop: unknown variant '[lindex $v 0]'" } +//! } +//! } +//! ``` +//! +//! The payload is unwrapped before the specialization runs — the +//! body of `proc handle_prop {v: Property::Scalar} ...` sees `$v` +//! as the bare payload (a `string`), matching Haskell `case` +//! semantics. +//! +//! Empty-payload variants still get `[lindex $v 1]` passed through +//! (it's the empty string for a single-element list) — for those +//! the specialization's body shouldn't reference `$v` and the +//! lowering should ideally drop the arg, but for v1 we pass +//! uniformly for simplicity. + +use crate::ast::OverloadInfo; + +/// Generate the public-name dispatcher proc for an overload set. +/// +/// `tail_arg_names` is the list of tail arg names (after the +/// dispatched first arg). They thread through via `{*}$args` so +/// the public signature stays `{v args}` regardless of arity — +/// keeping the dispatcher uniform across overload sets with +/// different tail shapes. Specializations always receive the +/// payload as their first positional arg, then the tail by +/// position. +pub fn emit_dispatcher(info: &OverloadInfo) -> String { + let mut out = String::new(); + // The dispatcher takes the same kwargs envelope every other + // proc takes (so calls can pass `- `). It + // extracts the dispatched-arg value from the kwargs args list + // and switches on its tag. Specializations receive the + // payload via the same kwargs protocol — `- ` + // — so their bodies bind `$` to the unwrapped payload + // naturally. + let pub_name = &info.public_name; + let arg = &info.dispatch_arg_name; + out.push_str(&format!("proc {pub_name} {{args}} {{\n")); + // Grab the dispatched arg's value from the kwargs args list. + // We look for `- ` pairwise; if the user passes + // positional, well, that's not a supported form for overloaded + // procs in v1 (kwargs-only). + out.push_str(&format!( + " set __vw_disp \"\"\n \ + foreach {{__vw_k __vw_v}} $args {{\n \ + if {{$__vw_k eq \"-{arg}\"}} {{ set __vw_disp $__vw_v; break }}\n \ + }}\n" + )); + out.push_str(" switch -- [lindex $__vw_disp 0] {\n"); + for v in &info.variants { + // Build a new args list with the dispatched-arg's value + // replaced by the unwrapped payload, then forward the full + // args list (including any tail args we pass through for + // future multi-arg overload support) to the specialization. + out.push_str(&format!( + " {variant} {{\n \ + set __vw_new [list]\n \ + foreach {{__vw_k __vw_v}} $args {{\n \ + if {{$__vw_k eq \"-{arg}\"}} {{\n \ + lappend __vw_new $__vw_k [lindex $__vw_v 1]\n \ + }} else {{\n \ + lappend __vw_new $__vw_k $__vw_v\n \ + }}\n \ + }}\n \ + return [{mangled} {{*}}$__vw_new]\n \ + }}\n", + variant = v.variant_name, + mangled = v.mangled_proc_name, + )); + } + out.push_str(&format!( + " default {{ error \"{pub_name}: unknown variant '[lindex $__vw_disp 0]'\" }}\n" + )); + out.push_str(" }\n"); + out.push_str("}\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::{OverloadInfo, OverloadVariant}; + use crate::span::Span; + use crate::validate::mangle_specialization; + + fn info(public: &str, variants: &[&str]) -> OverloadInfo { + OverloadInfo { + public_name: public.into(), + enum_name: "E".into(), + dispatch_arg_name: "v".into(), + variants: variants + .iter() + .map(|v| OverloadVariant { + variant_name: (*v).into(), + mangled_proc_name: mangle_specialization(public, v), + dispatch_arg_span: Span::new(0, 0), + }) + .collect(), + anchor_span: Span::new(0, 0), + } + } + + #[test] + fn dispatcher_two_arms() { + let i = info("handle_prop", &["Scalar", "Nested"]); + let d = emit_dispatcher(&i); + // Dispatcher takes the standard `{args}` kwargs envelope. + assert!(d.contains("proc handle_prop {args}")); + // Walks kwargs to find the `-v ` pair. + assert!(d.contains("if {$__vw_k eq \"-v\"}")); + assert!(d.contains("switch -- [lindex $__vw_disp 0]")); + // Each arm forwards to its mangled specialization with the + // unwrapped payload spliced back into the kwargs list. + assert!(d.contains("Scalar {")); + assert!(d.contains("__handle_prop__Scalar")); + assert!(d.contains("Nested {")); + assert!(d.contains("__handle_prop__Nested")); + assert!(d.contains("default {")); + assert!(d.contains("unknown variant")); + } + + #[test] + fn dispatcher_single_arm() { + let i = info("only_one", &["Solo"]); + let d = emit_dispatcher(&i); + assert!(d.contains("proc only_one {args}")); + assert!(d.contains("__only_one__Solo")); + } + + #[test] + fn dispatcher_includes_default_arm() { + // Future-proof against runtime corruption / unanticipated + // tag values. The validator's exhaustiveness check guards + // the source side; the default arm guards the runtime + // side. + let i = info("foo", &["A", "B"]); + let d = emit_dispatcher(&i); + assert!(d.contains("default { error")); + } +} diff --git a/vw-htcl/src/parser.rs b/vw-htcl/src/parser.rs new file mode 100644 index 0000000..2235430 --- /dev/null +++ b/vw-htcl/src/parser.rs @@ -0,0 +1,2472 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! htcl source parser. +//! +//! Builds a [`Document`] CST plus a list of [`ParseError`]s. The parser +//! is error-tolerant: when a command can't be parsed it is recorded as +//! a [`Stmt::Error`] and the parser resyncs at the next statement +//! boundary (newline or semicolon). This is what makes the same parser +//! usable from the LSP, where input is incomplete by definition. +//! +//! The outer statement loop is hand-rolled (it owns recovery and +//! collects doc comments); inner pieces — words, parts, escapes — +//! drive [`winnow::LocatingSlice`] for position tracking. As the grammar +//! grows past Phase 0 the inner pieces will lean on winnow combinators +//! more heavily. + +use winnow::stream::{Location, Stream}; +use winnow::LocatingSlice; + +use crate::ast::*; +use crate::span::Span; + +type Input<'i> = LocatingSlice<&'i str>; + +#[derive(Clone, Debug)] +pub struct ParseError { + pub message: String, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub struct ParseOutput { + pub document: Document, + pub errors: Vec, +} + +pub fn parse(source: &str) -> ParseOutput { + let mut input = LocatingSlice::new(source); + let mut errors = Vec::new(); + let mut document = + parse_document(&mut input, source, &mut errors, Mode::Toplevel); + populate_procs(&mut document.stmts, source, &mut errors); + ParseOutput { document, errors } +} + +/// Statement-termination mode for the parser. +/// +/// At the top level (and inside proc bodies, which are themselves +/// scripts) a newline ends a command — the historical Tcl rule. Inside +/// a `[ … ]` command substitution we relax that: newlines are +/// whitespace and only `;` (or the closing bracket, which is EOF for +/// the interior parser) terminates a command. That lets a single call +/// span lines without `\` continuations, e.g. +/// +/// ```htcl +/// set x [ +/// create_cpm5_cpm_pcie0 +/// -cell cpm5 +/// -max_link_speed 32.0_GT/s +/// ] +/// ``` +/// +/// Multi-command `[…]` (rare in practice — only the last command's +/// value flows out) still works via explicit `;`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Mode { + Toplevel, + BracketBody, +} + +/// Post-pass over every `proc` — top-level *and* nested — that fills +/// in the structured args [`signature`](crate::ast::Proc::signature) +/// and parses the proc [`body`](crate::ast::Proc::body) into real +/// statements. +/// +/// The body is parsed as a standalone fragment (its braces are +/// already stripped by [`inner_text_span`]); the resulting spans are +/// relative to the fragment, so they're shifted by the body's start +/// offset back into whole-source coordinates before being stored. +/// After shifting they're absolute, which lets the recursion process +/// nested procs against the original `source` uniformly. +pub(crate) fn populate_procs( + stmts: &mut [crate::ast::Stmt], + source: &str, + errors: &mut Vec, +) { + use crate::ast::{CommandKind, Stmt}; + use crate::proc_args::parse_proc_args; + for stmt in stmts.iter_mut() { + let Stmt::Command(cmd) = stmt else { continue }; + + // Walk every word's parts and parse `[…]` command substitution + // interiors into statements. Spans inside the parsed body get + // shifted into whole-source coordinates so downstream analyses + // can navigate them uniformly with top-level commands. + for word in &mut cmd.words { + populate_cmd_subst_parts(&mut word.parts, source, errors); + } + + match &mut cmd.kind { + CommandKind::Proc(proc) => { + let (sig, errs) = parse_proc_args(source, proc.args_span); + errors.extend(errs); + proc.signature = Some(sig); + + // Parse the return-type annotation if the outer + // parse recorded one. We have `source` and the + // error sink here, so this is the right place to + // do it — `classify_command` only recorded the + // (inner, brace-stripped) span. + if let Some(rt_span) = proc.return_type_span { + let text = rt_span.slice(source); + match crate::type_parse::parse(text, rt_span.start) { + Ok(ty) => proc.return_type = Some(ty), + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + // Mirror the return type onto the signature so the + // signature-table-based lookup paths (REPL repr + // formatter, hover) see it without re-walking + // back to the Proc node. + if let Some(sig) = proc.signature.as_mut() { + sig.return_type = proc.return_type.clone(); + } + + let delta = proc.body_span.start; + let body_text = proc.body_span.slice(source); + // Proc bodies are scripts — newlines still terminate + // statements there. + let (mut body_stmts, body_errs) = + parse_fragment(body_text, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + proc.body = body_stmts; + + // Spans are now absolute, so nested procs can be processed + // against the same `source`. + populate_procs(&mut proc.body, source, errors); + } + CommandKind::TypeDecl(td) => { + let text = td.underlying_span.slice(source); + match crate::type_parse::parse(text, td.underlying_span.start) { + Ok(ty) => td.underlying = Some(ty), + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + CommandKind::EnumDecl(ed) => { + let text = ed.body_span.slice(source); + match crate::enum_parse::parse(text, ed.body_span.start) { + Ok(vs) => ed.variants = vs, + Err(e) => errors.push(ParseError { + message: e.message, + span: e.span, + }), + } + } + CommandKind::NamespaceEval(ns) => { + // Same body-recursion as `proc` — the braced body is + // a script fragment, parsed in toplevel mode so + // newlines terminate statements normally. + let delta = ns.body_span.start; + let body_text = ns.body_span.slice(source); + let (mut body_stmts, body_errs) = + parse_fragment(body_text, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + ns.body = body_stmts; + populate_procs(&mut ns.body, source, errors); + } + _ => { + // Generic (unrecognized) command: check the head + // word against the Tcl-builtin control-flow set + // and, when it's one of `foreach` / `for` / + // `while` / `if` / `catch` / `dict for`, + // recursively parse the arg-word(s) that are + // KNOWN to be script bodies. Populates + // `Word::body`; downstream tools (validator, + // putr rewrite, hover, goto, syntax highlight) + // key off that field to descend uniformly with + // top-level statements. + populate_control_flow_bodies(cmd, source, errors); + } + } + } +} + +/// Body-arg positions per builtin. Word-index 0 is the head, so +/// `foreach var list body` puts the body at word 3. `dict for +/// {kv} dict body` is the two-word head form. +fn populate_control_flow_bodies( + cmd: &mut crate::ast::Command, + source: &str, + errors: &mut Vec, +) { + // Word 0: head. Everything below tests it against the known + // set. Non-literal heads (interpolation, subst) fall through + // — we can't statically match them and the analyzer treats + // them as generic. + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + return; + }; + // Collect word-indices whose braced form should be parsed as + // a script. Multiple positions handle `for INIT COND STEP + // BODY` and `if COND BODY [elseif COND BODY]* [else BODY]`. + let body_positions: Vec = match head { + "foreach" => vec![cmd.words.len().saturating_sub(1)], + "while" => vec![2], + "for" => vec![1, 3, 4], // init, step, body — cond is an expr + "catch" => vec![1], + "dict" + // `dict for {kv} DICT BODY` — head is the two-word + // sub-command form. Only recognize the `for` variant + // for body descent; other `dict` sub-commands have no + // script args. + if cmd.words.get(1).and_then(|w| w.as_text()) == Some("for") => { + vec![cmd.words.len().saturating_sub(1)] + } + "if" => { + // `if COND BODY [elseif COND BODY]* [else BODY]`. + // Scan word-by-word: after `if`/`elseif` we skip the + // condition and mark the next word as a body; after + // `else` the very next word is a body. + let mut out = Vec::new(); + let mut i = 1usize; + while i < cmd.words.len() { + let w = cmd.words.get(i).and_then(|w| w.as_text()); + if w == Some("elseif") || i == 1 { + // condition at i (or i+1 for elseif), body at i+1 (or i+2) + let (cond_idx, body_idx) = if w == Some("elseif") { + (i + 1, i + 2) + } else { + (i, i + 1) + }; + let _ = cond_idx; + if body_idx < cmd.words.len() { + out.push(body_idx); + } + i = body_idx + 1; + } else if w == Some("else") { + if i + 1 < cmd.words.len() { + out.push(i + 1); + } + i += 2; + } else { + i += 1; + } + } + out + } + _ => return, + }; + for idx in body_positions { + let Some(word) = cmd.words.get_mut(idx) else { + continue; + }; + if word.form != crate::ast::WordForm::Braced { + // Only descend when the arg is a braced literal — + // an interpolated body (`$body_var`) can't be + // statically parsed. + continue; + } + // Interior text: `span` covers `{...}`; strip the two + // brace bytes. + let interior_start = word.span.start + 1; + let interior_end = word.span.end.saturating_sub(1); + if interior_end <= interior_start { + continue; + } + let interior = &source[interior_start as usize..interior_end as usize]; + let (mut body_stmts, body_errs) = + parse_fragment(interior, Mode::Toplevel); + for stmt in &mut body_stmts { + shift_stmt(stmt, interior_start); + } + for mut err in body_errs { + err.span = err.span.shifted(interior_start); + errors.push(err); + } + populate_procs(&mut body_stmts, source, errors); + word.body = Some(body_stmts); + } +} + +fn populate_cmd_subst_parts( + parts: &mut [WordPart], + source: &str, + errors: &mut Vec, +) { + for part in parts { + let WordPart::CmdSubst { + source: text, + span, + body, + } = part + else { + continue; + }; + // `span` covers the whole `[...]` including the brackets, and + // `text` is the interior; the first interior byte sits at + // `span.start + 1`. + let delta = span.start + 1; + // Bracket-body mode: newlines are whitespace, so multi-line + // calls inside `[ … ]` parse as one command without `\`. + let (mut body_stmts, body_errs) = + parse_fragment(text, Mode::BracketBody); + for s in &mut body_stmts { + shift_stmt(s, delta); + } + for mut err in body_errs { + err.span = err.span.shifted(delta); + errors.push(err); + } + *body = body_stmts; + populate_procs(body, source, errors); + } +} + +/// Parse a fragment of htcl (e.g. a proc body) into statements. Spans +/// are relative to `text`; the caller shifts them into whole-source +/// coordinates. +pub(crate) fn parse_fragment( + text: &str, + mode: Mode, +) -> (Vec, Vec) { + let mut input = LocatingSlice::new(text); + let mut errors = Vec::new(); + let document = parse_document(&mut input, text, &mut errors, mode); + (document.stmts, errors) +} + +pub(crate) fn shift_stmt(stmt: &mut crate::ast::Stmt, delta: u32) { + use crate::ast::Stmt; + match stmt { + Stmt::Command(cmd) => shift_command(cmd, delta), + Stmt::Comment(c) => c.span = c.span.shifted(delta), + Stmt::Error(e) => e.span = e.span.shifted(delta), + } +} + +fn shift_command(cmd: &mut Command, delta: u32) { + cmd.span = cmd.span.shifted(delta); + for word in &mut cmd.words { + shift_word(word, delta); + } + // At this stage nested procs carry only the spans produced by + // `parse_document`; `signature` is still `None` and `body` empty, + // both filled later by the caller's `populate_procs` recursion. + match &mut cmd.kind { + CommandKind::Proc(proc) => { + proc.name_span = proc.name_span.shifted(delta); + proc.args_span = proc.args_span.shifted(delta); + proc.body_span = proc.body_span.shifted(delta); + if let Some(ref mut s) = proc.return_type_span { + *s = s.shifted(delta); + } + // `return_type` is None at this point (parsed later in + // `populate_procs` using the now-absolute span), so + // there's nothing to shift inside it. + } + CommandKind::NamespaceEval(ns) => { + ns.name_span = ns.name_span.shifted(delta); + ns.body_span = ns.body_span.shifted(delta); + } + CommandKind::TypeDecl(td) => { + td.name_span = td.name_span.shifted(delta); + td.underlying_span = td.underlying_span.shifted(delta); + } + CommandKind::EnumDecl(ed) => { + ed.name_span = ed.name_span.shifted(delta); + ed.body_span = ed.body_span.shifted(delta); + // Variants are filled later by `populate_procs` using + // the now-absolute body_span, so there's nothing to + // shift inside them yet. + } + _ => {} + } +} + +fn shift_word(word: &mut Word, delta: u32) { + word.span = word.span.shifted(delta); + for part in &mut word.parts { + let span = match part { + WordPart::Text { span, .. } + | WordPart::VarRef { span, .. } + | WordPart::CmdSubst { span, .. } + | WordPart::Escape { span, .. } => span, + }; + *span = span.shifted(delta); + } +} + +#[derive(Clone, Debug)] +struct InnerError { + message: String, + #[allow(dead_code)] + span: Span, +} + +fn parse_document( + input: &mut Input<'_>, + source: &str, + errors: &mut Vec, + mode: Mode, +) -> Document { + let start = input.location(); + let mut stmts = Vec::new(); + let mut pending_docs: Vec = Vec::new(); + // Span covering all currently-pending `##` lines from the first + // `#` byte to the last line's end. Grows with each new `##` + // encountered; cleared alongside `pending_docs`. Used to seed + // the attached command's `doc_comments_span` so the analyzer + // can answer "is the cursor inside this doc block?" + let mut pending_docs_span: Option = None; + // Attributes at statement position (`@test`, `@test(dedicated-eda)`, + // etc.) attach to the next `proc` command, mirroring + // `pending_docs`. Cleared with a warning if the next command + // isn't a proc. + let mut pending_attrs: Vec = Vec::new(); + + loop { + skip_inline_ws(input, source, mode); + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + // In `BracketBody`, `\n` is whitespace consumed by + // `skip_inline_ws`, so it never reaches this match. + let is_separator = match mode { + Mode::Toplevel => c == '\n' || c == ';', + Mode::BracketBody => c == ';', + }; + match c { + _ if is_separator => { + advance_char(input); + // A statement separator drops any orphan doc comments; + // doc comments only attach to the immediately + // following command. + if matches!(c, ';') { + // semicolons don't break doc attachment within a line + } else { + // Blank line breaks the doc-comment run only if + // the next non-whitespace is itself a blank line. + // For v0 we keep this simple: any `\n` between a + // doc comment and the next command keeps the + // attachment so long as nothing else intervenes. + } + continue; + } + '#' => { + let comment = parse_comment(input, source); + if comment.is_doc { + pending_docs.push(comment.text.clone()); + // Extend the block-span to cover this line. The + // comment's own span starts at its `#` and ends + // at the line's end. + pending_docs_span = Some(match pending_docs_span { + Some(prev) => Span::new(prev.start, comment.span.end), + None => comment.span, + }); + } else { + pending_docs.clear(); + pending_docs_span = None; + } + stmts.push(Stmt::Comment(comment)); + } + '@' => { + // Statement-position attribute — `@name` or + // `@name(v1, v2)`. Accumulates into + // `pending_attrs` and attaches to the next `proc` + // command via the drain in the `Ok(cmd)` arm below. + match parse_top_level_attribute(input, source, errors) { + Some(attr) => pending_attrs.push(attr), + None => { + // Parser already recorded the error and + // resynced. Drop any accumulated attrs so + // a garbage line doesn't attach half a + // block to the next proc. + pending_attrs.clear(); + } + } + } + _ => { + let cmd_start = input.location(); + match parse_command(input, source, mode) { + Ok(mut cmd) => { + cmd.doc_comments = std::mem::take(&mut pending_docs); + cmd.doc_comments_span = pending_docs_span.take(); + // Drain pending attributes into the command: + // procs get them via `Proc.attributes`; any + // other command shape drops them with a + // warning. + if !pending_attrs.is_empty() { + match &mut cmd.kind { + CommandKind::Proc(proc) => { + proc.attributes = + std::mem::take(&mut pending_attrs); + } + _ => { + let first = + pending_attrs.first().unwrap().span; + let last = + pending_attrs.last().unwrap().span; + errors.push(ParseError { + message: "attribute attached to \ + non-proc statement — \ + only `proc` declarations \ + accept attributes here" + .into(), + span: Span::new(first.start, last.end), + }); + pending_attrs.clear(); + } + } + } + stmts.push(Stmt::Command(cmd)); + } + Err(err) => { + pending_docs.clear(); + pending_docs_span = None; + // Resync to the next statement boundary. In + // `BracketBody` only `;` breaks; the surrounding + // `]` is EOF for the interior parser. + while !at_eof(input, source) { + let c = current_char(input, source); + let stop = match mode { + Mode::Toplevel => c == '\n' || c == ';', + Mode::BracketBody => c == ';', + }; + if stop { + break; + } + advance_char(input); + } + let span = Span::new( + cmd_start as u32, + input.location() as u32, + ); + errors.push(ParseError { + message: err.message.clone(), + span, + }); + stmts.push(Stmt::Error(ParseFailure { + message: err.message, + span, + })); + } + } + } + } + } + + Document { + stmts, + span: Span::new(start as u32, input.location() as u32), + } +} + +/// Parse a statement-position attribute — `@name` or +/// `@name(v1, v2, …)`. Attribute values accept the same shapes +/// [`crate::proc_args`]' parser does (int, string, ident), plus +/// kebab-case idents (`dedicated-eda`) for readable multi-word +/// tokens like `@test(dedicated-eda)`. +/// +/// Returns `None` when the leading identifier is missing or the +/// argument list is malformed — errors are appended to `errors` +/// verbatim; the caller drops any accumulated attributes to avoid +/// half-parsed items sticking to the next proc. +fn parse_top_level_attribute( + input: &mut Input<'_>, + source: &str, + errors: &mut Vec, +) -> Option { + let start = input.location() as u32; + advance_char(input); // '@' + let name_start = input.location() as u32; + let name = consume_ident(input, source); + let name_end = input.location() as u32; + if name.is_empty() { + errors.push(ParseError { + message: "expected attribute name after `@`".into(), + span: Span::new(start, input.location() as u32), + }); + return None; + } + let name_span = Span::new(name_start, name_end); + let mut values: Vec = Vec::new(); + if !at_eof(input, source) && current_char(input, source) == '(' { + advance_char(input); + loop { + skip_attr_ws(input, source); + if at_eof(input, source) { + errors.push(ParseError { + message: "unterminated attribute argument list".into(), + span: Span::new(start, input.location() as u32), + }); + break; + } + if current_char(input, source) == ')' { + advance_char(input); + break; + } + match parse_attribute_item(input, source) { + Some(v) => values.push(v), + None => { + errors.push(ParseError { + message: "expected attribute value".into(), + span: Span::new( + input.location() as u32, + input.location() as u32 + 1, + ), + }); + // Resync to comma/whitespace/`)` so the rest + // of the list still parses. + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ',' + || c == ')' + || c == '\n' + || c == ' ' + || c == '\t' + { + break; + } + advance_char(input); + } + } + } + skip_attr_ws(input, source); + if at_eof(input, source) { + continue; + } + // Item separator: optional comma. Whitespace between + // items (already consumed by `skip_attr_ws`) is also + // a valid separator — matches the shape + // `@test(dedicated-eda part=xcvm3358…)` where items + // are space-separated. + if current_char(input, source) == ',' { + advance_char(input); + } + } + } + Some(Attribute { + name, + name_span, + values, + span: Span::new(start, input.location() as u32), + }) +} + +/// Consume `[A-Za-z_][A-Za-z0-9_]*` at the input cursor. +fn consume_ident(input: &mut Input<'_>, source: &str) -> String { + let mut out = String::new(); + let mut first = true; + while !at_eof(input, source) { + let c = current_char(input, source); + let ok = if first { + c.is_alphabetic() || c == '_' + } else { + c.is_alphanumeric() || c == '_' + }; + if !ok { + break; + } + out.push(c); + advance_char(input); + first = false; + } + out +} + +/// Ident with support for internal hyphens (`dedicated-eda`). +/// Consumes `[A-Za-z_]([A-Za-z0-9_-]*[A-Za-z0-9_])?` — a leading +/// alphabetic/underscore, followed by any mix of alphanumeric, +/// underscore, or hyphen, but disallowing a trailing hyphen. Used +/// only for attribute value idents; keeps proc names and +/// everything else on the stricter `consume_ident` rule. +fn consume_kebab_ident(input: &mut Input<'_>, source: &str) -> String { + let mut out = String::new(); + let mut first = true; + while !at_eof(input, source) { + let c = current_char(input, source); + let ok = if first { + c.is_alphabetic() || c == '_' + } else { + c.is_alphanumeric() || c == '_' || c == '-' + }; + if !ok { + break; + } + out.push(c); + advance_char(input); + first = false; + } + // Trim a trailing hyphen — `foo-` isn't a valid identifier and + // rolling it back lets the surrounding parser see the `-` as + // its own token if it wants to. + while out.ends_with('-') { + out.pop(); + // We can't easily un-advance the winnow cursor here, so + // trailing hyphens are consumed but not part of the name. + // In attribute-value context the `-` would then be + // followed by `,` or `)`, and neither position accepts a + // dangling hyphen — the caller's resync handles it. + } + out +} + +/// Skip horizontal whitespace + newlines inside an attribute +/// argument list. Attribute lists can wrap across lines, so we +/// consume `\n` as freely as space/tab. +fn skip_attr_ws(input: &mut Input<'_>, source: &str) { + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + advance_char(input); + } else { + break; + } + } +} + +/// Parse one item in an attribute argument list — either a plain +/// positional value (int, string, ident, kebab-ident) or a +/// `key=value` pair. Wraps `key=value` in [`AttributeValue::Keyed`] +/// so downstream can distinguish. +fn parse_attribute_item( + input: &mut Input<'_>, + source: &str, +) -> Option { + let start = input.location() as u32; + // Peek: if the current position parses as `ident=`, it's a + // keyed item. Otherwise, delegate to positional + // `parse_attribute_value`. We do the peek without committing: + // save the cursor, tentatively consume an ident, check for + // `=`, and either commit or rewind. + // + // Rewind story: `input` here is a winnow `LocatingSlice`; + // its clone captures the internal cursor so the tentative + // parse is undoable. + let saved = *input; + let key_start = input.location() as u32; + let key = consume_ident(input, source); + let is_keyed = !key.is_empty() + && !at_eof(input, source) + && current_char(input, source) == '='; + if !is_keyed { + // Not a `key=value` item — rewind and parse as positional. + *input = saved; + return parse_attribute_value(input, source); + } + let key_span = Span::new(key_start, input.location() as u32); + advance_char(input); // consume '=' + // Value can be any regular attribute value shape. + let value = parse_attribute_value(input, source)?; + let end = input.location() as u32; + Some(AttributeValue::Keyed { + key, + key_span, + value: Box::new(value), + span: Span::new(start, end), + }) +} + +fn parse_attribute_value( + input: &mut Input<'_>, + source: &str, +) -> Option { + let start = input.location() as u32; + if at_eof(input, source) { + return None; + } + let c = current_char(input, source); + if c == '"' { + advance_char(input); + let mut buf = String::new(); + while !at_eof(input, source) && current_char(input, source) != '"' { + if current_char(input, source) == '\\' { + advance_char(input); + if !at_eof(input, source) { + buf.push(current_char(input, source)); + advance_char(input); + } + } else { + buf.push(current_char(input, source)); + advance_char(input); + } + } + if !at_eof(input, source) { + advance_char(input); // closing " + } + Some(AttributeValue::String { + value: buf, + span: Span::new(start, input.location() as u32), + }) + } else if c == '-' || c.is_ascii_digit() { + let mut buf = String::new(); + if c == '-' { + buf.push('-'); + advance_char(input); + } + while !at_eof(input, source) + && current_char(input, source).is_ascii_digit() + { + buf.push(current_char(input, source)); + advance_char(input); + } + buf.parse::() + .ok() + .map(|value| AttributeValue::Integer { + value, + span: Span::new(start, input.location() as u32), + }) + } else if c.is_alphabetic() || c == '_' { + let value = consume_kebab_ident(input, source); + Some(AttributeValue::Ident { + value, + span: Span::new(start, input.location() as u32), + }) + } else { + None + } +} + +fn parse_comment(input: &mut Input<'_>, source: &str) -> Comment { + let start = input.location(); + advance_char(input); // leading `#` + let mut is_doc = false; + if !at_eof(input, source) && current_char(input, source) == '#' { + is_doc = true; + advance_char(input); + } + // Leading single space after `#` / `##` is conventionally part of + // the marker; trim it so callers see the raw comment text. + if !at_eof(input, source) && current_char(input, source) == ' ' { + advance_char(input); + } + let text_start = input.location(); + while !at_eof(input, source) { + let c = current_char(input, source); + if c == '\n' { + break; + } + advance_char(input); + } + let text_end = input.location(); + Comment { + text: source[text_start..text_end].to_string(), + span: Span::new(start as u32, text_end as u32), + is_doc, + } +} + +fn parse_command( + input: &mut Input<'_>, + source: &str, + mode: Mode, +) -> Result { + let start = input.location(); + let mut words = Vec::new(); + loop { + skip_inline_ws(input, source, mode); + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + // Line-continuation on a leading-dash next line: the + // configurator shape `cmd\n -flag val\n -flag val\n` + // parses as one command without needing `\` at every EOL. + // Only triggers mid-command (`!words.is_empty()`) — a `-` + // at the start of a fresh statement stays a new statement, + // even if it doesn't lex as a command name. + if mode == Mode::Toplevel + && c == '\n' + && !words.is_empty() + && next_line_is_flag_continuation(input, source) + { + advance_char(input); + continue; + } + let terminate = match mode { + Mode::Toplevel => c == '\n' || c == ';', + // In bracket-body, only `;` terminates a command — `\n` + // is whitespace consumed by `skip_inline_ws`. + Mode::BracketBody => c == ';', + }; + if terminate { + break; + } + // Inline comment at word-start position (mid-command). The + // configurator idiom for commenting out an arg line — + // + // set cfg [ + // versal_cips::configure + // -enable_reg_interface 1 + // #-intf_parent_pin_list 0 + // ] + // + // — needs the parser to eat the `#-intf_parent_pin_list 0` + // as a comment, otherwise it lands as a word and the + // analyzer flags `expected keyword argument`. Only fires + // MID-command (`!words.is_empty()`) so `#` at + // command-start on the top-level (a real Tcl comment) still + // reaches the outer `parse_document` handler; inside + // brackets `words.is_empty()` at line-start is normal + // because bracket-body has no prior context, but the + // enclosing `[…]` was already parsed as a CmdSubst so any + // `#` INSIDE the subst body's first command *does* have + // words already (the command name). + if c == '#' && !words.is_empty() { + skip_to_end_of_line(input, source); + continue; + } + words.push(parse_word(input, source)?); + } + if words.is_empty() { + return Err(InnerError { + message: "expected command".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let span = Span::new(start as u32, input.location() as u32); + let kind = classify_command(&words); + Ok(Command { + words, + span, + kind, + doc_comments: Vec::new(), + doc_comments_span: None, + }) +} + +fn classify_command(words: &[Word]) -> CommandKind { + let Some(first) = words.first() else { + return CommandKind::Generic; + }; + match first.as_text() { + Some("set") => CommandKind::Set, + Some("src") if words.len() == 2 => { + let path_word = &words[1]; + CommandKind::Src(SrcImport { + path: path_word.as_text().map(String::from), + path_span: path_word.span, + }) + } + Some("proc") if words.len() >= 4 => { + let name_word = &words[1]; + let args_word = &words[2]; + // 5 words = return-type slot present: + // proc NAME { args } TYPE { body } + // 4 words = no return type: + // proc NAME { args } { body } + // (>5 words is treated as 5+ junk; the body is taken + // from words[4] and the rest is silently ignored. + // The return-type slot is parsed in `populate_procs` + // where we have `source` and an error sink — we only + // record the span here.) + let (return_type_span, body_word) = if words.len() >= 5 { + (Some(inner_text_span(&words[3])), &words[4]) + } else { + (None, &words[3]) + }; + let name = name_word.as_text().map(|s| s.to_string()); + CommandKind::Proc(Proc { + name, + name_span: name_word.span, + args_span: inner_text_span(args_word), + body_span: inner_text_span(body_word), + signature: None, + return_type: None, + return_type_span, + body: Vec::new(), + attributes: Vec::new(), + }) + } + // `type NAME = UNDERLYING` newtype declaration. The `=` may + // be its own word (`type T = U`) or fused (`type T=U`) — + // Tcl word splitting is whitespace-driven, so we accept + // either by checking the third word. The underlying type + // is parsed in `populate_procs`'s second pass (same + // rationale as `proc`'s return type). + Some("type") if words.len() >= 3 => { + let name_word = &words[1]; + let underlying_word = + if words.len() >= 4 && words[2].as_text() == Some("=") { + &words[3] + } else { + &words[2] + }; + CommandKind::TypeDecl(crate::ast::TypeDecl { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + underlying: None, + underlying_span: inner_text_span(underlying_word), + }) + } + // `enum NAME = { …variants… }` sum-type declaration. + // Same `=`-may-or-may-not-be-its-own-word convention as + // `type`. The body word is brace-wrapped; its contents + // (the variant list) are parsed in `populate_procs`'s + // second pass, when we have the source + error sink. + Some("enum") if words.len() >= 3 => { + let name_word = &words[1]; + let body_word = + if words.len() >= 4 && words[2].as_text() == Some("=") { + &words[3] + } else { + &words[2] + }; + CommandKind::EnumDecl(crate::ast::EnumDecl { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + variants: Vec::new(), + body_span: inner_text_span(body_word), + }) + } + Some("namespace") + if words.len() >= 4 + && words.get(1).and_then(Word::as_text) == Some("eval") => + { + let name_word = &words[2]; + let body_word = &words[3]; + CommandKind::NamespaceEval(crate::ast::NamespaceEval { + name: name_word.as_text().map(String::from), + name_span: name_word.span, + body_span: inner_text_span(body_word), + body: Vec::new(), + }) + } + _ => CommandKind::Generic, + } +} + +/// For a braced word, return the span of the brace contents (without +/// the braces themselves). For any other word, return its full span. +/// Used so Phase 2's structured-proc reparse and the LSP can point at +/// the parseable interior directly. +fn inner_text_span(word: &Word) -> Span { + if word.form == WordForm::Braced { + if let [WordPart::Text { span, .. }] = word.parts.as_slice() { + return *span; + } + } + word.span +} + +fn parse_word(input: &mut Input<'_>, source: &str) -> Result { + let start = input.location(); + let c = current_char(input, source); + let (form, parts) = match c { + '{' => parse_braced_word(input, source)?, + '"' => parse_quoted_word(input, source)?, + _ => parse_bare_word(input, source)?, + }; + let end = input.location(); + Ok(Word { + form, + parts, + span: Span::new(start as u32, end as u32), + // `body` is set by `populate_procs` for known control-flow + // builtins whose Nth arg is a script body. Initial parse + // leaves it None. + body: None, + }) +} + +fn parse_braced_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let open_cp = input.checkpoint(); + let open = input.location(); + advance_char(input); // { + let inner_start = input.location(); + let mut depth = 1usize; + while !at_eof(input, source) { + let c = current_char(input, source); + match c { + '\\' => { + advance_char(input); + if !at_eof(input, source) { + advance_char(input); + } + } + '{' => { + depth += 1; + advance_char(input); + } + '}' => { + depth -= 1; + if depth == 0 { + let inner_end = input.location(); + advance_char(input); + let text = source[inner_start..inner_end].to_string(); + return Ok(( + WordForm::Braced, + vec![WordPart::Text { + value: text, + span: Span::new( + inner_start as u32, + inner_end as u32, + ), + }], + )); + } + advance_char(input); + } + _ => advance_char(input), + } + } + // Unterminated: rewind to just past the open brace so the outer + // loop's resync can find the next statement boundary instead of + // being stuck at EOF. + input.reset(&open_cp); + advance_char(input); + Err(InnerError { + message: "unterminated brace group".into(), + span: Span::new(open as u32, (open + 1) as u32), + }) +} + +fn parse_quoted_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let open = input.location(); + advance_char(input); // " + let parts = collect_parts(input, source, Some('"'))?; + if at_eof(input, source) || current_char(input, source) != '"' { + return Err(InnerError { + message: "unterminated string".into(), + span: Span::new(open as u32, input.location() as u32), + }); + } + advance_char(input); // closing " + Ok((WordForm::Quoted, parts)) +} + +fn parse_bare_word( + input: &mut Input<'_>, + source: &str, +) -> Result<(WordForm, Vec), InnerError> { + let start = input.location(); + let parts = collect_parts(input, source, None)?; + if parts.is_empty() { + return Err(InnerError { + message: "expected word".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + Ok((WordForm::Bare, parts)) +} + +/// Accumulate [`WordPart`]s. +/// +/// `terminator` controls the stop condition: `Some('"')` for +/// double-quoted words (stops at the closing quote, newlines are +/// content), `None` for bare words (stops at whitespace, `;`, `\n`, +/// EOF). +fn collect_parts( + input: &mut Input<'_>, + source: &str, + terminator: Option, +) -> Result, InnerError> { + let mut parts = Vec::new(); + let mut text_buf = String::new(); + let mut text_start: Option = None; + + let flush = |parts: &mut Vec, + buf: &mut String, + start: &mut Option, + end: u32| { + if let Some(s) = start.take() { + if !buf.is_empty() { + parts.push(WordPart::Text { + value: std::mem::take(buf), + span: Span::new(s, end), + }); + } + buf.clear(); + } + }; + + loop { + if at_eof(input, source) { + break; + } + let c = current_char(input, source); + if Some(c) == terminator { + break; + } + if terminator.is_none() { + match c { + ' ' | '\t' | '\r' | '\n' | ';' => break, + _ => {} + } + } + match c { + '$' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_var_ref(input, source)?); + } + '[' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_cmd_subst(input, source)?); + } + '\\' => { + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + parts.push(parse_escape(input, source)?); + } + _ => { + if text_start.is_none() { + text_start = Some(input.location() as u32); + } + text_buf.push(c); + advance_char(input); + } + } + } + flush( + &mut parts, + &mut text_buf, + &mut text_start, + input.location() as u32, + ); + Ok(parts) +} + +fn parse_var_ref( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // $ + if at_eof(input, source) { + return Err(InnerError { + message: "expected variable name after `$`".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let mut name = String::new(); + if current_char(input, source) == '{' { + advance_char(input); + while !at_eof(input, source) { + let c = current_char(input, source); + if c == '}' { + advance_char(input); + return Ok(WordPart::VarRef { + name, + span: Span::new(start as u32, input.location() as u32), + braced: true, + }); + } + name.push(c); + advance_char(input); + } + return Err(InnerError { + message: "unterminated `${...}`".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + while !at_eof(input, source) { + let c = current_char(input, source); + if c.is_alphanumeric() || c == '_' || c == ':' { + name.push(c); + advance_char(input); + } else { + break; + } + } + Ok(WordPart::VarRef { + name, + span: Span::new(start as u32, input.location() as u32), + braced: false, + }) +} + +fn parse_cmd_subst( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // [ + let inner_start = input.location(); + let mut depth = 1usize; + while !at_eof(input, source) { + let c = current_char(input, source); + match c { + '\\' => { + advance_char(input); + if !at_eof(input, source) { + advance_char(input); + } + } + '[' => { + depth += 1; + advance_char(input); + } + ']' => { + depth -= 1; + if depth == 0 { + let inner_end = input.location(); + advance_char(input); + let span = Span::new(start as u32, input.location() as u32); + let text = source[inner_start..inner_end].to_string(); + return Ok(WordPart::CmdSubst { + source: text, + span, + body: Vec::new(), + }); + } + advance_char(input); + } + _ => advance_char(input), + } + } + Err(InnerError { + message: "unterminated `[...]` command substitution".into(), + span: Span::new(start as u32, input.location() as u32), + }) +} + +fn parse_escape( + input: &mut Input<'_>, + source: &str, +) -> Result { + let start = input.location(); + advance_char(input); // backslash + if at_eof(input, source) { + return Err(InnerError { + message: "trailing `\\` at end of input".into(), + span: Span::new(start as u32, input.location() as u32), + }); + } + let c = current_char(input, source); + advance_char(input); + let value = match c { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '\\' => '\\', + '"' => '"', + '[' => '[', + ']' => ']', + '{' => '{', + '}' => '}', + '$' => '$', + other => other, + }; + Ok(WordPart::Escape { + value, + span: Span::new(start as u32, input.location() as u32), + }) +} + +/// Peek past a `\n` and any inline whitespace on the immediately- +/// following line: does that line's first non-whitespace byte +/// begin a flag-shaped token (`-` followed by a letter, digit, +/// or another `-`)? Called by [`parse_command`] to decide whether +/// a newline should be treated as command continuation. +/// +/// Deliberately does NOT peek across a second `\n` — a blank line +/// terminates the continuation. Callers rely on this to model +/// "paragraph breaks" naturally, matching a reader's intuition. +/// +/// Doesn't consume input; only inspects `source` bytes. +fn next_line_is_flag_continuation(input: &Input<'_>, source: &str) -> bool { + let bytes = source.as_bytes(); + // Cursor sits on `\n`; look ahead starting at the byte after. + let mut i = input.location() + 1; + while i < bytes.len() { + match bytes[i] { + b' ' | b'\t' | b'\r' => i += 1, + _ => break, + } + } + if i >= bytes.len() || bytes[i] != b'-' { + return false; + } + // Look at what follows the `-`. Flag-shaped: letter, digit, + // underscore, or a second `-` (for `--end-of-options` idiom). + // Anything else (whitespace, EOF, punctuation) declines to + // continue. Underscore is included because real Vivado flag + // names like `-_64bit` (create_bd_cell's 64-bit BAR flag) are + // valid — without `_` in the set, the continuation rule + // breaks on them and Tcl tries to execute `-_64bit` as a + // command. + let next = bytes.get(i + 1).copied().unwrap_or(b'\0'); + next.is_ascii_alphanumeric() || next == b'-' || next == b'_' +} + +/// Consume input up to the next `\n` (not consuming the `\n` +/// itself). Used to treat `#`-prefixed lines mid-command as +/// inline comments — see the callsite in `parse_command`. +fn skip_to_end_of_line(input: &mut Input<'_>, source: &str) { + while !at_eof(input, source) { + if current_char(input, source) == '\n' { + break; + } + advance_char(input); + } +} + +fn skip_inline_ws(input: &mut Input<'_>, source: &str, mode: Mode) { + while !at_eof(input, source) { + let c = current_char(input, source); + if c == ' ' || c == '\t' || c == '\r' { + advance_char(input); + } else if mode == Mode::BracketBody && c == '\n' { + // Inside `[ … ]` the newline isn't a statement terminator; + // it's just whitespace. + advance_char(input); + } else if c == '\\' { + let pos = input.location(); + if pos + 1 < source.len() && source.as_bytes()[pos + 1] == b'\n' { + advance_char(input); + advance_char(input); + } else { + break; + } + } else { + break; + } + } +} + +fn at_eof(input: &Input<'_>, source: &str) -> bool { + input.location() >= source.len() +} + +fn current_char(input: &Input<'_>, source: &str) -> char { + source[input.location()..].chars().next().unwrap_or('\0') +} + +fn advance_char(input: &mut Input<'_>) { + let _ = input.next_token(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_empty() { + let out = parse(""); + assert!(out.document.stmts.is_empty()); + assert!(out.errors.is_empty()); + } + + #[test] + fn control_flow_bodies_get_parsed_into_word_body() { + // `dict for {kv} DICT BODY` — head is 2-word `dict for`, + // body is the LAST arg. The braced body should get its + // interior parsed and populated on `Word::body`, with + // absolute spans, so hover/goto/putr can descend. + let src = "dict for {lib srcs} $deps {\n puts $lib\n putr $srcs\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Last word is the body braced-word. + let body_word = cmd.words.last().unwrap(); + assert_eq!(body_word.form, crate::ast::WordForm::Braced); + let body = body_word + .body + .as_ref() + .expect("dict-for body should have parsed statements"); + // Body has three top-level commands: puts, putr, and a + // synthesized command boundary if parsing picked one up. + // Expect at least two commands (puts + putr). + assert!(body.len() >= 2, "body: {body:?}"); + let Stmt::Command(first) = &body[0] else { + panic!("expected first body stmt to be a command"); + }; + assert_eq!(first.words[0].as_text(), Some("puts")); + // Spans should be absolute (whole-source). + assert!( + first.span.start > 20, + "expected absolute span past outer command head, got {:?}", + first.span, + ); + } + + #[test] + fn foreach_body_populated() { + let src = "foreach x {a b c} {\n puts $x\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Body is the last (third-index) word. + let body_word = cmd.words.last().unwrap(); + assert!( + body_word.body.is_some(), + "foreach body should have parsed statements", + ); + } + + #[test] + fn if_bodies_populated() { + let src = "if {$x > 0} {\n puts big\n} else {\n puts small\n}"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + // Both bodies (word 2 for the `if` body, word 4 for the + // else body) should be populated. + assert!(cmd.words[2].body.is_some(), "if-body should be parsed"); + assert!(cmd.words[4].body.is_some(), "else-body should be parsed"); + } + + #[test] + fn parse_comment_and_doc() { + let src = "# regular\n## doc text\nputs hi\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + assert_eq!(out.document.stmts.len(), 3); + let Stmt::Command(cmd) = &out.document.stmts[2] else { + panic!("expected command, got {:?}", out.document.stmts[2]); + }; + assert_eq!(cmd.doc_comments, vec!["doc text".to_string()]); + assert_eq!(cmd.words[0].as_text(), Some("puts")); + assert_eq!(cmd.words[1].as_text(), Some("hi")); + } + + #[test] + fn parse_set_command() { + let out = parse("set x 42"); + assert!(out.errors.is_empty()); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(cmd.kind, CommandKind::Set)); + assert_eq!(cmd.words.len(), 3); + } + + #[test] + fn parse_proc_braced() { + let src = "proc greet {name} { puts \"hi $name\" }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc, got {:?}", cmd.kind); + }; + assert_eq!(proc.name.as_deref(), Some("greet")); + let args = proc.args_span.slice(src); + let body = proc.body_span.slice(src); + assert_eq!(args, "name"); + assert!(body.contains("puts")); + // No return type slot. + assert!(proc.return_type.is_none()); + assert!(proc.return_type_span.is_none()); + } + + #[test] + fn parse_proc_with_return_type_named() { + let src = "proc f {} string { return foo }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + assert_eq!(proc.name.as_deref(), Some("f")); + let body = proc.body_span.slice(src); + assert!(body.contains("return foo")); + let ty = proc.return_type.as_ref().expect("return type set"); + assert_eq!(ty.name(), "string"); + match ty { + crate::ast::TypeExpr::Named { .. } => {} + _ => panic!("expected Named"), + } + } + + #[test] + fn parse_proc_with_return_type_generic_no_whitespace() { + let src = "proc f {} list { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { name, args, .. } = ty else { + panic!("expected Generic") + }; + assert_eq!(name, "list"); + assert_eq!(args.len(), 1); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn parse_proc_with_return_type_nested_generic() { + let src = "proc f {} list> { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { args, .. } = ty else { + panic!() + }; + let crate::ast::TypeExpr::Generic { + name: inner_name, + args: inner_args, + .. + } = &args[0] + else { + panic!("expected nested Generic") + }; + assert_eq!(inner_name, "dict"); + assert_eq!(inner_args.len(), 2); + } + + #[test] + fn parse_proc_with_return_type_bracketed_whitespace() { + let src = "proc f {} {dict} { return {} }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let ty = proc.return_type.as_ref().unwrap(); + let crate::ast::TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "int"); + } + + #[test] + fn parse_proc_with_invalid_return_type_emits_diagnostic() { + let src = "proc f {} list< { return {} }\n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected a parse-error diagnostic for bad type" + ); + assert!(out.errors.iter().any(|e| e.message.contains("expected") + || e.message.contains("unterminated"))); + } + + #[test] + fn parse_type_decl_named() { + let src = "type bd_cell = string\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!("expected TypeDecl, got {:?}", cmd.kind) + }; + assert_eq!(td.name.as_deref(), Some("bd_cell")); + let underlying = td.underlying.as_ref().unwrap(); + assert_eq!(underlying.name(), "string"); + } + + #[test] + fn parse_type_decl_generic_underlying() { + let src = "type fancy_dict = {dict}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!() + }; + let crate::ast::TypeExpr::Generic { name, args, .. } = + td.underlying.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + + #[test] + fn parse_type_decl_without_equals_works() { + // `type T U` (no `=`) is also accepted — the `=` is sugar. + let src = "type widget string\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::TypeDecl(td) = &cmd.kind else { + panic!() + }; + assert_eq!(td.name.as_deref(), Some("widget")); + assert_eq!(td.underlying.as_ref().unwrap().name(), "string"); + } + + #[test] + fn parse_type_decl_with_bad_underlying_emits_diagnostic() { + let src = "type foo = \n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected diagnostic for malformed underlying type" + ); + } + + // --- enum declarations ----------------------------------------- + + #[test] + fn parse_enum_decl_simple() { + let src = "enum Direction = {\n North\n South\n East\n West\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!("expected EnumDecl, got {:?}", cmd.kind); + }; + assert_eq!(ed.name.as_deref(), Some("Direction")); + assert_eq!(ed.variants.len(), 4); + for v in &ed.variants { + assert!(v.payload.is_none(), "expected empty-payload variant"); + } + let names: Vec<&str> = + ed.variants.iter().map(|v| v.name.as_str()).collect(); + assert_eq!(names, vec!["North", "South", "East", "West"]); + } + + #[test] + fn parse_enum_decl_with_payloads() { + let src = "enum Property = {\n Scalar: string\n Nested: dict\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.name.as_deref(), Some("Property")); + assert_eq!(ed.variants.len(), 2); + assert_eq!(ed.variants[0].name, "Scalar"); + assert_eq!(ed.variants[0].payload.as_ref().unwrap().name(), "string"); + assert_eq!(ed.variants[1].name, "Nested"); + let crate::ast::TypeExpr::Generic { name, args, .. } = + ed.variants[1].payload.as_ref().unwrap() + else { + panic!(); + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "Property"); + } + + #[test] + fn parse_enum_decl_mixed_payload_and_empty() { + let src = + "enum Mix = {\n Empty\n WithInt: int\n Other\n WithList: list\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.variants.len(), 4); + assert!(ed.variants[0].payload.is_none()); + assert_eq!(ed.variants[1].payload.as_ref().unwrap().name(), "int"); + assert!(ed.variants[2].payload.is_none()); + assert_eq!(ed.variants[3].payload.as_ref().unwrap().name(), "list"); + } + + #[test] + fn parse_enum_decl_without_equals() { + let src = "enum Color {\n Red\n Green\n Blue\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::EnumDecl(ed) = &cmd.kind else { + panic!() + }; + assert_eq!(ed.name.as_deref(), Some("Color")); + assert_eq!(ed.variants.len(), 3); + } + + #[test] + fn parse_enum_decl_with_bad_variant_emits_diagnostic() { + // 123Foo is not a valid identifier — should diagnose. + let src = "enum Bad = {\n 123Foo: int\n}\n"; + let out = parse(src); + assert!( + !out.errors.is_empty(), + "expected diagnostic for malformed variant" + ); + } + + #[test] + fn parse_proc_with_qualified_arg_type() { + // The `E::V` qualified syntax for overloaded handler args. + let src = + "proc handle_prop {v: Property::Scalar} string { return $v }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!() + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!() + }; + let sig = proc.signature.as_ref().unwrap(); + assert_eq!(sig.args.len(), 1); + let arg = &sig.args[0]; + assert_eq!(arg.name, "v"); + let crate::ast::TypeExpr::Qualified { + namespace, variant, .. + } = arg.type_annotation.as_ref().unwrap() + else { + panic!( + "expected Qualified type annotation, got {:?}", + arg.type_annotation + ); + }; + assert_eq!(namespace, "Property"); + assert_eq!(variant, "Scalar"); + } + + #[test] + fn parse_variable_and_subst() { + let src = "puts $x [foo bar]"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert_eq!(cmd.words.len(), 3); + let WordPart::VarRef { name, .. } = &cmd.words[1].parts[0] else { + panic!("expected var ref"); + }; + assert_eq!(name, "x"); + let WordPart::CmdSubst { source: src, .. } = &cmd.words[2].parts[0] + else { + panic!("expected cmd subst"); + }; + assert_eq!(src, "foo bar"); + } + + #[test] + fn parse_quoted_with_subst() { + let src = r#"puts "hello $name""#; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert_eq!(cmd.words[1].form, WordForm::Quoted); + assert_eq!(cmd.words[1].parts.len(), 2); + let WordPart::Text { value, .. } = &cmd.words[1].parts[0] else { + panic!(); + }; + assert_eq!(value, "hello "); + let WordPart::VarRef { name, .. } = &cmd.words[1].parts[1] else { + panic!(); + }; + assert_eq!(name, "name"); + } + + #[test] + fn recovers_from_unterminated_brace() { + let src = "puts {oops\nputs ok\n"; + let out = parse(src); + assert!(!out.errors.is_empty()); + assert!(out.errors[0].message.contains("brace group")); + // After the error we should still see the second `puts ok`. + let ok_cmd = out.document.stmts.iter().find_map(|s| match s { + Stmt::Command(c) + if c.words.first().and_then(|w| w.as_text()) + == Some("puts") + && c.words.get(1).and_then(|w| w.as_text()) + == Some("ok") => + { + Some(c) + } + _ => None, + }); + assert!( + ok_cmd.is_some(), + "expected recovery: {:?}", + out.document.stmts + ); + } + + #[test] + fn proc_body_parses_into_statements_with_absolute_spans() { + let src = "proc outer {\n a\n} {\n inner_call foo\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(proc.body.len(), 1, "{:?}", proc.body); + let Stmt::Command(body_cmd) = &proc.body[0] else { + panic!("expected command in body"); + }; + // Span is absolute: it slices back to the original source. + assert_eq!(body_cmd.words[0].span.slice(src), "inner_call"); + assert_eq!( + body_cmd.span.start as usize, + src.find("inner_call").unwrap() + ); + } + + #[test] + fn nested_proc_body_is_parsed_recursively() { + let src = + "proc outer {\n a\n} {\n proc inner {\n b\n} {\n deep\n}\n}\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(outer) = &cmd.kind else { + panic!("expected outer proc"); + }; + let Stmt::Command(inner_cmd) = &outer.body[0] else { + panic!("expected inner proc command"); + }; + let CommandKind::Proc(inner) = &inner_cmd.kind else { + panic!("expected inner proc"); + }; + assert_eq!(inner.name.as_deref(), Some("inner")); + // Inner proc got its signature and body populated too. + assert!(inner.signature.is_some()); + let Stmt::Command(deep) = &inner.body[0] else { + panic!("expected deep command"); + }; + assert_eq!(deep.words[0].span.slice(src), "deep"); + } + + #[test] + fn parses_src_statement() { + let src = "src common/log\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Src(import) = &cmd.kind else { + panic!("expected Src, got {:?}", cmd.kind); + }; + assert_eq!(import.path.as_deref(), Some("common/log")); + assert_eq!(import.path_span.slice(src), "common/log"); + } + + #[test] + fn parses_src_with_named_dep_prefix() { + let out = parse("src @xilinx-ip/cpm5\n"); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Src(import) = &cmd.kind else { + panic!("expected Src"); + }; + assert_eq!(import.path.as_deref(), Some("@xilinx-ip/cpm5")); + } + + #[test] + fn src_with_extra_words_is_generic() { + // `src a b` isn't a valid import — it falls back to generic so + // the validator can report it as an unknown command rather than + // the parser silently accepting it. + let out = parse("src a b\n"); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(cmd.kind, CommandKind::Generic), "{:?}", cmd.kind); + } + + #[test] + fn bracket_body_treats_newlines_as_whitespace() { + // Multi-line call inside `[ … ]` parses as a *single* command, + // no backslash continuations needed. + let src = "\ +set cell [ + create_cpm5_cpm_pcie0 + -cell cpm5 + -max_link_speed 32.0_GT/s +] +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(set_cmd) = &out.document.stmts[0] else { + panic!(); + }; + assert!(matches!(set_cmd.kind, CommandKind::Set)); + // The `set`'s value word is the cmd-subst; its body should be + // a single command with five words. + let WordPart::CmdSubst { body, .. } = &set_cmd.words[2].parts[0] else { + panic!("expected CmdSubst"); + }; + assert_eq!(body.len(), 1, "{body:#?}"); + let Stmt::Command(inner) = &body[0] else { + panic!(); + }; + let word_texts: Vec<&str> = + inner.words.iter().filter_map(|w| w.as_text()).collect(); + assert_eq!( + word_texts, + vec![ + "create_cpm5_cpm_pcie0", + "-cell", + "cpm5", + "-max_link_speed", + "32.0_GT/s", + ] + ); + } + + #[test] + fn bracket_body_still_separates_on_semicolon() { + // Explicit `;` keeps the multi-command form available inside + // brackets for users who want it. + let src = "set x [a 1 ; b 2]\n"; + let out = parse(src); + assert!(out.errors.is_empty()); + let Stmt::Command(set_cmd) = &out.document.stmts[0] else { + panic!(); + }; + let WordPart::CmdSubst { body, .. } = &set_cmd.words[2].parts[0] else { + panic!(); + }; + assert_eq!(body.len(), 2, "{body:#?}"); + } + + #[test] + fn toplevel_newlines_still_terminate() { + // The bracket-body relaxation does not leak to the top level. + let src = "puts a\nputs b\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + /// Line continuation via a leading `-` on the next line: the + /// common flag-per-line configurator shape (`create_foo`, + /// newline, indented `-bar val`, newline, `-baz val`, …) + /// parses as one command without needing a trailing `\`. + #[test] + fn dash_leading_next_line_continues_command() { + let src = "create_foo\n -bar 1\n -baz 2\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + let words: Vec<&str> = + cmds[0].words.iter().filter_map(Word::as_text).collect(); + assert_eq!(words, ["create_foo", "-bar", "1", "-baz", "2"]); + } + + /// A `--` (end-of-options) continuation also chains — same + /// leading-dash shape. + #[test] + fn double_dash_next_line_continues_command() { + let src = "cmd -a 1\n -- rest\n"; + let out = parse(src); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("{:#?}", out.document.stmts); + }; + assert_eq!(cmd.words.len(), 5); + } + + /// Non-dash next line still terminates. A regression here + /// would break every existing top-level script. + #[test] + fn non_dash_next_line_terminates_as_before() { + let src = "puts a\nputs b\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + /// A blank line between the header and a dash-led line breaks + /// the continuation — the header stands alone and the dash- + /// led line becomes a new (probably weird) command. This + /// matches how a reader intuits paragraph breaks: an empty + /// line is a stronger separator than a newline. + /// Vivado flags like `-_64bit` start with `-_` — the underscore + /// must be recognized as flag-shaped so the continuation rule + /// keeps the line inside the enclosing command instead of + /// starting a new (invalid) command. + #[test] + fn dash_underscore_line_continues_command() { + let src = "create_bar\n -cell foo\n -_64bit 1\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + let words: Vec<&str> = + cmds[0].words.iter().filter_map(Word::as_text).collect(); + assert_eq!(words, ["create_bar", "-cell", "foo", "-_64bit", "1"]); + } + + /// Trailing whitespace on the previous line must not defeat the + /// dash-continuation rule — real-world files often carry a stray + /// space at end of line, and we want the multi-line command to + /// still parse as one command. + #[test] + fn dash_continuation_survives_trailing_ws_on_previous_line() { + let src = "cmd\n -foo 1 \n -bar 2\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 1, "{cmds:#?}"); + assert_eq!(cmds[0].words.len(), 5); + } + + #[test] + fn blank_line_before_dash_breaks_continuation() { + let src = "cmd\n\n -a 1\n"; + let out = parse(src); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2, "{cmds:#?}"); + } + + #[test] + fn proc_body_newlines_still_terminate() { + // Proc bodies are scripts; the relaxation is bracket-only. + let src = "proc f {} {\n puts a\n puts b\n}\n"; + let out = parse(src); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.body.len(), 2); + } + + #[test] + fn semicolon_separates_commands() { + let src = "set a 1; set b 2"; + let out = parse(src); + assert!(out.errors.is_empty()); + let cmds: Vec<&Command> = out + .document + .stmts + .iter() + .filter_map(|s| { + if let Stmt::Command(c) = s { + Some(c) + } else { + None + } + }) + .collect(); + assert_eq!(cmds.len(), 2); + } + + #[test] + fn inline_comment_arg_line_is_stripped() { + // The configurator idiom for commenting out an arg line. + // Parser should eat `#-intf_parent_pin_list 0` and leave a + // clean word list `[configure, -enable_reg_interface, 1]`. + let src = "\ +set cfg [ + configure + -enable_reg_interface 1 + #-intf_parent_pin_list 0 +]\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "parse errors: {:?}", out.errors); + // The set command has three words: `set`, `cfg`, `[…]`. + let set = out + .document + .stmts + .iter() + .find_map(|s| match s { + crate::ast::Stmt::Command(c) + if c.words.first().and_then(|w| w.as_text()) + == Some("set") => + { + Some(c) + } + _ => None, + }) + .expect("set command"); + // The CmdSubst body should contain one command with 3 words + // (the `#-intf_parent_pin_list 0` line is eaten). + let bracket = &set.words[2]; + let crate::ast::WordPart::CmdSubst { body, .. } = &bracket.parts[0] + else { + panic!("expected CmdSubst"); + }; + let inner_cmd = body + .iter() + .find_map(|s| match s { + crate::ast::Stmt::Command(c) => Some(c), + _ => None, + }) + .expect("configure command"); + let word_texts: Vec<&str> = + inner_cmd.words.iter().filter_map(|w| w.as_text()).collect(); + assert_eq!( + word_texts, + vec!["configure", "-enable_reg_interface", "1"], + "expected the commented arg line to be gone", + ); + } + + #[test] + fn hash_mid_command_line_is_still_comment() { + // `#` mid-command at word-start, even without a newline + // in-between, is treated as a comment. Matches the + // configurator ergonomics — inside `[cmd -a x #-b y]` + // the `#-b y` gets eaten. + let src = "[configure -a x #-b y]\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "parse errors: {:?}", out.errors); + } + + #[test] + fn top_level_attribute_attaches_to_proc() { + // Regression: `@test` above `proc foo {} { ... }` should + // populate `proc.attributes` with a single attribute named + // `test`. Consumed by `vw test`. + let src = "@test\nproc foo {} { puts hi }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!("expected command"); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!("expected proc"); + }; + assert_eq!(proc.attributes.len(), 1); + assert_eq!(proc.attributes[0].name, "test"); + assert!(proc.attributes[0].values.is_empty()); + } + + #[test] + fn attribute_with_kebab_value_parses() { + // `@test(dedicated-eda)` — kebab-case ident allowed in + // attribute value position only. This shape drives the + // `vw test` runner's shared-vs-dedicated bucket choice. + let src = "@test(dedicated-eda)\nproc foo {} { }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 1); + let attr = &proc.attributes[0]; + assert_eq!(attr.name, "test"); + assert_eq!(attr.values.len(), 1); + match &attr.values[0] { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "dedicated-eda") + } + other => panic!("expected Ident, got {other:?}"), + } + } + + #[test] + fn multiple_attributes_stack_on_one_proc() { + let src = "@test\n@another\nproc foo {} { }\n"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 2); + assert_eq!(proc.attributes[0].name, "test"); + assert_eq!(proc.attributes[1].name, "another"); + } + + #[test] + fn attribute_with_whitespace_separated_items() { + // `@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S)` — + // no commas between items, one positional + one keyed. + let src = "\ +@test(dedicated-eda part=xcvm3358-vsvh1747-2M-e-S) +proc foo {} { } +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + let Stmt::Command(cmd) = &out.document.stmts[0] else { + panic!(); + }; + let CommandKind::Proc(proc) = &cmd.kind else { + panic!(); + }; + assert_eq!(proc.attributes.len(), 1); + let attr = &proc.attributes[0]; + assert_eq!(attr.name, "test"); + assert_eq!(attr.values.len(), 2); + // Positional item. + match &attr.values[0] { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "dedicated-eda") + } + other => panic!("expected Ident, got {other:?}"), + } + // Keyed item. + match &attr.values[1] { + AttributeValue::Keyed { key, value, .. } => { + assert_eq!(key, "part"); + match value.as_ref() { + AttributeValue::Ident { value, .. } => { + assert_eq!(value, "xcvm3358-vsvh1747-2M-e-S") + } + other => panic!("expected Ident, got {other:?}"), + } + } + other => panic!("expected Keyed, got {other:?}"), + } + } + + #[test] + fn attribute_comma_separated_still_works() { + // Backward-compat: `@enum(a, b, c)` still parses as three + // positional idents. + let src = "\ +proc foo { + @enum(a, b, c) x +} { } +"; + let out = parse(src); + assert!(out.errors.is_empty(), "{:?}", out.errors); + } + + #[test] + fn attribute_on_non_proc_command_reports_error() { + // `@test` above a `set` command doesn't make sense — the + // parser records a diagnostic and drops the attrs so they + // don't leak onto whatever the NEXT command is. + let src = "@test\nset x 1\n"; + let out = parse(src); + assert!( + out.errors.iter().any(|e| e + .message + .contains("attribute attached to non-proc statement")), + "expected non-proc-attachment diagnostic, got {:?}", + out.errors, + ); + } +} diff --git a/vw-htcl/src/proc_args.rs b/vw-htcl/src/proc_args.rs new file mode 100644 index 0000000..60d207c --- /dev/null +++ b/vw-htcl/src/proc_args.rs @@ -0,0 +1,599 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parser for the structured proc-arg grammar (Phase 2). +//! +//! Operates on the inner contents of a `proc`'s args braces — the +//! span passed in must point at the text between the braces, not +//! including the braces themselves. All spans on the returned AST +//! nodes are absolute file offsets, so they slot directly into the +//! main document's diagnostics without rebasing. +//! +//! Grammar: +//! +//! ```text +//! args := arg_item* +//! arg_item := doc_comment* attribute* IDENT ( ':' TYPE )? +//! attribute := '@' IDENT ( '(' value ( ',' value )* ')' )? +//! value := integer | string | ident +//! TYPE := IDENT ( '<' TYPE ( ',' TYPE )* '>' )? +//! ``` +//! +//! The optional `: TYPE` slot turns an arg from an opaque +//! identifier into a typed one. Adoption is gradual — existing +//! libraries without annotations still parse identically. +//! +//! Whitespace, blank lines, and non-doc comments are skippable +//! between items. + +use crate::ast::{Attribute, AttributeValue, ProcArg, ProcSignature}; +use crate::parser::ParseError; +use crate::span::Span; + +pub fn parse_proc_args( + full_source: &str, + args_span: Span, +) -> (ProcSignature, Vec) { + let inner = args_span.slice(full_source); + let mut state = State { + inner, + base: args_span.start, + pos: 0, + errors: Vec::new(), + }; + let mut args = Vec::new(); + state.parse_args(&mut args); + let State { errors, .. } = state; + ( + ProcSignature { + args, + span: args_span, + // Filled in later by the parser's `populate_procs` + // pass once the return-type annotation has been parsed. + return_type: None, + }, + errors, + ) +} + +struct State<'a> { + inner: &'a str, + /// Absolute file offset where `inner` starts. + base: u32, + /// Byte offset into `inner`. + pos: usize, + errors: Vec, +} + +impl<'a> State<'a> { + fn at_eof(&self) -> bool { + self.pos >= self.inner.len() + } + + fn current(&self) -> char { + self.inner[self.pos..].chars().next().unwrap_or('\0') + } + + fn peek_at(&self, offset: usize) -> char { + let target = self.pos + offset; + if target >= self.inner.len() { + '\0' + } else { + self.inner[target..].chars().next().unwrap_or('\0') + } + } + + fn abs(&self) -> u32 { + self.base + self.pos as u32 + } + + fn bump(&mut self) { + if let Some(c) = self.inner[self.pos..].chars().next() { + self.pos += c.len_utf8(); + } + } + + fn skip_horizontal_ws(&mut self) { + while !self.at_eof() { + let c = self.current(); + if c == ' ' || c == '\t' || c == '\r' { + self.bump(); + } else { + break; + } + } + } + + /// Consume blank lines, comments, and whitespace; doc comments + /// (`##`) are collected and returned so they can attach to the + /// next arg item. The returned span (when non-empty) covers the + /// whole `##` block from its first `#` byte to the newline after + /// its last line, matching [`Command::doc_comments_span`]'s + /// convention. + fn skip_separators(&mut self) -> (Vec, Option) { + let mut docs = Vec::new(); + let mut docs_span: Option = None; + loop { + // Whitespace including newlines + while !self.at_eof() { + let c = self.current(); + if c.is_whitespace() { + self.bump(); + } else { + break; + } + } + if self.at_eof() { + break; + } + if self.current() == '#' { + let line_start = self.abs(); + let is_doc = self.peek_at(1) == '#'; + self.bump(); + if is_doc { + self.bump(); + } + if !self.at_eof() && self.current() == ' ' { + self.bump(); + } + let text_start = self.pos; + while !self.at_eof() && self.current() != '\n' { + self.bump(); + } + let text = self.inner[text_start..self.pos].to_string(); + if is_doc { + let line_end = self.abs(); + docs.push(text); + docs_span = Some(match docs_span { + Some(prev) => Span::new(prev.start, line_end), + None => Span::new(line_start, line_end), + }); + } + continue; + } + break; + } + (docs, docs_span) + } + + fn parse_args(&mut self, out: &mut Vec) { + loop { + let (docs, docs_span) = self.skip_separators(); + if self.at_eof() { + if !docs.is_empty() { + // Doc comments with nothing to attach to. Warn so + // the user knows they're unused. + self.errors.push(ParseError { + message: "doc comment with no following argument" + .into(), + span: Span::new(self.abs(), self.abs()), + }); + } + break; + } + let item_start = self.abs(); + let mut attributes = Vec::new(); + // Attributes can be interleaved with whitespace and + // doc comments themselves can't appear between attrs, + // only at the head — that's the convention from the + // project plan ("doc comments first, then attributes in + // any order, then the argument name"). + while !self.at_eof() && self.current() == '@' { + if let Some(attr) = self.parse_attribute() { + attributes.push(attr); + } + self.skip_horizontal_ws(); + // Allow newlines between attributes. + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + } + // Identifier. + self.skip_horizontal_ws(); + if self.at_eof() { + self.errors.push(ParseError { + message: "expected argument name".into(), + span: Span::new(item_start, self.abs()), + }); + break; + } + let name_start = self.abs(); + let name = self.consume_ident(); + if name.is_empty() { + let c = self.current(); + self.errors.push(ParseError { + message: format!("expected argument name, found {c}"), + span: Span::new(self.abs(), self.abs() + 1), + }); + // Resync: drop whatever non-whitespace junk is here + // so we can try the next item. + while !self.at_eof() && !self.current().is_whitespace() { + self.bump(); + } + continue; + } + let name_span = Span::new(name_start, self.abs()); + // Optional `: TYPE` annotation. Tcl strings allow `:` + // in bare words, but we're in the structured-args + // sub-grammar — distinct rules apply here. A `:` + // immediately after the arg name (with optional + // horizontal whitespace) opens the annotation slot. + self.skip_horizontal_ws(); + let type_annotation = if !self.at_eof() && self.current() == ':' { + self.bump(); // ':' + self.skip_horizontal_ws(); + let ty_start = self.abs(); + // Consume up to whitespace or end of arg. The type + // mini-parser handles its own internal grammar + // (idents, '<', ',', '>'); we just need to slice + // the right text out of the source. + while !self.at_eof() { + let c = self.current(); + if c.is_whitespace() || c == '#' { + break; + } + self.bump(); + } + let ty_end = self.abs(); + let text = &self.inner[(ty_start - self.base) as usize + ..(ty_end - self.base) as usize]; + match crate::type_parse::parse(text, ty_start) { + Ok(ty) => Some(ty), + Err(e) => { + self.errors.push(ParseError { + message: e.message, + span: e.span, + }); + None + } + } + } else { + None + }; + let span = Span::new(item_start, self.abs()); + out.push(ProcArg { + name, + name_span, + doc_comments: docs, + doc_comments_span: docs_span, + attributes, + type_annotation, + span, + }); + } + } + + fn parse_attribute(&mut self) -> Option { + let start = self.abs(); + self.bump(); // '@' + let name_start = self.abs(); + let name = self.consume_ident(); + if name.is_empty() { + self.errors.push(ParseError { + message: "expected attribute name after @".into(), + span: Span::new(start, self.abs()), + }); + return None; + } + let name_span = Span::new(name_start, self.abs()); + let mut values = Vec::new(); + if !self.at_eof() && self.current() == '(' { + self.bump(); + loop { + self.skip_horizontal_ws(); + // Allow newlines inside the value list + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + if self.at_eof() { + self.errors.push(ParseError { + message: "unterminated attribute argument list".into(), + span: Span::new(start, self.abs()), + }); + break; + } + if self.current() == ')' { + self.bump(); + break; + } + match self.parse_value() { + Some(v) => values.push(v), + None => { + // Drop characters up to `,` or `)` to resync. + while !self.at_eof() { + let c = self.current(); + if c == ',' || c == ')' || c == '\n' { + break; + } + self.bump(); + } + } + } + self.skip_horizontal_ws(); + while !self.at_eof() && self.current() == '\n' { + self.bump(); + self.skip_horizontal_ws(); + } + if self.at_eof() { + continue; + } + if self.current() == ',' { + self.bump(); + } + } + } + Some(Attribute { + name, + name_span, + values, + span: Span::new(start, self.abs()), + }) + } + + fn parse_value(&mut self) -> Option { + let start = self.abs(); + let c = self.current(); + if c == '"' { + self.bump(); + let text_start = self.pos; + let mut buf = String::new(); + while !self.at_eof() && self.current() != '"' { + if self.current() == '\\' { + self.bump(); + if !self.at_eof() { + buf.push(self.current()); + self.bump(); + } + } else { + buf.push(self.current()); + self.bump(); + } + } + let _ = text_start; + if self.at_eof() { + self.errors.push(ParseError { + message: "unterminated string".into(), + span: Span::new(start, self.abs()), + }); + } else { + self.bump(); // closing " + } + Some(AttributeValue::String { + value: buf, + span: Span::new(start, self.abs()), + }) + } else if c == '-' || c.is_ascii_digit() { + let mut buf = String::new(); + if c == '-' { + buf.push('-'); + self.bump(); + } + while !self.at_eof() && self.current().is_ascii_digit() { + buf.push(self.current()); + self.bump(); + } + match buf.parse::() { + Ok(value) => Some(AttributeValue::Integer { + value, + span: Span::new(start, self.abs()), + }), + Err(_) => { + self.errors.push(ParseError { + message: format!("invalid integer: {buf}"), + span: Span::new(start, self.abs()), + }); + None + } + } + } else if is_ident_start(c) { + let value = self.consume_ident(); + Some(AttributeValue::Ident { + value, + span: Span::new(start, self.abs()), + }) + } else { + self.errors.push(ParseError { + message: format!("expected attribute value, found {c}"), + span: Span::new(start, self.abs() + 1), + }); + None + } + } + + fn consume_ident(&mut self) -> String { + let mut out = String::new(); + let mut first = true; + while !self.at_eof() { + let c = self.current(); + let ok = if first { + is_ident_start(c) + } else { + is_ident_continue(c) + }; + if !ok { + break; + } + out.push(c); + self.bump(); + first = false; + } + out + } +} + +fn is_ident_start(c: char) -> bool { + c.is_alphabetic() || c == '_' +} + +fn is_ident_continue(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(input: &str) -> (ProcSignature, Vec) { + // Pretend the inner args text starts at offset 0 in a virtual + // source identical to `input`. + let span = Span::new(0, input.len() as u32); + parse_proc_args(input, span) + } + + #[test] + fn empty_signature() { + let (sig, errs) = parse(""); + assert!(errs.is_empty()); + assert!(sig.args.is_empty()); + } + + #[test] + fn plain_arg_names() { + let (sig, errs) = parse("a b c"); + assert!(errs.is_empty(), "{:?}", errs); + let names: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b", "c"]); + } + + #[test] + fn arg_with_named_type_annotation() { + let (sig, errs) = parse("object: bd_cell"); + assert!(errs.is_empty(), "{:?}", errs); + assert_eq!(sig.args.len(), 1); + let a = &sig.args[0]; + assert_eq!(a.name, "object"); + let ty = a.type_annotation.as_ref().expect("type set"); + assert_eq!(ty.name(), "bd_cell"); + } + + #[test] + fn arg_with_generic_type_annotation() { + let (sig, errs) = parse("cells: list"); + assert!(errs.is_empty(), "{:?}", errs); + let a = &sig.args[0]; + let crate::ast::TypeExpr::Generic { name, args, .. } = + a.type_annotation.as_ref().unwrap() + else { + panic!() + }; + assert_eq!(name, "list"); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn typed_and_untyped_args_mix() { + let (sig, errs) = parse("a b: string c"); + assert!(errs.is_empty(), "{:?}", errs); + assert_eq!(sig.args.len(), 3); + assert!(sig.args[0].type_annotation.is_none()); + assert_eq!( + sig.args[1].type_annotation.as_ref().unwrap().name(), + "string" + ); + assert!(sig.args[2].type_annotation.is_none()); + } + + #[test] + fn arg_with_attrs_and_type() { + let (sig, errs) = parse("@default(0) count: int"); + assert!(errs.is_empty(), "{:?}", errs); + let a = &sig.args[0]; + assert_eq!(a.name, "count"); + assert_eq!(a.attributes.len(), 1); + assert_eq!(a.attributes[0].name, "default"); + assert_eq!(a.type_annotation.as_ref().unwrap().name(), "int"); + } + + #[test] + fn arg_with_invalid_type_emits_diagnostic() { + let (_sig, errs) = parse("v: ` call sites into +//! `puts [::repr -v ]` (typed) or `puts ` +//! (untyped fallback). +//! +//! `putr` is a compile-time-only shim — by the time source reaches +//! Tcl, every occurrence has been rewritten in place. This lets one +//! syntactic form cover the ergonomic case (`putr $cpm5_cfg` at the +//! REPL prompt to dump a typed value's `repr` output) without +//! needing a runtime `putr` proc, and without depending on Tcl-level +//! shape detection. +//! +//! ## Walker +//! +//! The rewrite pass walks the FULL parsed AST — top-level +//! statements, proc bodies (populated by +//! [`crate::parser::populate_procs`]), namespace-eval bodies, and +//! command-substitution interiors. It mirrors the scope discipline +//! in `validate::validate_stmts`: +//! +//! - Proc bodies push a fresh [`VarTypeTable`] frame seeded with +//! the proc's typed parameters (from `ProcArg.type_annotation`). +//! - Namespace-eval bodies and command-substitution bodies share +//! the enclosing frame (mirrors Tcl semantics — `namespace eval` +//! creates a namespace but not a fresh local-variable scope, and +//! `[…]` runs in the caller's frame). +//! - `set VAR ` records the value's inferred type in the +//! current frame so downstream `putr $VAR` sees it. +//! +//! ## Rewriting +//! +//! Rewrites are exposed as a `HashMap` keyed by the +//! putr command's source span. `crate::lower::lower_command` +//! consults the map at emit time — when the current command's +//! span matches a key, it emits the replacement Tcl instead of +//! lowering the original. This avoids mutating the source +//! string, which would shift byte offsets and break +//! `LoadedProgram::locate_span` for anything after the rewrite +//! site. +//! +//! ## Fallbacks +//! +//! - `putr $x` where `x`'s type is unknown → `puts $x` (plain +//! fallback; no worse than what the user would type directly). +//! - `putr` with zero args or more than one → left untouched. The +//! analyzer's builtin recognition still accepts them; if the +//! caller wanted something else the diagnostic layer flags it +//! through the normal path. +//! - `putr ` → falls to the untyped path. Literal +//! strings don't carry `T::repr` targets; `puts "hello"` is what +//! the user gets and what they probably want. + +use std::collections::HashMap; + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcSignature, Stmt, WordPart, +}; +use crate::span::Span; +use crate::validate::{ + build_proc_table, build_signature_table, value_type_with_procs, + VarTypeTable, +}; + +/// A map from putr command span → replacement Tcl. `crate::lower` +/// consults this at emit time so the lowered proc body and the +/// lowered top-level statements both pick up the rewrite. Empty +/// when the input contained no `putr` calls; safe to build and +/// pass into lowering unconditionally. +pub type RewriteMap = HashMap; + +/// Build the rewrite map for every `putr ` command in +/// `document`, dispatching through the argument's type's `repr` +/// proc when the type is statically knowable and falling back to +/// plain `puts` when it isn't. Equivalent to +/// [`rewrite_with_extras`] with an empty extras map — most +/// callers that don't have prior-batch state use this. +pub fn rewrite(source: &str, document: &Document) -> RewriteMap { + rewrite_with_extras(source, document, &HashMap::new(), &HashMap::new()) +} + +/// Same as [`rewrite`] but accepts prior-batch context so the +/// REPL can see variable types that came from earlier commits. +/// +/// - `extra_sigs`: signatures from prior batches, merged into the +/// local signature table so `putr [prior_batch_proc]` resolves. +/// - `extra_var_types`: prior-batch top-level variable bindings +/// (from `Session::top_level_var_types()`). Seeded into the +/// walker's initial `VarTypeTable` frame so `putr $prior_var` +/// dispatches through the right `T::repr`. Later `set` bindings +/// in the current document shadow. +/// +/// `source` must be the same source `document` was parsed from — +/// the walker uses AST spans as byte offsets into it. +pub fn rewrite_with_extras( + source: &str, + document: &Document, + extra_sigs: &HashMap, + extra_var_types: &HashMap, +) -> RewriteMap { + let mut sig_diags = Vec::new(); + let mut table = build_signature_table(document, &mut sig_diags); + // Prior-batch signatures fill in the gaps; the current + // document's entries win (entry().or_insert is a no-op on + // present keys). + for (name, sig) in extra_sigs { + table.entry(name.clone()).or_insert(*sig); + } + // Proc table for return-type inference on unannotated procs + // — lets `putr [some_proc]` and `set x [some_proc]; putr $x` + // both dispatch through the right `T::repr` even when + // `some_proc` has no `-> T` annotation. Built from the + // current document only; prior-batch procs aren't inferrable + // here (their bodies aren't in `document`), but their + // annotated returns already flow through `extra_sigs`. + let proc_table = build_proc_table(document); + let mut rewrites: RewriteMap = HashMap::new(); + let mut top_var_table: VarTypeTable = extra_var_types.clone(); + walk_stmts( + source, + &document.stmts, + &table, + &proc_table, + &mut top_var_table, + &mut rewrites, + ); + rewrites +} + +fn walk_stmts( + source: &str, + stmts: &[Stmt], + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &mut VarTypeTable, + rewrites: &mut RewriteMap, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // `set VAR ` binding — seed the var-type table + // before recursing so downstream `putr $VAR` sees the + // type. Same shape as validate.rs's set-binding hook. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + var_table, + Some(proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + } + } + // `putr ` — the actual work. + if let Some(replacement) = + try_rewrite_putr(source, cmd, sig_table, proc_table, var_table) + { + rewrites.insert(cmd.span, replacement); + } + // Recurse into structured commands. + match &cmd.kind { + CommandKind::Proc(proc) => { + // Fresh scope per proc body, seeded with typed + // parameters. Same pattern as + // `validate::validate_stmts`'s proc handling. + let mut proc_scope = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for a in &sig.args { + if let Some(ty) = &a.type_annotation { + proc_scope.insert(a.name.clone(), ty.clone()); + } + } + } + walk_stmts( + source, + &proc.body, + sig_table, + proc_table, + &mut proc_scope, + rewrites, + ); + } + CommandKind::NamespaceEval(ns) => { + walk_stmts( + source, &ns.body, sig_table, proc_table, var_table, + rewrites, + ); + } + _ => {} + } + // Descend into `[ … ]` command substitution bodies on any + // word — matches how validate.rs walks these. `putr` + // buried inside a `[…]` still gets rewritten. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + walk_stmts( + source, body, sig_table, proc_table, var_table, + rewrites, + ); + } + } + // Descend into control-flow braced bodies (foreach, + // dict for, for, while, if, catch — populated by + // `parser::populate_control_flow_bodies`). Without + // this, `putr` inside a `foreach { … }` body would + // reach Tcl as a literal command call. + if let Some(body) = &word.body { + walk_stmts( + source, body, sig_table, proc_table, var_table, rewrites, + ); + } + } + } +} + +/// If `cmd` is a `putr ` call, return the replacement Tcl +/// source. `None` for non-putr commands and for `putr` with the +/// wrong arg count (which we leave untouched — the analyzer will +/// flag the arity issue through its normal path). +fn try_rewrite_putr( + source: &str, + cmd: &Command, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &VarTypeTable, +) -> Option { + let head = cmd.words.first()?; + if head.as_text() != Some("putr") { + return None; + } + // Exactly one argument. `putr` matches `puts`'s single-value + // shape; multi-word invocations get left as-is. + if cmd.words.len() != 2 { + return None; + } + let arg = &cmd.words[1]; + // `value_type_with_procs` covers `$var`, `[proc-call]` (with + // return-type inference for unannotated procs via proc_table), + // and bare `true`/`false`. Everything else returns None and + // lands in the plain-puts fallback. + let inferred = + value_type_with_procs(arg, sig_table, var_table, Some(proc_table)); + let arg_source = arg.span.slice(source); + Some(match inferred { + Some(ty) => { + let dispatch = crate::repr::dispatch_name(&ty); + format!("puts [{dispatch} -v {arg_source}]") + } + None => { + // Fallback: `puts `. Same behavior the + // caller would get from typing `puts $x` directly — + // no worse than that, and consistent with what happens + // for `putr `. + format!("puts {arg_source}") + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + /// Test helper: apply the rewrite map to the source top-down + /// so tests can assert on the fully-substituted result. The + /// real emit path doesn't do this — it consults the map + /// per-command during lowering — but for unit tests it's the + /// clearest way to see what came out. + fn rewrite_str(input: &str) -> String { + let parsed = parse(input); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors, + ); + let map = rewrite(input, &parsed.document); + // Apply in reverse span order to preserve earlier byte + // offsets. + let mut entries: Vec<_> = map.into_iter().collect(); + entries.sort_by_key(|(s, _)| std::cmp::Reverse(s.start)); + let mut out = input.to_string(); + for (span, replacement) in entries { + out.replace_range( + span.start as usize..span.end as usize, + &replacement, + ); + } + out + } + + #[test] + fn typed_var_dispatches_through_repr() { + // `configure_x` is annotated → return type flows to `x` + // via `set x [configure_x]` → `putr $x` sees x's type. + let src = "\ +proc configure_x {} MyType { return foo } +set x [configure_x] +putr $x +"; + let out = rewrite_str(src); + // MyType::repr with the -v envelope. + assert!( + out.contains("puts [MyType::repr -v $x]"), + "expected MyType::repr dispatch, got:\n{out}", + ); + // Original putr line is gone. + assert!(!out.contains("putr $x"), "putr $x still present:\n{out}"); + } + + #[test] + fn untyped_var_falls_back_to_plain_puts() { + // No proc annotation → var type unknown → plain `puts $x`. + let src = "\ +proc make_something {} { return foo } +set x [make_something] +putr $x +"; + let out = rewrite_str(src); + assert!(out.contains("puts $x"), "expected plain puts, got:\n{out}",); + // The rewrite should NOT have introduced a repr dispatch. + assert!( + !out.contains("::repr -v $x"), + "unexpected repr dispatch for untyped var:\n{out}", + ); + } + + #[test] + fn inline_proc_call_uses_return_type() { + // `putr [make]` — no intermediate binding; the arg is a + // direct `[proc-call]`, `value_type` sees the return type. + let src = "\ +proc make {} MyType { return foo } +putr [make] +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v [make]]"), + "expected inline dispatch, got:\n{out}", + ); + } + + #[test] + fn inside_proc_body_uses_arg_type() { + // Proc parameter with a type annotation → visible to the + // walker's per-proc scope frame. + let src = "\ +proc show { v: MyType } { + putr $v +} +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v $v]"), + "expected in-body dispatch, got:\n{out}", + ); + } + + #[test] + fn wrong_arity_left_alone() { + // `putr` with zero args — outside the rewrite's target + // shape, we leave the source untouched. Analyzer will + // handle the arity complaint through its normal path. + let src = "putr\n"; + let out = rewrite_str(src); + assert_eq!(out, src); + } + + #[test] + fn unannotated_proc_return_type_inferred_from_body() { + // `wrapper` has no return-type annotation, but its body + // ends with `return $inner` where `inner` was set from a + // typed proc. The rewrite should still resolve + // `wrapper`'s return type via body inference, then flow + // it into `$x`, then dispatch `putr $x` through the + // right repr — the specific pattern that made + // `putr $_gtm` fall to plain puts before this fix. + let src = "\ +proc typed_ctor {} MyType { return foo } +proc wrapper {} { + set inner [typed_ctor] + return $inner +} +set x [wrapper] +putr $x +"; + let out = rewrite_str(src); + assert!( + out.contains("puts [MyType::repr -v $x]"), + "expected inferred MyType dispatch, got:\n{out}", + ); + } +} diff --git a/vw-htcl/src/references.rs b/vw-htcl/src/references.rs new file mode 100644 index 0000000..a005bf2 --- /dev/null +++ b/vw-htcl/src/references.rs @@ -0,0 +1,1324 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Symbol-references core. +//! +//! The vw-htcl side of `textDocument/references` and +//! `textDocument/rename`. Splits the problem into two orthogonal +//! pieces: +//! +//! 1. **[`identify_at`]** — given a cursor `offset`, figure out what +//! the user is pointing at. Returns a [`ReferenceTarget`] that +//! names the symbol in a way independent of the source file it +//! was found in. +//! 2. **[`find_references_in`]** — given a target, walk a document +//! and return every source span that is a use or a decl of that +//! target. +//! +//! Both procs and types cross file boundaries in real workspaces, +//! so the LSP layer typically calls (1) on the file the cursor is +//! in and (2) on every `.htcl` file under the workspace root. The +//! two functions never talk to each other — the target flows in as +//! a plain value. +//! +//! Locals and proc args have file-local scope by construction, so +//! `find_references_in` returns the empty set for them when passed +//! a document that doesn't contain the declaration. The LSP layer +//! uses this to skip the cross-file scan for local kinds. + +use crate::ast::{ + AttributeValue, Command, CommandKind, Document, Proc, ProcSignature, Stmt, + TypeExpr, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::scope::{resolve_var_def, scan_var_ref, VarDef}; +use crate::span::Span; + +/// A symbol whose references we want to find. Kinds carry the +/// identifying data needed to match uses across files (procs and +/// types by qualified name) or to bound the scope to a single +/// declaration (locals and proc args). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReferenceTarget { + /// A proc named exactly `name` — either bare (`configure_gtm`) or + /// namespaced (`vivado_cmd::create_bd_cell`). Matching is + /// literal on the head-word text of every command, plus the + /// name-word of every `proc` decl. Cross-file. + Proc { name: String }, + /// A `type NAME = …` declaration and every `: NAME` / + /// `Generic` reference. `name` is the declared (possibly + /// qualified) form. Cross-file. + Type { name: String }, + /// A single variant of an enum. Referred to as + /// `::` in qualified type annotations and as + /// `::` at construction sites. Cross-file. + EnumVariant { enum_name: String, variant: String }, + /// A `set VAR …` / `variable VAR` / `foreach VAR …` / `upvar + /// … LOCAL` inside a specific scope. File-local by + /// construction. `decl_scope_span` is the span that bounds the + /// scope (a proc body's span, or the whole document for + /// top-level locals) so that scans in OTHER files return the + /// empty set instead of accidentally matching a same-name + /// local elsewhere. + Local { name: String, decl_scope_span: Span }, + /// A proc-arg parameter. Emitted when the cursor is on the arg + /// itself, on a body-level `$name` reference, or on an + /// attribute-ident value that names the arg. File-local by + /// the same reasoning as [`Local`]. + ProcArg { + proc_name: Option, + arg_name: String, + /// Span of the enclosing proc body, used to bound the ref + /// scan the same way `Local` does. + decl_scope_span: Span, + }, +} + +/// If `offset` lands on something we can identify as a reference +/// target, return it. Order of tries is narrowest-first so the +/// less-specific fallbacks don't misclassify. +/// +/// Returns `None` when the cursor isn't on an identifier we know +/// how to track (whitespace, comment interior, keyword, arbitrary +/// argument text, etc.). +pub fn identify_at( + document: &Document, + source: &str, + offset: u32, +) -> Option { + // 1. Proc-arg identification. Narrowest by construction — + // only fires when the cursor is on an arg-name-shaped word + // inside a proc signature, an attr ident referring to a + // sibling arg, or a `$name` in a body resolving to an arg. + if let Some((proc, arg_name)) = + find_proc_arg_at(&document.stmts, source, offset) + { + return Some(ReferenceTarget::ProcArg { + proc_name: proc.name.clone(), + arg_name, + decl_scope_span: proc.body_span, + }); + } + // 2. Enum-variant reference (`Enum::Variant` in a type + // annotation, or `Enum::Variant` in a construction call + // like `Enum::Variant -payload $x`). Narrower than plain + // Proc because a Qualified type annotation is only + // parsed that way when the AST tagged it as such. + if let Some(t) = identify_enum_variant_at(document, offset) { + return Some(t); + } + // 3. Type reference or type decl. Fires on `type NAME = …` + // name spans and on any Named/Generic type-expression + // name-span in a decl or annotation. + if let Some(t) = identify_type_at(document, offset) { + return Some(t); + } + // 4. Proc reference or proc decl. Fires on the name span of + // a `proc` decl and on the head word of any Command. + if let Some(t) = identify_proc_at(document, offset) { + return Some(t); + } + // 5. Local variable — `set`/`variable`/`foreach`/`upvar` decl + // spans, and `$name` references in the enclosing scope. + if let Some(t) = identify_local_at(document, source, offset) { + return Some(t); + } + None +} + +/// Walk `document` collecting every span that references +/// `target`. Includes both use sites and (for cross-file kinds) +/// the declaration span itself so the rename pipeline gets a +/// single unified list. +/// +/// For file-local kinds (`Local`, `ProcArg`), returns the empty +/// set when the document doesn't contain the declaration — +/// callers use this to skip the cross-file scan. +pub fn find_references_in( + document: &Document, + source: &str, + target: &ReferenceTarget, +) -> Vec { + let mut out = Vec::new(); + match target { + ReferenceTarget::Proc { name } => { + find_proc_refs(&document.stmts, name, &mut out); + } + ReferenceTarget::Type { name } => { + find_type_refs(&document.stmts, name, &mut out); + } + ReferenceTarget::EnumVariant { enum_name, variant } => { + find_enum_variant_refs( + &document.stmts, + enum_name, + variant, + &mut out, + ); + } + ReferenceTarget::Local { + name, + decl_scope_span, + } => { + // Only scan if the scope is present in THIS document. + // `decl_scope_span` is an absolute-source offset from + // whatever document defined the local; a different + // file would have different byte offsets and this + // check would fail — which is the intent (file-local + // kinds don't leak across files). + if let Some(scope_stmts) = + scope_stmts_by_span(document, *decl_scope_span) + { + collect_local_decl_spans(scope_stmts, name, &mut out); + collect_var_ref_spans(scope_stmts, source, name, &mut out); + } + } + ReferenceTarget::ProcArg { + arg_name, + decl_scope_span, + .. + } => { + if let Some(proc) = + find_proc_by_body_span(&document.stmts, *decl_scope_span) + { + if let Some(sig) = &proc.signature { + if let Some(arg) = + sig.args.iter().find(|a| &a.name == arg_name) + { + out.push(arg.name_span); + } + for attr_span in attribute_ident_ref_spans(sig, arg_name) { + out.push(attr_span); + } + } + collect_var_ref_spans(&proc.body, source, arg_name, &mut out); + } + } + } + out.sort_by_key(|s| (s.start, s.end)); + out.dedup(); + out +} + +// ─── identify_at helpers ──────────────────────────────────────────── + +fn identify_proc_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on a `proc NAME { … }` decl? + if let Some(name) = proc_decl_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Proc { name }); + } + // Cursor on a command's head word (any call)? + if let Some(name) = command_head_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Proc { name }); + } + None +} + +fn proc_decl_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = &proc.name { + if proc.name_span.contains(offset) { + return Some(name.clone()); + } + } + if let Some(name) = proc_decl_name_at(&proc.body, offset) { + return Some(name); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(name) = proc_decl_name_at(&ns.body, offset) { + return Some(name); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(name) = proc_decl_name_at(body, offset) { + return Some(name); + } + } + } + } + } + None +} + +fn command_head_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let CommandKind::Generic = cmd.kind { + if let Some(head) = cmd.words.first() { + if head.span.contains(offset) { + if let Some(t) = head.as_text() { + return Some(t.to_string()); + } + } + } + } + // Recurse for nested calls / bodies. + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(n) = command_head_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = command_head_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(n) = command_head_name_at(body, offset) { + return Some(n); + } + } + } + } + } + None +} + +fn identify_type_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on a `type NAME = …` decl name? + if let Some(name) = type_decl_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Type { name }); + } + // Cursor on a type-expression name (annotation or nested)? + if let Some(name) = type_expr_name_at(&document.stmts, offset) { + return Some(ReferenceTarget::Type { name }); + } + None +} + +fn type_decl_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + if let Some(name) = &td.name { + if td.name_span.contains(offset) { + return Some(name.clone()); + } + } + } + CommandKind::Proc(proc) => { + if let Some(n) = type_decl_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = type_decl_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + } + None +} + +fn type_expr_name_at(stmts: &[Stmt], offset: u32) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + if let Some(n) = type_expr_name_span_hit(ty, offset) + { + return Some(n); + } + } + } + } + if let Some(ty) = &proc.return_type { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + if let Some(n) = type_expr_name_at(&proc.body, offset) { + return Some(n); + } + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = &td.underlying { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + } + CommandKind::EnumDecl(ed) => { + for variant in &ed.variants { + if let Some(ty) = &variant.payload { + if let Some(n) = type_expr_name_span_hit(ty, offset) { + return Some(n); + } + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(n) = type_expr_name_at(&ns.body, offset) { + return Some(n); + } + } + _ => {} + } + } + None +} + +/// If `offset` lands on the name portion of `ty`, return that +/// name. Recurses through generic args. For `Qualified` we +/// deliberately return `None` — those are handled by the +/// enum-variant identifier, which is more specific. +fn type_expr_name_span_hit(ty: &TypeExpr, offset: u32) -> Option { + match ty { + TypeExpr::Named { name, span } => { + if span.contains(offset) { + Some(name.clone()) + } else { + None + } + } + TypeExpr::Generic { + name, + name_span, + args, + .. + } => { + if name_span.contains(offset) { + return Some(name.clone()); + } + for a in args { + if let Some(n) = type_expr_name_span_hit(a, offset) { + return Some(n); + } + } + None + } + TypeExpr::Qualified { .. } => None, + } +} + +fn identify_enum_variant_at( + document: &Document, + offset: u32, +) -> Option { + // Cursor on an enum-decl variant name? + if let Some(t) = enum_decl_variant_at(&document.stmts, offset) { + return Some(t); + } + // Cursor on a `Qualified{ns, variant}` type annotation in a + // proc arg (overload-arm shape)? + if let Some(t) = qualified_type_variant_at(&document.stmts, offset) { + return Some(t); + } + // Cursor on a construction call `Enum::Variant -payload …` + // — matched purely by name shape (`NAME::NAME`) in a command + // head. Only recognized when there's a matching enum decl + // anywhere in the document. + if let Some(t) = enum_construct_head_at(document, offset) { + return Some(t); + } + None +} + +fn enum_decl_variant_at( + stmts: &[Stmt], + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) => { + let Some(enum_name) = &ed.name else { continue }; + for variant in &ed.variants { + if variant.name_span.contains(offset) { + return Some(ReferenceTarget::EnumVariant { + enum_name: enum_name.clone(), + variant: variant.name.clone(), + }); + } + } + } + CommandKind::Proc(proc) => { + if let Some(t) = enum_decl_variant_at(&proc.body, offset) { + return Some(t); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(t) = enum_decl_variant_at(&ns.body, offset) { + return Some(t); + } + } + _ => {} + } + } + None +} + +fn qualified_type_variant_at( + stmts: &[Stmt], + offset: u32, +) -> Option { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(TypeExpr::Qualified { + namespace, + variant, + span, + .. + }) = &arg.type_annotation + { + if span.contains(offset) { + return Some(ReferenceTarget::EnumVariant { + enum_name: namespace.clone(), + variant: variant.clone(), + }); + } + } + } + } + if let Some(t) = qualified_type_variant_at(&proc.body, offset) { + return Some(t); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(t) = qualified_type_variant_at(&ns.body, offset) { + return Some(t); + } + } + _ => {} + } + } + None +} + +fn enum_construct_head_at( + document: &Document, + offset: u32, +) -> Option { + let (name, span) = command_head_qualified_at(&document.stmts, offset)?; + // Parse as `Enum::Variant` — first-level namespace only. + let (ns, variant) = split_first_scope(&name)?; + // Only accept if there's a matching enum decl in the doc. + if !enum_decl_exists(&document.stmts, ns) { + return None; + } + // `span` was the whole head-word; the cursor lands on the + // string covered by it, so ownership by variant vs namespace + // is captured by the "any part of the compound" rule the + // caller wanted. Return the variant identity either way. + let _ = span; + Some(ReferenceTarget::EnumVariant { + enum_name: ns.to_string(), + variant: variant.to_string(), + }) +} + +fn command_head_qualified_at( + stmts: &[Stmt], + offset: u32, +) -> Option<(String, Span)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if let CommandKind::Generic = cmd.kind { + if let Some(head) = cmd.words.first() { + if head.span.contains(offset) { + if let Some(t) = head.as_text() { + if t.contains("::") { + return Some((t.to_string(), head.span)); + } + } + } + } + } + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(x) = command_head_qualified_at(&proc.body, offset) { + return Some(x); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(x) = command_head_qualified_at(&ns.body, offset) { + return Some(x); + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + if let Some(x) = command_head_qualified_at(body, offset) { + return Some(x); + } + } + } + } + } + None +} + +fn split_first_scope(name: &str) -> Option<(&str, &str)> { + let (ns, rest) = name.split_once("::")?; + // Only match single-segment variants: `Enum::Variant`, not + // `Enum::Nested::Something`. + if rest.contains("::") { + return None; + } + Some((ns, rest)) +} + +fn enum_decl_exists(stmts: &[Stmt], name: &str) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) if ed.name.as_deref() == Some(name) => { + return true; + } + CommandKind::Proc(proc) if enum_decl_exists(&proc.body, name) => { + return true; + } + CommandKind::NamespaceEval(ns) + if enum_decl_exists(&ns.body, name) => + { + return true; + } + _ => {} + } + } + false +} + +fn identify_local_at( + document: &Document, + source: &str, + offset: u32, +) -> Option { + let (scope_stmts, scope_span, enclosing) = + innermost_scope(document, offset)?; + // On a decl target? + if let Some(name) = local_decl_name_at(scope_stmts, offset) { + return Some(ReferenceTarget::Local { + name: name.to_string(), + decl_scope_span: scope_span, + }); + } + // On a `$var` that resolves to a local? + let name = var_ref_name_in_scope(scope_stmts, source, offset)?; + match resolve_var_def(&name, scope_stmts, enclosing, offset)? { + VarDef::Local(_) => Some(ReferenceTarget::Local { + name, + decl_scope_span: scope_span, + }), + VarDef::Param(_) => None, + } +} + +// ─── find_references_in helpers ───────────────────────────────────── + +fn find_proc_refs(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.name.as_deref() == Some(name) { + out.push(proc.name_span); + } + find_proc_refs(&proc.body, name, out); + } + CommandKind::NamespaceEval(ns) => { + find_proc_refs(&ns.body, name, out); + } + CommandKind::Generic => { + if let Some(head) = cmd.words.first() { + if head.as_text() == Some(name) { + out.push(head.span); + } + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + find_proc_refs(body, name, out); + } + } + } + } +} + +fn find_type_refs(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + if td.name.as_deref() == Some(name) { + out.push(td.name_span); + } + if let Some(ty) = &td.underlying { + collect_type_expr_name_spans(ty, name, out); + } + } + CommandKind::EnumDecl(ed) => { + for variant in &ed.variants { + if let Some(ty) = &variant.payload { + collect_type_expr_name_spans(ty, name, out); + } + } + } + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + collect_type_expr_name_spans(ty, name, out); + } + } + } + if let Some(ty) = &proc.return_type { + collect_type_expr_name_spans(ty, name, out); + } + find_type_refs(&proc.body, name, out); + } + CommandKind::NamespaceEval(ns) => { + find_type_refs(&ns.body, name, out); + } + _ => {} + } + } +} + +fn collect_type_expr_name_spans( + ty: &TypeExpr, + name: &str, + out: &mut Vec, +) { + match ty { + TypeExpr::Named { name: n, span } => { + if n == name { + out.push(*span); + } + } + TypeExpr::Generic { + name: n, + name_span, + args, + .. + } => { + if n == name { + out.push(*name_span); + } + for a in args { + collect_type_expr_name_spans(a, name, out); + } + } + TypeExpr::Qualified { .. } => {} + } +} + +fn find_enum_variant_refs( + stmts: &[Stmt], + enum_name: &str, + variant: &str, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) + if ed.name.as_deref() == Some(enum_name) => + { + for v in &ed.variants { + if v.name == variant { + out.push(v.name_span); + } + } + } + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(TypeExpr::Qualified { + namespace, + variant: v2, + span, + .. + }) = &arg.type_annotation + { + if namespace == enum_name && v2 == variant { + out.push(*span); + } + } + } + } + find_enum_variant_refs(&proc.body, enum_name, variant, out); + } + CommandKind::NamespaceEval(ns) => { + find_enum_variant_refs(&ns.body, enum_name, variant, out); + } + CommandKind::Generic => { + if let Some(head) = cmd.words.first() { + let expected = format!("{enum_name}::{variant}"); + if head.as_text() == Some(&expected) { + out.push(head.span); + } + } + } + _ => {} + } + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + find_enum_variant_refs(body, enum_name, variant, out); + } + } + } + } +} + +// ─── local + proc-arg helpers, adapted from rename.rs ────────────── + +/// Walk the AST looking for a scope whose enclosing span +/// contains `offset`. Returns the statements list, the scope's +/// bounding span, and the enclosing proc when nested inside one +/// (for arg-name resolution). +fn innermost_scope( + document: &Document, + offset: u32, +) -> Option<(&[Stmt], Span, Option<&Proc>)> { + // Try each proc body's stmts, deepest-first. Fall back to + // the document top level. + fn inner<'a>( + stmts: &'a [Stmt], + _top_span: Span, + offset: u32, + enclosing: Option<&'a Proc>, + ) -> Option<(&'a [Stmt], Span, Option<&'a Proc>)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) if proc.body_span.contains(offset) => { + return inner( + &proc.body, + proc.body_span, + offset, + Some(proc), + ) + .or(Some(( + &proc.body, + proc.body_span, + Some(proc), + ))); + } + CommandKind::NamespaceEval(ns) + if ns.body_span.contains(offset) => + { + return inner(&ns.body, ns.body_span, offset, enclosing) + .or(Some((&ns.body, ns.body_span, enclosing))); + } + _ => {} + } + } + None + } + let doc_span = Span::new(0, u32::MAX); + inner(&document.stmts, doc_span, offset, None).or(Some(( + &document.stmts, + doc_span, + None, + ))) +} + +/// Return the stmts of the scope whose bounding span equals +/// `scope_span`. `Span::new(0, u32::MAX)` means "the document +/// top level." +fn scope_stmts_by_span( + document: &Document, + scope_span: Span, +) -> Option<&[Stmt]> { + if scope_span.start == 0 && scope_span.end == u32::MAX { + return Some(&document.stmts); + } + scope_stmts_by_span_in(&document.stmts, scope_span) +} + +fn scope_stmts_by_span_in(stmts: &[Stmt], scope_span: Span) -> Option<&[Stmt]> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.body_span == scope_span { + return Some(&proc.body); + } + if let Some(s) = scope_stmts_by_span_in(&proc.body, scope_span) + { + return Some(s); + } + } + CommandKind::NamespaceEval(ns) => { + if ns.body_span == scope_span { + return Some(&ns.body); + } + if let Some(s) = scope_stmts_by_span_in(&ns.body, scope_span) { + return Some(s); + } + } + _ => {} + } + } + None +} + +fn find_proc_by_body_span(stmts: &[Stmt], body_span: Span) -> Option<&Proc> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if proc.body_span == body_span { + return Some(proc); + } + if let Some(p) = find_proc_by_body_span(&proc.body, body_span) { + return Some(p); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(p) = find_proc_by_body_span(&ns.body, body_span) { + return Some(p); + } + } + _ => {} + } + } + None +} + +/// Reuse of [`crate::rename`]'s proc-arg identification without +/// borrowing its private helpers. +fn find_proc_arg_at<'a>( + stmts: &'a [Stmt], + source: &str, + offset: u32, +) -> Option<(&'a Proc, String)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + if proc.args_span.contains(offset) { + for arg in &sig.args { + if arg.name_span.contains(offset) { + return Some((proc, arg.name.clone())); + } + for a in &arg.attributes { + for v in &a.values { + if let AttributeValue::Ident { + value, + span, + } = v + { + if span.contains(offset) + && sig + .args + .iter() + .any(|x| &x.name == value) + { + return Some((proc, value.clone())); + } + } + } + } + } + } + if proc.body_span.contains(offset) { + if let Some(name) = + var_ref_name_in(&proc.body, source, offset) + { + if sig.args.iter().any(|a| a.name == name) { + return Some((proc, name)); + } + } + } + } + if let Some(x) = find_proc_arg_at(&proc.body, source, offset) { + return Some(x); + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(x) = find_proc_arg_at(&ns.body, source, offset) { + return Some(x); + } + } + _ => {} + } + } + None +} + +fn var_ref_name_in( + _stmts: &[Stmt], + source: &str, + offset: u32, +) -> Option { + scan_var_ref(source, offset).map(|(n, _)| n) +} + +fn var_ref_name_in_scope( + _stmts: &[Stmt], + source: &str, + offset: u32, +) -> Option { + scan_var_ref(source, offset).map(|(n, _)| n) +} + +fn local_decl_name_at(stmts: &[Stmt], offset: u32) -> Option<&str> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let words = &cmd.words; + let Some(head) = words.first().and_then(|w| w.as_text()) else { + continue; + }; + // set / variable / foreach / upvar — same rules as rename.rs. + match head { + "set" | "variable" | "foreach" => { + if let Some(w) = words.get(1) { + if w.form == WordForm::Bare && w.span.contains(offset) { + if let Some(t) = w.as_text() { + return Some(t); + } + } + } + } + "upvar" => { + // `upvar [LEVEL] remote local [remote local]…` + // Skip an optional leading level (`#N` or digits), + // then walk pairs — cursor on the LOCAL of any + // pair identifies that local as the target. + let mut idx = 1; + if let Some(w) = words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + while idx + 1 < words.len() { + let local_word = &words[idx + 1]; + if local_word.span.contains(offset) { + if let Some(t) = local_word.as_text() { + return Some(t); + } + } + idx += 2; + } + } + _ => {} + } + if let CommandKind::Proc(proc) = &cmd.kind { + if let Some(n) = local_decl_name_at(&proc.body, offset) { + return Some(n); + } + } + if is_body_host(head) { + for word in words.iter().skip(1) { + if let WordForm::Braced = word.form { + if word.span.contains(offset) { + // We don't re-parse braced bodies here — + // rename.rs's local pass already covers + // that via its own reparse. Skip. + } + } + } + } + } + None +} + +fn collect_local_decl_spans(stmts: &[Stmt], name: &str, out: &mut Vec) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Skip nested scopes — locals don't cross. + if matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + continue; + } + let Some(head) = cmd.words.first().and_then(|w| w.as_text()) else { + continue; + }; + match head { + "set" | "variable" | "foreach" => { + if let Some(w) = cmd.words.get(1) { + if w.form == WordForm::Bare && w.as_text() == Some(name) { + out.push(w.span); + } + } + } + "upvar" => { + let mut idx = 1; + if let Some(w) = cmd.words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + while idx + 1 < cmd.words.len() { + let local_word = &cmd.words[idx + 1]; + if local_word.as_text() == Some(name) { + out.push(local_word.span); + } + idx += 2; + } + } + _ => {} + } + // Body-host commands (if/while/foreach/…) — descend into + // their braced bodies. They run in the SAME frame, so a + // `set foo` inside an `if` body is the enclosing scope's + // local, not a separate scope. + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner_stmts) = + crate::unused::reparse_braced_body(word, "") + { + collect_local_decl_spans(&inner_stmts, name, out); + } + } + } + } +} + +fn collect_var_ref_spans( + stmts: &[Stmt], + source: &str, + name: &str, + out: &mut Vec, +) { + walk_var_ref_spans(stmts, source, name, out); +} + +fn walk_var_ref_spans( + stmts: &[Stmt], + source: &str, + name: &str, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Skip nested scopes. + if matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + continue; + } + for word in &cmd.words { + walk_var_ref_spans_in_word(word, source, name, out); + } + // Descend into body-host braced bodies (if/while/foreach/…). + if let Some(head) = cmd.words.first().and_then(|w| w.as_text()) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner_stmts) = + crate::unused::reparse_braced_body(word, source) + { + walk_var_ref_spans(&inner_stmts, source, name, out); + } + } + } + } + } +} + +fn walk_var_ref_spans_in_word( + word: &Word, + source: &str, + name: &str, + out: &mut Vec, +) { + for part in &word.parts { + match part { + WordPart::VarRef { name: n, span, .. } => { + if n == name { + // Span covers `$name`; the target is the + // identifier portion (skip the leading `$`). + out.push(Span::new(span.start + 1, span.end)); + } + } + WordPart::CmdSubst { body, .. } => { + walk_var_ref_spans(body, source, name, out); + } + WordPart::Text { .. } | WordPart::Escape { .. } => {} + } + } +} + +fn attribute_ident_ref_spans(sig: &ProcSignature, arg_name: &str) -> Vec { + let mut out = Vec::new(); + for arg in &sig.args { + for a in &arg.attributes { + for v in &a.values { + if let AttributeValue::Ident { value, span } = v { + if value == arg_name { + out.push(*span); + } + } + } + } + } + out +} + +// Sanity: the AST re-exports we need for the pattern matches above. +#[allow(dead_code)] +fn _unused_shape_check(w: &Word, _s: &str, _c: &Command) { + let _ = w.form; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn parsed(src: &str) -> crate::ast::Document { + parse(src).document + } + + fn find_offset(src: &str, needle: &str) -> u32 { + src.find(needle).unwrap() as u32 + } + + #[test] + fn identify_proc_on_decl_name() { + let src = "proc configure_gtm {} { }\n"; + let d = parsed(src); + let offset = find_offset(src, "configure_gtm"); + let target = identify_at(&d, src, offset).expect("identified"); + assert_eq!( + target, + ReferenceTarget::Proc { + name: "configure_gtm".into() + } + ); + } + + #[test] + fn identify_proc_on_call_site() { + let src = "proc configure_gtm {} { }\nconfigure_gtm\n"; + let d = parsed(src); + // Cursor on the CALL of configure_gtm (after the decl). + let offset = src.rfind("configure_gtm").unwrap() as u32; + let target = identify_at(&d, src, offset).expect("identified"); + assert_eq!( + target, + ReferenceTarget::Proc { + name: "configure_gtm".into() + } + ); + } + + #[test] + fn find_proc_refs_covers_decl_and_calls() { + let src = "\ +proc configure_gtm {} { } +configure_gtm +proc other {} { configure_gtm } +"; + let d = parsed(src); + let target = ReferenceTarget::Proc { + name: "configure_gtm".into(), + }; + let refs = find_references_in(&d, src, &target); + // 3 hits: decl name + top-level call + nested call in `other`. + assert_eq!(refs.len(), 3, "spans: {refs:?}"); + } + + #[test] + fn identify_type_on_decl_and_annotation() { + let src = "\ +type MyThing = string +proc use_it {v: MyThing} { } +"; + let d = parsed(src); + // Cursor on the decl name. + let offset_decl = find_offset(src, "MyThing"); + assert_eq!( + identify_at(&d, src, offset_decl), + Some(ReferenceTarget::Type { + name: "MyThing".into() + }) + ); + // Cursor on the annotation. + let offset_ann = src.rfind("MyThing").unwrap() as u32; + assert_eq!( + identify_at(&d, src, offset_ann), + Some(ReferenceTarget::Type { + name: "MyThing".into() + }) + ); + } + + #[test] + fn find_type_refs_covers_decl_and_annotations() { + let src = "\ +type MyThing = string +proc a {v: MyThing} MyThing { } +proc b {v: MyThing} { } +"; + let d = parsed(src); + let target = ReferenceTarget::Type { + name: "MyThing".into(), + }; + let refs = find_references_in(&d, src, &target); + // decl + a's arg-type + a's return-type + b's arg-type = 4. + assert_eq!(refs.len(), 4, "spans: {refs:?}"); + } + + #[test] + fn identify_enum_variant_on_decl_variant() { + let src = "\ +enum Color = { + Red + Green +} +"; + let d = parsed(src); + let offset = find_offset(src, "Red"); + assert_eq!( + identify_at(&d, src, offset), + Some(ReferenceTarget::EnumVariant { + enum_name: "Color".into(), + variant: "Red".into(), + }) + ); + } +} diff --git a/vw-htcl/src/rename.rs b/vw-htcl/src/rename.rs new file mode 100644 index 0000000..f42d423 --- /dev/null +++ b/vw-htcl/src/rename.rs @@ -0,0 +1,441 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Local-scope rename. +//! +//! `textDocument/rename` for identifiers whose scope is confined to +//! the current file. Explicitly in scope: +//! +//! - proc parameters (rename the decl in the signature, every `$name` +//! in the body, and any attribute-ident value that names the arg — +//! e.g. `@requires(name)`) +//! - `set NAME value` locals +//! - `variable NAME` locals +//! - `foreach ITER $list { … }` iterators +//! - `upvar [LEVEL] remote LOCAL` locals +//! +//! Explicitly OUT of scope (returns `None`): +//! +//! - proc names, type names, enum names — renaming these would break +//! call sites we can't see from a single file, so refuse rather +//! than emit an incomplete edit set. +//! - proc-arg **flag references** at call sites (`caller -oldname …`) — +//! same reason: cross-file. The user renaming a proc arg only gets +//! the body-local rename; call sites keep the old flag name until +//! they're touched manually. +//! +//! The cursor is allowed on any of: the decl itself, a `$name` +//! reference to it, or an attribute-ident value referring to a +//! sibling arg. Each maps to the same rename operation. + +use crate::ast::Document; +use crate::span::Span; + +/// A single text-substitution edit to apply. Spans are absolute in +/// the source we were given. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameEdit { + pub span: Span, + pub new_text: String, +} + +/// Compute the edits needed to rename the identifier at `offset` to +/// `new_name`. Returns `None` when: +/// +/// - `new_name` isn't a valid Tcl identifier +/// - The cursor isn't on something we know how to rename locally +/// - The cursor is on a construct whose rename would leak beyond the +/// current file (proc names, types, etc.) +/// +/// The returned edits are sorted by span start and deduplicated so +/// clients can apply them as-is. +pub fn rename_at( + document: &Document, + source: &str, + offset: u32, + new_name: &str, +) -> Option> { + if !is_valid_tcl_ident_or_qualified(new_name) { + return None; + } + // Route through the [`crate::references`] core so proc / + // type / enum-variant renames flow through the same + // identify + collect pipeline as `textDocument/references`. + // Locals + proc args are still file-local; the LSP layer + // decides whether to also scan sibling files. + let target = crate::references::identify_at(document, source, offset)?; + let spans = + crate::references::find_references_in(document, source, &target); + if spans.is_empty() { + return None; + } + let replacement = replacement_for(&target, new_name); + let edits = spans + .into_iter() + .map(|span| RenameEdit { + span, + new_text: replacement.clone(), + }) + .collect(); + Some(finalize_edits(edits)) +} + +/// Pick the exact text to substitute at each ref span for a +/// given target. Straightforward for locals / proc args / type +/// names (the new bare name goes in verbatim). Procs preserve +/// their namespace prefix so a rename of a call site like +/// `vivado_cmd::create_bd_cell` targets the LAST segment only +/// when the user typed a bare name. +fn replacement_for( + target: &crate::references::ReferenceTarget, + new_name: &str, +) -> String { + use crate::references::ReferenceTarget; + match target { + ReferenceTarget::Proc { name } + if name.contains("::") && !new_name.contains("::") => + { + // Preserve the namespace prefix from the original. + let ns_prefix = + name.rsplit_once("::").map(|(ns, _)| ns).unwrap_or(""); + format!("{ns_prefix}::{new_name}") + } + ReferenceTarget::EnumVariant { enum_name, .. } => { + // Enum variant refs span the whole `Enum::Variant` + // form in call-head/qualified positions, so we + // preserve the enum prefix. + format!("{enum_name}::{new_name}") + } + _ => new_name.to_string(), + } +} + +/// A rename target's new text is either a plain identifier or a +/// fully qualified path (`ns::name`). Reject anything that +/// doesn't parse as one of those so the substituted text stays +/// valid htcl. +fn is_valid_tcl_ident_or_qualified(s: &str) -> bool { + if s.is_empty() { + return false; + } + for seg in s.split("::") { + if seg.is_empty() { + return false; + } + if !is_valid_tcl_ident(seg) { + return false; + } + } + true +} + +/// Tcl identifiers accept letters, digits, underscore, and `::` +/// (namespace separator). For rename we only allow the first three: +/// renaming across a namespace boundary changes visibility rules, +/// which is outside "local rename" semantics. +fn is_valid_tcl_ident(s: &str) -> bool { + if s.is_empty() { + return false; + } + let mut bytes = s.bytes(); + let first = bytes.next().unwrap(); + if !(first.is_ascii_alphabetic() || first == b'_') { + return false; + } + bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +fn finalize_edits(mut edits: Vec) -> Vec { + edits.sort_by_key(|e| (e.span.start, e.span.end)); + edits.dedup_by(|a, b| a.span == b.span && a.new_text == b.new_text); + edits +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn at(src: &str, needle: &str, occurrence: usize) -> u32 { + let mut start = 0; + for i in 0..=occurrence { + let pos = src[start..] + .find(needle) + .map(|p| start + p) + .expect("needle not found enough times"); + if i == occurrence { + return pos as u32; + } + start = pos + needle.len(); + } + unreachable!() + } + + /// Apply edits and return the resulting source. + fn apply(src: &str, edits: &[RenameEdit]) -> String { + let mut out = src.to_string(); + for edit in edits.iter().rev() { + let s = edit.span.start as usize; + let e = edit.span.end as usize; + out.replace_range(s..e, &edit.new_text); + } + out + } + + fn edits_of(src: &str, pos: u32, new_name: &str) -> Vec { + let parsed = parse(src); + rename_at(&parsed.document, src, pos, new_name).unwrap_or_default() + } + + #[test] + fn rename_set_local_from_decl_position() { + let src = "\ +proc f {} { + set mode fast + puts $mode + return $mode +} +"; + let pos = at(src, "mode", 0); // the `mode` in `set mode` + let edits = edits_of(src, pos, "kind"); + assert!(!edits.is_empty(), "no edits produced"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + assert!(!renamed.contains("mode"), "{renamed}"); + } + + #[test] + fn rename_set_local_from_var_ref_position() { + let src = "\ +proc f {} { + set mode fast + puts $mode +} +"; + // Cursor on the `m` inside `$mode`. + let pos = at(src, "$mode", 0) + 1; + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_from_decl() { + let src = "\ +proc f { + mode +} { + puts $mode + return $mode +} +"; + let pos = at(src, "mode", 0); // arg decl + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + // Arg decl updated + body refs updated. + assert!(renamed.contains(" kind"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + assert!(renamed.contains("return $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_from_body_var_ref() { + let src = "\ +proc f { + mode +} { + puts $mode +} +"; + let pos = at(src, "$mode", 0) + 1; + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains(" kind"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_proc_arg_updates_attribute_ident() { + // `@requires(has_a)` in the sig references the sibling arg + // by name; renaming `has_a` should update that reference. + let src = "\ +proc f { + has_a + @requires(has_a) has_b +} { + puts $has_a +} +"; + let pos = at(src, "has_a", 0); // decl of has_a + let edits = edits_of(src, pos, "has_x"); + let renamed = apply(src, &edits); + assert!(renamed.contains(" has_x\n"), "{renamed}"); + assert!(renamed.contains("@requires(has_x)"), "{renamed}"); + assert!(renamed.contains("puts $has_x"), "{renamed}"); + } + + #[test] + fn rename_foreach_iterator() { + let src = "\ +proc f {} { + foreach item [list 1 2 3] { + puts $item + } +} +"; + let pos = at(src, "item", 0); // cursor on iter decl + let edits = edits_of(src, pos, "elem"); + let renamed = apply(src, &edits); + assert!(renamed.contains("foreach elem "), "{renamed}"); + assert!(renamed.contains("puts $elem"), "{renamed}"); + } + + #[test] + fn rename_upvar_local() { + let src = "\ +proc f {} { + upvar 1 remote local + puts $local +} +"; + let pos = at(src, "local", 0); // upvar's local half + let edits = edits_of(src, pos, "here"); + let renamed = apply(src, &edits); + assert!(renamed.contains("upvar 1 remote here"), "{renamed}"); + assert!(renamed.contains("puts $here"), "{renamed}"); + } + + #[test] + fn rename_covers_uses_inside_if_body() { + let src = "\ +proc f {} { + set mode fast + if {$mode == fast} { + puts $mode + } +} +"; + let pos = at(src, "mode", 0); + let edits = edits_of(src, pos, "kind"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set kind fast"), "{renamed}"); + // Both the condition and the body refs get updated via the + // brace-body reparse. + assert!(renamed.contains("if {$kind == fast}"), "{renamed}"); + assert!(renamed.contains("puts $kind"), "{renamed}"); + } + + #[test] + fn rename_does_not_leak_into_nested_proc_scope() { + // Outer `set foo` and inner `proc g { foo }` share a name + // but are unrelated scopes. Renaming the outer must not + // touch the inner. + let src = "\ +proc outer {} { + set foo 1 + proc g {foo} { + puts $foo + } + puts $foo +} +"; + let pos = at(src, "foo", 0); // outer set decl + let edits = edits_of(src, pos, "bar"); + let renamed = apply(src, &edits); + // Outer decl + outer use renamed. + assert!(renamed.contains("set bar 1"), "{renamed}"); + // Inner proc's arg and its body ref stay `foo`. + assert!(renamed.contains("proc g {foo}"), "{renamed}"); + assert!(renamed.contains("puts $foo\n }"), "{renamed}"); + } + + #[test] + fn refuse_invalid_new_name() { + // Bad syntax for a Tcl identifier still gets refused. The + // qualified-name `ns::var` form is now accepted (proc / + // enum-variant renames need to write it). + let src = "proc f {} { set x 1; puts $x }\n"; + let pos = at(src, "set x", 0) + 4; + let parsed = parse(src); + assert!(rename_at(&parsed.document, src, pos, "").is_none()); + assert!(rename_at(&parsed.document, src, pos, "1foo").is_none()); + assert!(rename_at(&parsed.document, src, pos, "foo-bar").is_none()); + } + + #[test] + fn rename_proc_from_decl_covers_call_sites() { + // Cursor on the proc name at its decl → both the decl and + // every call to `greet` in the same file get rewritten. + let src = "\ +proc greet {} { puts hi } +greet +proc other {} { greet } +"; + let pos = at(src, "greet", 0); + let edits = edits_of(src, pos, "hello"); + let renamed = apply(src, &edits); + assert!(renamed.contains("proc hello {}"), "{renamed}"); + assert_eq!(renamed.matches("hello").count(), 3, "{renamed}"); + } + + #[test] + fn rename_proc_from_call_site() { + // Cursor on a call site → same set of edits as from the + // decl; the identify pass just picks the same target. + let src = "\ +proc greet {} { puts hi } +greet +"; + let pos = at(src, "greet", 1); + let edits = edits_of(src, pos, "hello"); + let renamed = apply(src, &edits); + assert!(renamed.contains("proc hello {}"), "{renamed}"); + assert!(!renamed.contains("greet"), "{renamed}"); + } + + #[test] + fn rename_top_level_set() { + let src = "\ +set root /tmp +puts $root +"; + let pos = at(src, "root", 0); + let edits = edits_of(src, pos, "dir"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set dir /tmp"), "{renamed}"); + assert!(renamed.contains("puts $dir"), "{renamed}"); + } + + #[test] + fn rename_from_cursor_on_dollar_sign() { + // Cursor on the `$` itself, not the letter after. + let src = "\ +proc f {} { + set x 1 + puts $x +} +"; + let pos = at(src, "$x", 0); + let edits = edits_of(src, pos, "y"); + let renamed = apply(src, &edits); + assert!(renamed.contains("set y 1"), "{renamed}"); + assert!(renamed.contains("puts $y"), "{renamed}"); + } + + #[test] + fn is_valid_ident_smoke() { + assert!(is_valid_tcl_ident("foo")); + assert!(is_valid_tcl_ident("_foo")); + assert!(is_valid_tcl_ident("foo_bar")); + assert!(is_valid_tcl_ident("f1")); + assert!(!is_valid_tcl_ident("")); + assert!(!is_valid_tcl_ident("1foo")); + assert!(!is_valid_tcl_ident("foo-bar")); + assert!(!is_valid_tcl_ident("foo bar")); + assert!(!is_valid_tcl_ident("foo::bar")); + } +} diff --git a/vw-htcl/src/repr.rs b/vw-htcl/src/repr.rs new file mode 100644 index 0000000..cf1dbb6 --- /dev/null +++ b/vw-htcl/src/repr.rs @@ -0,0 +1,987 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Compiler-emitted `repr` procs for typed runtime values. +//! +//! The REPL (and other consumers) ask "give me Tcl source that, when +//! applied to a value of type T, produces a display string." That's +//! `dispatch_name(T)`, which always resolves to `::repr` in +//! the running Tcl interpreter: +//! +//! - For **primitives** (`string`, `int`, `bool`, `unit`), `::repr` +//! is shipped once at session start via [`emit_primitive_prelude`]. +//! - For **user-declared newtypes** (`bd_cell`, `widget`, …), `::repr` +//! is the user's own proc — the validator enforces it exists (see +//! [`crate::validate::build_type_decl_table`]). +//! - For **generics** (`list`, `dict`, nested combinations), +//! [`emit_repr`] monomorphizes a per-instantiation `::repr` +//! that delegates to its element / key / value reprs. Each unique +//! nested instantiation gets its own proc. +//! +//! All emission goes through [`vw_quote::quote_tcl!`] so word +//! quoting is handled automatically rather than via `format!` string +//! concatenation. +//! +//! Mangling: dot-free, separator `_`. `dict` → +//! `dict_string_int`; `list>` → +//! `list_dict_string_bd_cell`. The mangled string is used as the +//! namespace of the emitted proc — `dict_string_int::repr`. This +//! corner-collides only when a user declares `type X` whose name +//! happens to equal a mangled compiler-generated namespace (e.g. +//! `type dict_string_int`); pathological in practice. + +use std::collections::{HashMap, HashSet}; + +use vw_quote::quote_tcl; + +use crate::ast::{EnumDecl, EnumVariant, TypeDecl, TypeExpr}; + +/// Output of [`emit_repr`]: the per-type Tcl procs to ship (in +/// dependency order) and the dispatch name to invoke after they're +/// in scope. +#[derive(Clone, Debug)] +pub struct ReprEmission { + /// Tcl proc declarations to ship to the worker before any + /// expression that needs them. Each entry is a complete + /// `proc ::repr { v } { … }` source. + pub procs: Vec, + /// Fully-qualified Tcl proc to invoke: `::repr`. + pub dispatch: String, +} + +/// Mangled namespace name for `ty`. The compiler-emitted repr proc +/// for this type lives at `::repr`. +pub fn mangle(ty: &TypeExpr) -> String { + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let mut out = String::with_capacity(name.len() + args.len() * 8); + out.push_str(name); + for arg in args { + out.push('_'); + out.push_str(&mangle(arg)); + } + out + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + // Qualified names that reached codegen at a value + // position resolve to a namespaced newtype — the + // validator has already verified the name refers to a + // declared `type ns::T = …`. Enum-variant Qualifieds + // never flow here (they're only legal as the dispatch + // first-arg annotation, which mangling doesn't touch). + // Mangle by joining with `::` so the resulting Tcl + // proc name matches the newtype's declared namespace. + format!("{namespace}::{variant}") + } + } +} + +/// Fully-qualified Tcl name of `ty`'s repr proc — what a caller +/// invokes on a value to format it. +pub fn dispatch_name(ty: &TypeExpr) -> String { + format!("{}::repr", mangle(ty)) +} + +/// Fully-qualified Tcl name of `ty`'s `to_raw` proc — the +/// boundary-lowering helper that flattens a typed htcl value +/// down to the bare-Tcl form Vivado consumes through `extern::`. +/// Used by [`emit_to_raw_arm`] and by wrappers that explicitly +/// invoke a type's lowering on a typed arg before forwarding to +/// `extern::`. +pub fn to_raw_dispatch_name(ty: &TypeExpr) -> String { + format!("{}::to_raw", mangle(ty)) +} + +/// Fully-qualified Tcl name of `ty`'s `from_raw` proc — the +/// boundary-lifting helper that wraps a raw extern-returned +/// value into the typed form htcl downstream consumes. +pub fn from_raw_dispatch_name(ty: &TypeExpr) -> String { + format!("{}::from_raw", mangle(ty)) +} + +/// Whether `name` is a primitive type the compiler ships repr for. +/// Anything else is either a user-declared newtype (whose triplet is +/// validated separately) or a generic instantiation (whose repr is +/// emitted by [`emit_repr`]). +pub fn is_primitive(name: &str) -> bool { + matches!(name, "string" | "int" | "bool" | "unit") +} + +/// Emit the primitive prelude — Tcl source for the +/// `string` / `int` / `bool` / `unit` triplets (`repr` + `from` + +/// `to`). Shipped once at session start so every typed expression +/// downstream can rely on the primitives being defined. +/// +/// Each type's procs are wrapped in an explicit `namespace eval` +/// block. `string` is a Tcl built-in command, so the otherwise- +/// implicit `proc string::repr` namespace-creation hits a +/// "unknown namespace" error from the interpreter; wrapping in +/// `namespace eval string {...}` sidesteps that (we're operating +/// on the namespace as a Tcl namespace, not as a command class). +/// The same wrapping is applied uniformly to `int` / `bool` / +/// `unit` for consistency and so a future Tcl that promotes +/// `bool` or `int` to a built-in doesn't silently break us. +/// +/// `from` / `to` for primitives are identity (or coerce to the +/// canonical representation, e.g. `expr {int(...)}` for `int`). +pub fn emit_primitive_prelude() -> Vec { + // Compiler-emitted reprs share the same kwargs envelope as + // user-written newtype reprs (`proc ::repr {v: T} string + // { … }` lowers to `proc repr {args} { ::vw::kwargs $args + // {v ""}; … }`). The dispatch site (see + // `vw-repl::lower::wrap_with_repr`) always calls them with + // `-v ` so the kwargs envelope binds `$v` uniformly. + // Without this uniformity, user-written reprs (which can't + // avoid the kwargs wrap) would error on positional calls. + vec![ + // string: identity at every slot — including to_raw / from_raw, + // since the Tcl runtime representation of a string IS the raw + // value the extern boundary expects. + quote_tcl!( + "namespace eval string {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // int: format / coerce. to_raw / from_raw mirror to / from + // since Vivado consumes integer-shaped strings. + quote_tcl!( + "namespace eval int {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [format %d $v] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {int($v)}] }\n\ + }\n" + ), + // bool: textual form; 0/1 round-trip for from/to. to_raw / + // from_raw use the same 0/1 form Vivado expects. + quote_tcl!( + "namespace eval bool {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? \"true\" : \"false\"}] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return [expr {$v ? 1 : 0}] }\n\ + }\n" + ), + // unit: empty value. The App suppresses on the *type*, not + // on the value — these procs exist so generics over `unit` + // still type-check, even though they're unusual. + quote_tcl!( + "namespace eval unit {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n\ + }\n" + ), + // list: newline-joined for repr. Tcl's default string form + // is space-separated with brace-escaping — technically the + // canonical rendering, but unreadable at scale (lists of + // paths are the common case at the REPL). Rendering one + // entry per line is the pragmatic choice. + // + // `from` / `to` stay identity — the boundary with Vivado + // still expects the Tcl-list byte form, only the human + // rendering changes. `to_raw` / `from_raw` also identity + // for the same reason. + quote_tcl!( + "namespace eval list {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; return [join $v \"\\n\"] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // dict: readable, structure-aware rendering. + // + // - Single-value entries (`llength $v == 1`) render inline + // as `key = value`. + // - Multi-value entries render as `key:` followed by one + // indented item per line. + // + // The `llength > 1` heuristic isn't perfect — a bare + // multi-word string is ambiguous with a Tcl list at this + // level — but the concrete case that motivated this + // (JSON-derived dicts of lists via the RPC) is exactly + // the shape where `llength` accurately distinguishes + // structure from prose. `key = 1 2 3` (all-on-one-line) is + // unreadable for paths; `key:\n a\n b\n c` is exactly + // what a human wants to skim. + // + // `from` / `to` / `to_raw` / `from_raw` are identity so + // the extern boundary still round-trips the Tcl byte + // form. Only human display changes. + quote_tcl!( + "namespace eval dict {\n \ + proc repr {args} { ::vw::kwargs $args {v \"\"}; set _out {}; dict for {_k _dv} $v { if {[llength $_dv] > 1} { lappend _out \"$_k:\"; foreach _item $_dv { lappend _out \" $_item\" } } else { lappend _out \"$_k = $_dv\" } }; return [join $_out \"\\n\"] }\n \ + proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc to_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n \ + proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return $v }\n\ + }\n" + ), + // Runtime `::putr` — fallback for the compile-time putr + // rewrite. `vw-htcl/src/putr.rs` catches every TOP-LEVEL + // `putr ` at parse time and dispatches it to the right + // `T::repr` when the type is statically known. It does + // NOT descend into braced-word bodies of control-flow + // builtins (`foreach { … }`, `dict for { … } { … } { … }`, + // `for { … } { … } { … }`, etc.), so a `putr $x` inside + // one of those lands at Tcl as a literal command call. + // Without this runtime proc, Tcl would report `invalid + // command name \"putr\"`. + // + // Best-effort heuristic dispatch at runtime: multi-element + // values render through `list::repr` (one-per-line); + // scalars land through plain `puts`. Callers who want + // dict rendering call `::dict::repr -v $x` directly. + quote_tcl!( + "proc ::putr {v} { if {[llength $v] > 1} { puts [::list::repr -v $v] } else { puts $v } }\n" + ), + ] +} + +/// Walk `ty` depth-first, emitting one `::repr` proc per +/// unique generic instantiation along the way. Plain [`Named`] types +/// (primitives or user newtypes) don't get codegen here — their +/// reprs come from [`emit_primitive_prelude`] or from the user's +/// own `::repr` proc (validator-enforced). +/// +/// The returned dispatch name is `::repr` — the caller +/// invokes it on a value of type `ty` to get the display string. +pub fn emit_repr(ty: &TypeExpr) -> ReprEmission { + emit_repr_with_types(ty, &HashMap::new()) +} + +/// Same as [`emit_repr`] but also walks user-declared newtypes +/// (`type T = U`) — when the dispatch type is a newtype whose +/// underlying is a generic, the generic's repr needs to be in +/// scope so the user's `proc T::repr` body can call it. +/// +/// Without this recursion, `Properties::repr` (which delegates to +/// `dict_string_Property::repr`) errors at runtime with +/// `invalid command name "dict_string_Property::repr"` because +/// the monomorphized generic was never emitted. +pub fn emit_repr_with_types( + ty: &TypeExpr, + types: &HashMap, +) -> ReprEmission { + let mut procs = Vec::new(); + let mut seen = HashSet::new(); + emit_recursive(ty, &mut procs, &mut seen, types); + ReprEmission { + procs, + dispatch: dispatch_name(ty), + } +} + +/// Emit the auto-generated `namespace eval { … }` prelude +/// for an enum declaration. Contains: +/// +/// - **Constructors** — one per variant. Payload variants take a +/// `v` arg and return `[list $v]`; empty-payload +/// variants take no args and return `[list ]`. +/// - **`tag` / `payload`** — explicit unwrap accessors wrappers +/// use to bridge enum values into bare-Tcl `extern::` calls. +/// - **`repr`** — switches on `[lindex $v 0]`, calls each variant +/// payload type's `repr` and wraps as `()` for +/// payload variants, bare `` for empty ones. +/// - **`from` / `to`** — identity (enum values are already in their +/// canonical tagged-tuple form; the triplet exists so generics +/// over enums type-check uniformly with newtypes). +/// +/// The block is wrapped in `namespace eval` — Tcl auto-creates +/// the namespace on `proc ::` ONLY when nothing else +/// claims the name. For defensiveness (and so users can pick +/// enum names that happen to match a Tcl built-in's namespace +/// later without a confusing failure mode), we use the explicit +/// form, mirroring the primitive prelude. +pub fn emit_enum_prelude(enum_decl: &EnumDecl) -> String { + let Some(name) = enum_decl.name.as_deref() else { + // Anonymous enum — shouldn't happen post-parser, but + // bail rather than emit junk. + return String::new(); + }; + let mut body = String::new(); + body.push_str(&format!("namespace eval {name} {{\n")); + // Constructors — plain positional Tcl, called by user code as + // `Property::Scalar foo` (positional). NOT through the kwargs + // envelope. + for v in &enum_decl.variants { + emit_constructor(&mut body, &v.name, v.payload.is_some()); + } + // tag / payload — also positional; called by wrappers in + // `extern::` bridging code as `Property::payload $v`. + body.push_str(" proc tag {v} { return [lindex $v 0] }\n"); + body.push_str(" proc payload {v} { return [lindex $v 1] }\n"); + // repr / from / to — kwargs envelope so they're callable + // uniformly with all other reprs (the dispatch site emits + // `-v ` form universally; see + // `vw-repl::lower::wrap_with_repr`). + body.push_str(" proc repr {args} {\n"); + body.push_str(" ::vw::kwargs $args {v \"\"}\n"); + body.push_str(" switch -- [lindex $v 0] {\n"); + for v in &enum_decl.variants { + emit_repr_arm(&mut body, v); + } + body.push_str(" default { return \"\" }\n"); + body.push_str(" }\n"); + body.push_str(" }\n"); + // from / to are identity for enums (the constructors are the + // user-facing lift). + body.push_str( + " proc from {args} { ::vw::kwargs $args {v \"\"}; return $v }\n", + ); + body.push_str( + " proc to {args} { ::vw::kwargs $args {v \"\"}; return $v }\n", + ); + // to_raw: lower the tagged enum value to its raw extern-side + // representation. Switch on variant tag; for payload variants + // recurse via the payload type's to_raw; for empty variants + // emit the variant name (the convention extern Vivado calls + // recognize for tag-style values). See [docs/htcl-extern-boundary.md] + // for the rationale on this being mechanical / compiler-emitted. + body.push_str(" proc to_raw {args} {\n"); + body.push_str(" ::vw::kwargs $args {v \"\"}\n"); + body.push_str(" switch -- [lindex $v 0] {\n"); + for v in &enum_decl.variants { + emit_to_raw_arm(&mut body, v); + } + body.push_str( + " default { error \"unknown variant: [lindex $v 0]\" }\n", + ); + body.push_str(" }\n"); + body.push_str(" }\n"); + // from_raw: default lift wraps the raw value as the FIRST + // variant. For sum types where the right variant depends on + // the value's shape (e.g. Property — Scalar vs Nested + // chosen by structural inference), users override via + // `proc ::from_raw` AFTER the compiler-emitted prelude; + // Tcl's last-`proc`-wins lets the user override take + // precedence. + if let Some(first) = enum_decl.variants.first() { + emit_from_raw_default(&mut body, first); + } else { + body.push_str( + " proc from_raw {args} { ::vw::kwargs $args {v \"\"}; return \"\" }\n", + ); + } + body.push_str("}\n"); + body +} + +/// Emit one arm of `::to_raw`'s `switch -- [lindex $v 0]` +/// body: for payload variants, recurse via the payload type's +/// `to_raw`; for empty variants, emit the variant name as the +/// raw value (matches how extern Vivado callers receive bare +/// enum-style tags). +fn emit_to_raw_arm(out: &mut String, v: &EnumVariant) { + let variant = &v.name; + match &v.payload { + None => { + out.push_str(&format!( + " {variant} {{ return \"{variant}\" }}\n" + )); + } + Some(payload_ty) => { + let dispatch = to_raw_dispatch_name(payload_ty); + out.push_str(&format!( + " {variant} {{ return [{dispatch} -v [lindex $v 1]] }}\n" + )); + } + } +} + +/// Default `::from_raw` body — wrap input as the first +/// variant. For payload variants, the input flows through the +/// payload type's `from_raw` first. For empty variants, the +/// input is ignored and we return the bare-variant constructor. +fn emit_from_raw_default(out: &mut String, first: &EnumVariant) { + let variant = &first.name; + match &first.payload { + None => { + out.push_str(&format!( + " proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + return [list {variant}]\n \ + }}\n", + )); + } + Some(payload_ty) => { + let dispatch = from_raw_dispatch_name(payload_ty); + out.push_str(&format!( + " proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + return [list {variant} [{dispatch} -v $v]]\n \ + }}\n", + )); + } + } +} + +fn emit_constructor(out: &mut String, variant: &str, has_payload: bool) { + if has_payload { + out.push_str(&format!( + " proc {variant} {{v}} {{ return [list {variant} $v] }}\n" + )); + } else { + out.push_str(&format!( + " proc {variant} {{}} {{ return [list {variant}] }}\n" + )); + } +} + +fn emit_repr_arm(out: &mut String, v: &EnumVariant) { + let variant = &v.name; + match &v.payload { + None => { + // Empty-payload: just the bare variant name. + out.push_str(&format!( + " {variant} {{ return \"{variant}\" }}\n" + )); + } + Some(payload_ty) => { + // Payload variant: `()`. Formatting + // depends on whether the inner repr fits on one line: + // + // single-line: `Variant(inner)` + // multi-line: `Variant(\n line1\n line2\n)` + // + // The multi-line shape (opening paren followed by + // newline + 2-space indent for the first child, + // closing paren on its own line, every inner line + // indented one extra level) keeps deeply-nested + // values readable instead of arrowing off the right + // margin. + // + // 2-space indent applies to ALL inner lines + // (including their pre-existing continuation indents), + // so each nesting level adds exactly 2 spaces of + // indent uniformly. + let dispatch = dispatch_name(payload_ty); + out.push_str(&format!( + " {variant} {{\n \ + set __vw_inner [{dispatch} -v [lindex $v 1]]\n \ + if {{[string first \"\\n\" $__vw_inner] >= 0}} {{\n \ + set __vw_indented [string map [list \\n \"\\n \"] $__vw_inner]\n \ + return \"{variant}(\\n $__vw_indented\\n)\"\n \ + }} else {{\n \ + return \"{variant}($__vw_inner)\"\n \ + }}\n \ + }}\n" + )); + } + } +} + +fn emit_recursive( + ty: &TypeExpr, + out: &mut Vec, + seen: &mut HashSet, + types: &HashMap, +) { + match ty { + TypeExpr::Named { name, .. } => { + // No codegen for plain names directly — `::repr` + // is either a primitive (shipped via + // `emit_primitive_prelude`) or a user newtype + // (validator-enforced to exist). BUT if `name` + // resolves to a user newtype whose underlying is a + // generic, we have to recurse so the underlying's + // monomorphized repr is shipped — the user's + // `proc ::repr` body typically delegates to it. + if let Some(decl) = types.get(name.as_str()) { + if let Some(underlying) = decl.underlying.as_ref() { + emit_recursive(underlying, out, seen, types); + } + } + } + TypeExpr::Generic { name, args, .. } => { + // Depth-first: emit each arg's repr first so this + // proc's body can call them. + for a in args { + emit_recursive(a, out, seen, types); + } + let m = mangle(ty); + if !seen.insert(m.clone()) { + return; // Already emitted this instantiation. + } + let body = match name.as_str() { + "dict" if args.len() == 2 => { + emit_dict_repr(&m, &args[0], &args[1]) + } + "list" if args.len() == 1 => emit_list_repr(&m, &args[0]), + _ => emit_unknown_generic_repr(&m), + }; + out.push(body); + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + // Namespaced newtype reference (`dcmac::GtChProps` and + // friends). Look up by the joined qualified name — the + // types table is keyed exactly that way (see + // `validate::build_type_decl_table`). If found and its + // underlying is a generic, recurse so the generic's + // monomorphized repr ships alongside. + let qualified = format!("{namespace}::{variant}"); + if let Some(decl) = types.get(qualified.as_str()) { + if let Some(underlying) = decl.underlying.as_ref() { + emit_recursive(underlying, out, seen, types); + } + } + } + } +} + +/// `dict::repr` — iterate pairs, format each as +/// ` ` joined with newlines. +/// +/// The body uses braced `expr {…}` and avoids interpolating the +/// dispatch names raw via `quote_tcl!` because Tcl's word quoting +/// would brace the `::` separators (those are bare-safe but the +/// macro's `Word::lit` doesn't know that). The proc names go in via +/// raw substitution at template time instead — they're already +/// valid Tcl, and the macro template's literal regions pass through +/// untouched. +fn emit_dict_repr(mangled: &str, k: &TypeExpr, v: &TypeExpr) -> String { + let key_repr = dispatch_name(k); + let val_repr = dispatch_name(v); + let key_to_raw = to_raw_dispatch_name(k); + let val_to_raw = to_raw_dispatch_name(v); + let key_from_raw = from_raw_dispatch_name(k); + let val_from_raw = from_raw_dispatch_name(v); + // Uses the same kwargs envelope as `emit_primitive_prelude` + // so the dispatch site can uniformly call all reprs with + // `-v `. Sub-element reprs are invoked through the + // same `-v` convention. + // + // to_raw / from_raw are emitted in the SAME namespace so + // callers can dispatch via `::{repr,to_raw,from_raw}` + // uniformly. to_raw walks the dict, applying K::to_raw and + // V::to_raw element-wise and rebuilding as a flat paired + // list (the shape Vivado consumes). from_raw is the inverse + // — walks a paired list, applies K::from_raw / V::from_raw + // element-wise, builds a typed dict. + format!( + "namespace eval {ns} {{\n \ + proc repr {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out \"\"\n \ + set first 1\n \ + foreach {{k val}} $v {{\n \ + if {{!$first}} {{ append out \"\\n\" }}\n \ + set first 0\n \ + append out [{kr} -v $k] \" \" [{vr} -v $val]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc to_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach {{k val}} $v {{\n \ + # Recurse into the element to_raw calls with a\n \ + # catch that prepends this level's key on error,\n \ + # so a failure deep inside a nested Properties\n \ + # tree bubbles up as a dotted path (e.g.\n \ + # `LR0_SETTINGS.RX_REFCLK_FREQUENCY.`).\n \ + if {{[catch {{\n \ + set __vw_kraw [{ktr} -v $k]\n \ + set __vw_vraw [{vtr} -v $val]\n \ + }} __vw_msg]}} {{\n \ + # Lowercase the key in the error so\n \ + # nested-property failures surface with\n \ + # the HTCL-surface arg name\n \ + # (`tx_refclk_frequency`) rather than\n \ + # the Vivado SCREAM_CASE dict key\n \ + # (`TX_REFCLK_FREQUENCY`). Innocuous for\n \ + # non-Vivado dicts — lowercasing an\n \ + # already-lowercase key is a no-op.\n \ + error \"[string tolower $k].$__vw_msg\"\n \ + }}\n \ + lappend out $__vw_kraw $__vw_vraw\n \ + }}\n \ + return $out\n \ + }}\n \ + proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [dict create]\n \ + foreach {{k val}} $v {{\n \ + dict set out [{kfr} -v $k] [{vfr} -v $val]\n \ + }}\n \ + return $out\n \ + }}\n\ + }}\n", + ns = mangled, + kr = key_repr, + vr = val_repr, + ktr = key_to_raw, + vtr = val_to_raw, + kfr = key_from_raw, + vfr = val_from_raw, + ) +} + +/// `list::repr` — iterate elements, format each via `T::repr`, +/// join with newlines. Also emits `to_raw` / `from_raw` element- +/// wise dispatching through `T::to_raw` / `T::from_raw`. +fn emit_list_repr(mangled: &str, elem: &TypeExpr) -> String { + let elem_repr = dispatch_name(elem); + let elem_to_raw = to_raw_dispatch_name(elem); + let elem_from_raw = from_raw_dispatch_name(elem); + format!( + "namespace eval {ns} {{\n \ + proc repr {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out \"\"\n \ + set first 1\n \ + foreach item $v {{\n \ + if {{!$first}} {{ append out \"\\n\" }}\n \ + set first 0\n \ + append out [{er} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc to_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach item $v {{\n \ + lappend out [{etr} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n \ + proc from_raw {{args}} {{\n \ + ::vw::kwargs $args {{v \"\"}}\n \ + set out [list]\n \ + foreach item $v {{\n \ + lappend out [{efr} -v $item]\n \ + }}\n \ + return $out\n \ + }}\n\ + }}\n", + ns = mangled, + er = elem_repr, + etr = elem_to_raw, + efr = elem_from_raw, + ) +} + +/// Fallback for generic shapes we don't have a specialized shell +/// for (e.g. a hypothetical `tuple<…>` we haven't designed yet). +/// Renders the raw Tcl value — at least the user sees *something* +/// instead of an "unknown generic" error. +fn emit_unknown_generic_repr(mangled: &str) -> String { + format!( + "namespace eval {mangled} {{ \ + proc repr {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + proc to_raw {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + proc from_raw {{args}} {{ ::vw::kwargs $args {{v \"\"}}; return $v }} \ + }}\n" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::span::Span; + + fn named(name: &str) -> TypeExpr { + TypeExpr::Named { + name: name.into(), + span: Span::new(0, 0), + } + } + + fn generic(name: &str, args: Vec) -> TypeExpr { + TypeExpr::Generic { + name: name.into(), + name_span: Span::new(0, 0), + args, + span: Span::new(0, 0), + } + } + + #[test] + fn mangle_primitives() { + assert_eq!(mangle(&named("string")), "string"); + assert_eq!(mangle(&named("bd_cell")), "bd_cell"); + assert_eq!(mangle(&named("unit")), "unit"); + } + + #[test] + fn mangle_dict_two_args() { + let ty = generic("dict", vec![named("string"), named("int")]); + assert_eq!(mangle(&ty), "dict_string_int"); + } + + #[test] + fn mangle_list_one_arg() { + let ty = generic("list", vec![named("bd_cell")]); + assert_eq!(mangle(&ty), "list_bd_cell"); + } + + #[test] + fn mangle_nested() { + let inner = generic("dict", vec![named("string"), named("bd_cell")]); + let outer = generic("list", vec![inner]); + assert_eq!(mangle(&outer), "list_dict_string_bd_cell"); + } + + #[test] + fn dispatch_for_primitive_uses_name() { + assert_eq!(dispatch_name(&named("string")), "string::repr"); + assert_eq!(dispatch_name(&named("bd_cell")), "bd_cell::repr"); + } + + #[test] + fn dispatch_for_generic_uses_mangled() { + let ty = generic("dict", vec![named("string"), named("string")]); + assert_eq!(dispatch_name(&ty), "dict_string_string::repr"); + } + + #[test] + fn primitive_prelude_emits_one_namespace_block_per_type() { + let procs = emit_primitive_prelude(); + // 6 types (namespace-eval blocks) plus the standalone + // runtime `::putr` fallback. + assert_eq!(procs.len(), 7); + assert!(procs.iter().any(|p| p.contains("namespace eval string"))); + assert!(procs.iter().any(|p| p.contains("namespace eval int"))); + assert!(procs.iter().any(|p| p.contains("namespace eval bool"))); + assert!(procs.iter().any(|p| p.contains("namespace eval unit"))); + assert!(procs.iter().any(|p| p.contains("namespace eval list"))); + assert!(procs.iter().any(|p| p.contains("namespace eval dict"))); + // Standalone runtime `::putr` fallback for braced-body + // scripts where the compile-time rewrite can't fire. + assert!(procs.iter().any(|p| p.contains("proc ::putr"))); + // Every NAMESPACE-BLOCK entry contains the full + // repr/from/to triplet — the standalone `::putr` entry + // (which is a bare proc, not a namespace block) is + // exempted. + for p in &procs { + if !p.contains("namespace eval") { + continue; + } + assert!(p.contains("proc repr"), "missing repr in: {p}"); + assert!(p.contains("proc from"), "missing from in: {p}"); + assert!(p.contains("proc to"), "missing to in: {p}"); + } + } + + #[test] + fn emit_repr_named_emits_no_procs() { + // Primitives and user newtypes don't need codegen — repr + // lives in the primitive prelude or the user's own proc. + let e = emit_repr(&named("string")); + assert!(e.procs.is_empty()); + assert_eq!(e.dispatch, "string::repr"); + + let e = emit_repr(&named("bd_cell")); + assert!(e.procs.is_empty()); + assert_eq!(e.dispatch, "bd_cell::repr"); + } + + #[test] + fn emit_repr_dict_string_string() { + let ty = generic("dict", vec![named("string"), named("string")]); + let e = emit_repr(&ty); + assert_eq!(e.dispatch, "dict_string_string::repr"); + assert_eq!(e.procs.len(), 1); + let body = &e.procs[0]; + // The proc is defined inside its `namespace eval`, so the + // textual proc name is just `repr` — the namespace is in + // the surrounding `namespace eval dict_string_string`. + assert!(body.contains("namespace eval dict_string_string")); + assert!(body.contains("proc repr {args}")); + assert!(body.contains("::vw::kwargs $args")); + assert!(body.contains("foreach {k val} $v")); + // Element reprs called by their fully-qualified name via + // the universal `-v ` kwargs form. + assert!(body.contains("[string::repr -v $k]")); + assert!(body.contains("[string::repr -v $val]")); + } + + #[test] + fn emit_repr_list_bd_cell() { + let ty = generic("list", vec![named("bd_cell")]); + let e = emit_repr(&ty); + assert_eq!(e.dispatch, "list_bd_cell::repr"); + assert_eq!(e.procs.len(), 1); + let body = &e.procs[0]; + assert!(body.contains("namespace eval list_bd_cell")); + assert!(body.contains("proc repr {args}")); + assert!(body.contains("[bd_cell::repr -v $item]")); + } + + #[test] + fn emit_repr_nested_topologically_orders_sub_procs() { + // dict>: emits list::repr first, + // then dict_string_list_int::repr. + let inner = generic("list", vec![named("int")]); + let outer = generic("dict", vec![named("string"), inner]); + let e = emit_repr(&outer); + assert_eq!(e.dispatch, "dict_string_list_int::repr"); + assert_eq!(e.procs.len(), 2); + // First proc emitted is the inner list, second is the + // outer dict. + assert!(e.procs[0].contains("namespace eval list_int")); + assert!(e.procs[1].contains("namespace eval dict_string_list_int")); + // Outer body calls the inner by its fully-qualified name. + assert!(e.procs[1].contains("[list_int::repr")); + } + + #[test] + fn emit_repr_dedups_repeated_subtypes() { + // dict — bd_cell is a leaf (Named), so no + // codegen for it, but if we had dict, list> + // we'd want list_int::repr emitted only ONCE. + let inner = generic("list", vec![named("int")]); + let outer = generic("dict", vec![inner.clone(), inner]); + let e = emit_repr(&outer); + // list_int's namespace block appears once even though it's + // referenced twice in the outer dict. + let list_int_count = e + .procs + .iter() + .filter(|p| p.contains("namespace eval list_int ")) + .count(); + assert_eq!(list_int_count, 1); + } + + #[test] + fn emit_repr_unknown_generic_falls_back_to_identity() { + let ty = generic("tuple", vec![named("string"), named("int")]); + let e = emit_repr(&ty); + assert_eq!(e.procs.len(), 1); + assert!( + e.procs[0].contains("return $v"), + "expected identity body, got {:?}", + e.procs[0] + ); + } + + #[test] + fn is_primitive_table() { + assert!(is_primitive("string")); + assert!(is_primitive("int")); + assert!(is_primitive("bool")); + assert!(is_primitive("unit")); + assert!(!is_primitive("bd_cell")); + assert!(!is_primitive("widget")); + assert!(!is_primitive("dict")); + } + + // --- enum prelude emission -------------------------------------- + + fn ed_with_variants( + name: &str, + vs: Vec<(&str, Option)>, + ) -> EnumDecl { + EnumDecl { + name: Some(name.into()), + name_span: Span::new(0, 0), + variants: vs + .into_iter() + .map(|(n, p)| EnumVariant { + name: n.into(), + name_span: Span::new(0, 0), + payload: p, + payload_span: Span::new(0, 0), + span: Span::new(0, 0), + }) + .collect(), + body_span: Span::new(0, 0), + } + } + + #[test] + fn enum_prelude_with_payload_variants() { + let ed = ed_with_variants( + "Property", + vec![ + ("Scalar", Some(named("string"))), + ( + "Nested", + Some(generic( + "dict", + vec![named("string"), named("string")], + )), + ), + ], + ); + let p = emit_enum_prelude(&ed); + // Wrapped in namespace eval. + assert!(p.contains("namespace eval Property")); + // Constructors with payload. + assert!(p.contains("proc Scalar {v} { return [list Scalar $v] }")); + assert!(p.contains("proc Nested {v} { return [list Nested $v] }")); + // Accessors. + assert!(p.contains("proc tag {v}")); + assert!(p.contains("proc payload {v}")); + // Repr switch — kwargs envelope around the body. + assert!(p.contains("proc repr {args}")); + assert!(p.contains("::vw::kwargs $args")); + assert!(p.contains("switch -- [lindex $v 0]")); + // Each variant's body now uses an intermediate + // `__vw_inner` after applying the continuation-indent + // `string map` transform. + assert!(p.contains("Scalar($__vw_inner)")); + assert!(p.contains("Nested($__vw_inner)")); + // Payload reprs dispatched via mangled names with `-v`. + assert!(p.contains("string::repr -v")); + assert!(p.contains("dict_string_string::repr -v")); + // Identity from/to — also kwargs envelope. + assert!(p.contains("proc from {args}")); + assert!(p.contains("proc to {args}")); + } + + #[test] + fn enum_prelude_with_empty_payload_variants() { + let ed = ed_with_variants( + "Direction", + vec![ + ("North", None), + ("South", None), + ("East", None), + ("West", None), + ], + ); + let p = emit_enum_prelude(&ed); + // Empty-payload constructors take no args. + assert!(p.contains("proc North {} { return [list North] }")); + assert!(p.contains("proc West {} { return [list West] }")); + // Repr arms render bare variant name (no parens). + assert!(p.contains("North { return \"North\" }")); + assert!(p.contains("West { return \"West\" }")); + // No `(` after variant names in the repr arms. + let arm = "North { return \"North("; + assert!(!p.contains(arm), "shouldn't have parens for empty variants"); + } + + #[test] + fn enum_prelude_mixed_payload_and_empty() { + let ed = ed_with_variants( + "Maybe", + vec![("Some", Some(named("int"))), ("None", None)], + ); + let p = emit_enum_prelude(&ed); + assert!(p.contains("proc Some {v} { return [list Some $v] }")); + assert!(p.contains("proc None {} { return [list None] }")); + // Payload arm uses `__vw_inner` after the + // continuation-indent `string map` transform. + assert!(p.contains("int::repr -v")); + assert!(p.contains("Some($__vw_inner)")); + assert!(p.contains("None { return \"None\" }")); + } +} diff --git a/vw-htcl/src/scope.rs b/vw-htcl/src/scope.rs new file mode 100644 index 0000000..80d3343 --- /dev/null +++ b/vw-htcl/src/scope.rs @@ -0,0 +1,390 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Variable scope resolution shared by goto and hover. +//! +//! Tcl variables are local to a proc (its parameters plus whatever it +//! `set`s / `variable`s); top-level code shares the global scope. That +//! lexical model is enough to point a `$name` reference at its +//! definition. +//! +//! Two entry styles: +//! +//! - [`resolve_var_def`] resolves a name given a known scope — used by +//! the structured path when the reference is a real [`WordPart::VarRef`]. +//! - [`scan_var_ref`] + [`innermost_scope`] recover a reference the +//! structured parser left buried in opaque text (a command +//! substitution body, or an `if`/`while` condition), by reading the +//! raw source at the cursor and locating the enclosing proc by span. + +use crate::ast::{ + Command, CommandKind, Document, Proc, ProcArg, Stmt, TypeDecl, TypeExpr, +}; +use crate::span::Span; + +/// What a `$name` reference resolves to. +#[derive(Clone, Copy, Debug)] +pub enum VarDef<'a> { + /// A parameter of the enclosing proc. + Param(&'a ProcArg), + /// A local established by `set name ...` or `variable name ...`. + /// Carries the span of the defined name. + Local(Span), +} + +impl VarDef<'_> { + /// The span to navigate to / anchor hover on. + pub fn def_span(&self) -> Span { + match self { + VarDef::Param(arg) => arg.name_span, + VarDef::Local(span) => *span, + } + } +} + +/// Resolve `name` within `scope_stmts` (the statements of the current +/// scope), falling back to a parameter of `enclosing`. `offset` biases +/// local resolution toward the last definition at or before the +/// reference (the value in effect there). +pub fn resolve_var_def<'a>( + name: &str, + scope_stmts: &'a [Stmt], + enclosing: Option<&'a Proc>, + offset: u32, +) -> Option> { + let mut best: Option = None; + let mut first: Option = None; + for stmt in scope_stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let Some(def) = local_def_target(cmd, name) else { + continue; + }; + first.get_or_insert(def); + if def.start <= offset { + best = Some(def); + } + } + if let Some(span) = best.or(first) { + return Some(VarDef::Local(span)); + } + + let sig = enclosing?.signature.as_ref()?; + sig.args.iter().find(|a| a.name == name).map(VarDef::Param) +} + +/// If `cmd` defines variable `name` — via `set NAME …` / `variable +/// NAME …` / `foreach NAME …` / `foreach {A B …} …` / `dict for {K +/// V} …` / `catch BODY NAME` — return the span to anchor +/// hover/goto on. Braced varname-list positions (`foreach {a b}`, +/// `dict for {k v}`) return the span of the whole braced word; +/// sub-token spans would need extra parser wiring. +fn local_def_target(cmd: &Command, name: &str) -> Option { + match &cmd.kind { + CommandKind::Set => { + let target = cmd.words.get(1)?; + (target.as_text()? == name).then_some(target.span) + } + CommandKind::Generic => { + let head = cmd.words.first()?.as_text()?; + match head { + "variable" => { + let target = cmd.words.get(1)?; + (target.as_text()? == name).then_some(target.span) + } + "foreach" => { + // Iterator target(s) at word 1 (and every + // second word after, up to but not including + // the body). Matches `unused::collect_foreach_decls`. + let body_idx = cmd.words.len().saturating_sub(1); + let mut i = 1; + while i < body_idx { + if word_declares_name(&cmd.words[i], name) { + return Some(cmd.words[i].span); + } + i += 2; + } + None + } + "dict" => { + // `dict for {K V} DICT BODY` — the kv list is at + // word 2, only for the `for` sub-command. + if cmd.words.get(1)?.as_text()? != "for" { + return None; + } + let target = cmd.words.get(2)?; + if word_declares_name(target, name) { + Some(target.span) + } else { + None + } + } + "catch" => { + // `catch BODY [RESVAR [OPTVAR]]` — words 2/3. + for w in cmd.words.iter().skip(2).take(2) { + if word_declares_name(w, name) { + return Some(w.span); + } + } + None + } + _ => None, + } + } + CommandKind::Proc(_) + | CommandKind::Src(_) + | CommandKind::NamespaceEval(_) + | CommandKind::TypeDecl(_) + | CommandKind::EnumDecl(_) => None, + } +} + +/// True when `word` names `target` — either as a bare identifier +/// (`foreach x …`) or as a whitespace-separated token inside a +/// braced list (`foreach {a b} …`, `dict for {k v} …`). +fn word_declares_name(word: &crate::ast::Word, target: &str) -> bool { + use crate::ast::{WordForm, WordPart}; + // Bare word: exact match. + if word.form == WordForm::Bare { + return word.as_text() == Some(target); + } + // Braced list: whitespace-split the interior Text. + if word.form != WordForm::Braced { + return false; + } + let Some(WordPart::Text { value, .. }) = word.parts.first() else { + return false; + }; + value.split_whitespace().any(|tok| tok == target) +} + +/// The innermost proc whose body contains `offset`, together with that +/// body's statements. `(document.stmts, None)` when `offset` is at the +/// top level. +pub fn innermost_scope( + document: &Document, + offset: u32, +) -> (&[Stmt], Option<&Proc>) { + fn helper(stmts: &[Stmt], offset: u32) -> Option<(&[Stmt], &Proc)> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.body_span.contains(offset) { + return Some( + helper(&proc.body, offset).unwrap_or((&proc.body, proc)), + ); + } + } + None + } + match helper(&document.stmts, offset) { + Some((stmts, proc)) => (stmts, Some(proc)), + None => (&document.stmts, None), + } +} + +/// If the cursor at `offset` sits on a `$name` (or `${name}`) +/// reference — even one the structured parser left inside opaque text +/// (a command substitution, or an expr condition) — return its name +/// and the span of the whole reference. +pub fn scan_var_ref(source: &str, offset: u32) -> Option<(String, Span)> { + let bytes = source.as_bytes(); + let len = bytes.len(); + let off = (offset as usize).min(len); + let is_ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + + // If the cursor sits on the `$` itself, step into the name. + let probe = if off < len && bytes[off] == b'$' { + off + 1 + } else { + off + }; + let probe = probe.min(len); + + let mut start = probe; + while start > 0 && is_ident(bytes[start - 1]) { + start -= 1; + } + let mut end = probe; + while end < len && is_ident(bytes[end]) { + end += 1; + } + if end <= start { + return None; + } + + // `$name` + if start > 0 && bytes[start - 1] == b'$' { + let name = source.get(start..end)?.to_string(); + return Some((name, Span::new((start - 1) as u32, end as u32))); + } + // `${name}` + if start >= 2 + && bytes[start - 1] == b'{' + && bytes[start - 2] == b'$' + && end < len + && bytes[end] == b'}' + { + let name = source.get(start..end)?.to_string(); + return Some((name, Span::new((start - 2) as u32, (end + 1) as u32))); + } + None +} + +/// Walk `document` and return the innermost [`TypeExpr`] whose +/// span contains `offset`. Considers proc-signature arg +/// annotations, proc return-type annotations, `type … = TYPE` +/// underlying, and generic type arguments (recursively). Returns +/// `None` when the cursor isn't on a type-expression position. +pub fn type_expr_at(document: &Document, offset: u32) -> Option<&TypeExpr> { + fn inner(ty: &TypeExpr, offset: u32) -> Option<&TypeExpr> { + if !ty.span().contains(offset) { + return None; + } + // Recurse into generic args first so the innermost match wins. + if let TypeExpr::Generic { args, .. } = ty { + for a in args { + if let Some(hit) = inner(a, offset) { + return Some(hit); + } + } + } + Some(ty) + } + fn walk(stmts: &[Stmt], offset: u32) -> Option<&TypeExpr> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !cmd.span.contains(offset) { + continue; + } + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = arg.type_annotation.as_ref() { + if let Some(hit) = inner(ty, offset) { + return Some(hit); + } + } + } + if let Some(ret) = sig.return_type.as_ref() { + if let Some(hit) = inner(ret, offset) { + return Some(hit); + } + } + } + if let Some(hit) = walk(&proc.body, offset) { + return Some(hit); + } + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = td.underlying.as_ref() { + if let Some(hit) = inner(ty, offset) { + return Some(hit); + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(hit) = walk(&ns.body, offset) { + return Some(hit); + } + } + _ => {} + } + } + None + } + walk(&document.stmts, offset) +} + +/// Find a top-level `type NAME = …` declaration whose name matches +/// `name`. Handles bare and qualified forms — a caller looking up +/// `dcmac::MacPortProps` and one looking up `Properties` both hit +/// the right decl since parser stores the raw textual name. +pub fn find_type_decl<'a>( + document: &'a Document, + name: &str, +) -> Option<&'a TypeDecl> { + fn walk<'a>(stmts: &'a [Stmt], name: &str) -> Option<&'a TypeDecl> { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) + if td.name.as_deref() == Some(name) => + { + return Some(td); + } + CommandKind::NamespaceEval(ns) => { + if let Some(hit) = walk(&ns.body, name) { + return Some(hit); + } + } + CommandKind::Proc(proc) => { + // Types can also be declared inside a proc body + // in principle (though rare). Search anyway. + if let Some(hit) = walk(&proc.body, name) { + return Some(hit); + } + } + _ => {} + } + } + None + } + walk(&document.stmts, name) +} + +/// Extract the qualified/bare identifier a [`TypeExpr`] references, +/// suitable for [`find_type_decl`] lookup. Returns the joined +/// `"namespace::variant"` for [`TypeExpr::Qualified`], and the raw +/// `name` field for the other two shapes. +pub fn type_expr_lookup_name(ty: &TypeExpr) -> String { + match ty { + TypeExpr::Named { name, .. } | TypeExpr::Generic { name, .. } => { + name.clone() + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scan_finds_bare_var_from_within() { + let src = "puts $kind here"; + // cursor on the `i` of `$kind` + let pos = (src.find("kind").unwrap() + 1) as u32; + let (name, span) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "kind"); + assert_eq!(span.slice(src), "$kind"); + } + + #[test] + fn scan_finds_var_on_dollar() { + let src = "x $y"; + let pos = src.find('$').unwrap() as u32; + let (name, _) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "y"); + } + + #[test] + fn scan_finds_braced_var() { + let src = "a ${foo} b"; + let pos = (src.find("foo").unwrap() + 1) as u32; + let (name, span) = scan_var_ref(src, pos).unwrap(); + assert_eq!(name, "foo"); + assert_eq!(span.slice(src), "${foo}"); + } + + #[test] + fn scan_returns_none_off_a_var() { + let src = "plain text"; + assert!(scan_var_ref(src, 2).is_none()); + } +} diff --git a/vw-htcl/src/signature_help.rs b/vw-htcl/src/signature_help.rs new file mode 100644 index 0000000..1929262 --- /dev/null +++ b/vw-htcl/src/signature_help.rs @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Signature help for htcl proc calls. +//! +//! When the cursor is inside a call to a known `proc`, report that +//! proc's signature and which parameter is "active" so the editor can +//! highlight it. The active parameter is the one named by the most +//! recent `-flag` typed on the line; before any flag is typed there is +//! no active parameter (the whole signature is shown). +//! +//! Pure analysis, like [`crate::complete`]: the LSP backend turns the +//! returned [`SignatureHelp`] into `lsp_types::SignatureHelp`. + +use crate::ast::{CommandKind, Document, ProcSignature, Stmt}; +use crate::cmdline::{self, CmdLine}; + +#[derive(Clone, Debug)] +pub struct SignatureHelp<'a> { + pub proc_name: String, + pub signature: &'a ProcSignature, + /// Proc-level doc comments (`##` above the declaration). + pub doc_comments: &'a [String], + /// Index into `signature.args` of the parameter under the cursor, + /// if one is determinable. + pub active_parameter: Option, +} + +/// Signature help for the call the cursor at `offset` is inside, or +/// `None` if the cursor isn't in a known proc call. +pub fn signature_help_at<'a>( + document: &'a Document, + source: &str, + offset: u32, +) -> Option> { + let line = cmdline::analyze(source, offset); + // `command_name` is `None` while the cursor is still on the first + // word, which is exactly when there's no call to describe yet. + let name = line.command_name()?; + let (signature, doc_comments) = find_proc(document, name)?; + Some(SignatureHelp { + proc_name: name.to_string(), + signature, + doc_comments, + active_parameter: active_parameter(signature, &line), + }) +} + +fn find_proc<'a>( + document: &'a Document, + name: &str, +) -> Option<(&'a ProcSignature, &'a [String])> { + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Proc(proc) = &cmd.kind else { + continue; + }; + if proc.name.as_deref() == Some(name) { + return Some((proc.signature.as_ref()?, &cmd.doc_comments)); + } + } + None +} + +/// The active parameter is the arg named by the most recent `-flag` +/// token. Complete flags must name an arg exactly; a flag still being +/// typed (the partial word) matches by prefix so the highlight tracks +/// as the user types. +fn active_parameter(sig: &ProcSignature, line: &CmdLine<'_>) -> Option { + let mut active = None; + for word in line.words.iter().skip(1) { + if let Some(flag) = word.strip_prefix('-') { + if let Some(i) = sig.args.iter().position(|a| a.name == flag) { + active = Some(i as u32); + } + } + } + if let Some(flag) = line.partial.strip_prefix('-') { + if !flag.is_empty() { + if let Some(i) = + sig.args.iter().position(|a| a.name.starts_with(flag)) + { + return Some(i as u32); + } + } + } + active +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn cursor(src_with_marker: &str) -> (String, u32) { + let offset = src_with_marker.find('|').expect("no cursor marker"); + (src_with_marker.replacen('|', "", 1), offset as u32) + } + + fn help(src_with_marker: &str) -> Option<(String, Option)> { + let (src, off) = cursor(src_with_marker); + let parsed = parse(&src); + signature_help_at(&parsed.document, &src, off) + .map(|h| (h.proc_name, h.active_parameter)) + } + + #[test] + fn shows_signature_after_name() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg |\n"; + let (name, active) = help(src).unwrap(); + assert_eq!(name, "cfg"); + assert_eq!(active, None); + } + + #[test] + fn active_parameter_follows_last_flag() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -depth |\n"; + let (_, active) = help(src).unwrap(); + assert_eq!(active, Some(1)); + } + + #[test] + fn active_parameter_tracks_partial_flag() { + let src = "\ +proc cfg {\n width\n depth\n} { }\n\ +cfg -wid|\n"; + let (_, active) = help(src).unwrap(); + assert_eq!(active, Some(0)); + } + + #[test] + fn none_while_typing_proc_name() { + let src = "\ +proc cfg {\n width\n} { }\n\ +cf|\n"; + assert!(help(src).is_none()); + } + + #[test] + fn none_for_unknown_command() { + let src = "puts |\n"; + assert!(help(src).is_none()); + } + + #[test] + fn works_inside_proc_body() { + let src = "\ +proc helper {\n size\n} { }\n\ +proc outer {} {\n helper -size |\n}\n"; + let (name, active) = help(src).unwrap(); + assert_eq!(name, "helper"); + assert_eq!(active, Some(0)); + } +} diff --git a/vw-htcl/src/span.rs b/vw-htcl/src/span.rs new file mode 100644 index 0000000..5dfb736 --- /dev/null +++ b/vw-htcl/src/span.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Byte-offset spans over source text. + +use std::ops::Range; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Span { + pub start: u32, + pub end: u32, +} + +impl Span { + pub const fn new(start: u32, end: u32) -> Self { + Self { start, end } + } + + pub fn range(self) -> Range { + self.start as usize..self.end as usize + } + + pub fn slice(self, source: &str) -> &str { + &source[self.range()] + } + + pub fn merge(self, other: Span) -> Span { + Span::new(self.start.min(other.start), self.end.max(other.end)) + } + + /// Translate this span by `delta` bytes. Used to lift spans from a + /// sub-parse (e.g. a proc body parsed as its own fragment) back + /// into whole-source coordinates. + pub const fn shifted(self, delta: u32) -> Span { + Span::new(self.start + delta, self.end + delta) + } + + /// True if `offset` lies within this span (start-inclusive, + /// end-inclusive). End-inclusive is the right call for hover and + /// "what's at the cursor" queries: a cursor visually positioned + /// right after a token is still on it. + pub fn contains(self, offset: u32) -> bool { + offset >= self.start && offset <= self.end + } +} + +impl From> for Span { + fn from(range: Range) -> Self { + Self { + start: range.start as u32, + end: range.end as u32, + } + } +} diff --git a/vw-htcl/src/src_path.rs b/vw-htcl/src/src_path.rs new file mode 100644 index 0000000..7b527b9 --- /dev/null +++ b/vw-htcl/src/src_path.rs @@ -0,0 +1,402 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Classification and resolution of `src` import paths. +//! +//! The plan defines three path shapes: +//! +//! - `relative/path` — relative to the importing file's directory. +//! - `/absolute/path` — filesystem-absolute (allowed but discouraged). +//! - `@name/path` — resolved via `vw.toml`'s `[dependencies.]` +//! entry; the cached repo root comes from `vw-lib`'s dependency +//! resolver and `` plus the rest of the path identify a file +//! in that repo. +//! +//! Resolution is split into two stages so the parser/AST side has no +//! filesystem dependency: [`classify`] decides which shape a path is, +//! [`Resolver`] turns a classified path into an actual on-disk file. + +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PathKind { + /// Relative to the importing file's directory. + Relative, + /// Filesystem-absolute (starts with `/`). + Absolute, + /// Resolved via a workspace dependency named `name`. `subpath` is + /// the rest of the path after `@name/` (may be empty). + Named { name: String, subpath: String }, +} + +#[derive(Clone, Debug)] +pub struct ClassifiedPath<'a> { + pub kind: PathKind, + /// The original path text, retained for diagnostics. + pub raw: &'a str, +} + +/// Classify an import path. Doesn't touch the filesystem. +pub fn classify(path: &str) -> ClassifiedPath<'_> { + let kind = if let Some(rest) = path.strip_prefix('@') { + let (name, subpath) = match rest.split_once('/') { + Some((n, s)) => (n.to_string(), s.to_string()), + None => (rest.to_string(), String::new()), + }; + PathKind::Named { name, subpath } + } else if path.starts_with('/') { + PathKind::Absolute + } else { + PathKind::Relative + }; + ClassifiedPath { kind, raw: path } +} + +#[derive(Debug, thiserror::Error)] +pub enum ResolveError { + #[error( + "unknown dependency `{name}` in `src @{name}{}`; \ + add a `[dependencies.{name}]` entry to your workspace's \ + vw.toml or run `vw add` to fetch it", + if .subpath.is_empty() { String::new() } else { format!("/{}", .subpath) } + )] + UnknownDependency { name: String, subpath: String }, + + #[error("imported file does not exist: {path}")] + NotFound { path: PathBuf }, + + #[error( + "import path `{raw}` reduces to an empty file path; \ + a `src` must name a real file" + )] + EmptyPath { raw: String }, +} + +/// Bare `src @` resolves to `/{DEFAULT_MODULE}.htcl`. +/// The convention is intentionally fixed (no `vw.toml` knob) so every +/// htcl module is laid out the same way — a reader can open +/// `module.htcl` and know they're at the entry point. +/// +/// The same convention applies to any directory that appears in a +/// `src` path — `src ip` where `ip/` is a directory resolves to +/// `ip/{DEFAULT_MODULE}.htcl` (analogous to Rust's `mod foo;` +/// picking `foo/mod.rs` when `foo.rs` is absent). +pub const DEFAULT_MODULE: &str = "module"; + +/// Resolver that turns import paths into on-disk file paths. Construct +/// one per workspace and reuse it across imports. +/// +/// Named deps are looked up in `cached_deps`, a `name → cache root` +/// map normally built from `vw.lock` via `vw-lib`. The caller is +/// responsible for filling this in — the htcl crate stays free of +/// `vw-lib` and filesystem-cache concerns. +#[derive(Clone, Debug, Default)] +pub struct Resolver { + cached_deps: std::collections::HashMap, +} + +impl Resolver { + pub fn new() -> Self { + Self::default() + } + + /// Register a dependency's cached root path (typically + /// `~/.vw/deps/-`). + pub fn with_dep(mut self, name: impl Into, root: PathBuf) -> Self { + self.cached_deps.insert(name.into(), root); + self + } + + /// Same as [`with_dep`], but only registers `name` when no + /// entry already exists. Cargo-parity semantic for + /// self-injecting the enclosing workspace as `@` + /// — a user-declared dep with the same name (rare but + /// possible) still wins. + pub fn with_dep_if_absent( + mut self, + name: impl Into, + root: PathBuf, + ) -> Self { + let name = name.into(); + self.cached_deps.entry(name).or_insert(root); + self + } + + /// Iterate the registered dependencies as `(name, root)` pairs. + /// Order is unspecified — callers that care should sort. + pub fn deps(&self) -> impl Iterator { + self.cached_deps + .iter() + .map(|(k, v)| (k.as_str(), v.as_path())) + } + + /// Look up a dependency's cached root by name. + pub fn dep_root(&self, name: &str) -> Option<&Path> { + self.cached_deps.get(name).map(PathBuf::as_path) + } + + /// Resolve `path` (as written in a `src` statement) against the + /// directory containing the importing file. Returns the canonical + /// path to the imported file, with `.htcl` appended if absent. + pub fn resolve( + &self, + importing_file_dir: &Path, + path: &str, + ) -> Result { + let classified = classify(path); + let candidate = match &classified.kind { + PathKind::Relative => importing_file_dir.join(path), + PathKind::Absolute => PathBuf::from(path), + PathKind::Named { name, subpath } => { + let Some(root) = self.cached_deps.get(name) else { + return Err(ResolveError::UnknownDependency { + name: name.clone(), + subpath: subpath.clone(), + }); + }; + // Bare `@` resolves to the dep's default entry + // point — `module.htcl` at the dep root, analogous to + // Rust's `src/lib.rs`. `@/` still picks a + // specific module under the dep. + if subpath.is_empty() { + root.join(DEFAULT_MODULE) + } else { + root.join(subpath) + } + } + }; + + // Directory-as-module: `src ip` where `ip/` is a real + // directory containing `module.htcl` resolves to + // `ip/module.htcl`. Mirrors the bare-`@dep` behavior — a + // dep root and an in-tree subdirectory both use + // `module.htcl` as the entry point — and gives users the + // Rust-style choice between `foo.htcl` and + // `foo/module.htcl` for a growing module. Checked BEFORE + // the `.htcl` append so a directory with a sibling + // `.htcl` file favors the file (predictable when + // both happen to exist during a rename). + if candidate.extension().is_none() { + let with_ext = candidate.with_extension("htcl"); + if with_ext.exists() { + return Ok(with_ext.canonicalize().unwrap_or(with_ext)); + } + if candidate.is_dir() { + let module = + candidate.join(DEFAULT_MODULE).with_extension("htcl"); + if module.exists() { + return Ok(module.canonicalize().unwrap_or(module)); + } + // Directory exists but has no module.htcl — surface + // the module path in the error so the fix is obvious + // ("create ip/module.htcl") rather than pointing at + // the sibling `.htcl` we tried first. + return Err(ResolveError::NotFound { path: module }); + } + return Err(ResolveError::NotFound { path: with_ext }); + } + + // Path already carries an extension — take it verbatim. + if !candidate.exists() { + return Err(ResolveError::NotFound { path: candidate }); + } + Ok(candidate.canonicalize().unwrap_or(candidate)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn classify_relative() { + assert_eq!(classify("foo/bar").kind, PathKind::Relative); + assert_eq!(classify("bar").kind, PathKind::Relative); + } + + #[test] + fn classify_absolute() { + assert_eq!(classify("/opt/x/y").kind, PathKind::Absolute); + } + + #[test] + fn classify_named() { + assert_eq!( + classify("@quartz/ip/bacd").kind, + PathKind::Named { + name: "quartz".into(), + subpath: "ip/bacd".into() + } + ); + assert_eq!( + classify("@bare").kind, + PathKind::Named { + name: "bare".into(), + subpath: String::new() + } + ); + } + + fn fixture() -> (tempfile::TempDir, Resolver) { + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(dep_root.join("ip")).unwrap(); + fs::write(dep_root.join("ip").join("bacd.htcl"), "## stub\n").unwrap(); + fs::write(dir.path().join("local.htcl"), "## local\n").unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root); + (dir, resolver) + } + + #[test] + fn resolve_relative_appends_htcl() { + let (dir, resolver) = fixture(); + let resolved = resolver.resolve(dir.path(), "local").unwrap(); + assert_eq!( + resolved.file_name().and_then(|s| s.to_str()), + Some("local.htcl") + ); + } + + #[test] + fn resolve_named_dependency() { + let (dir, resolver) = fixture(); + let resolved = resolver.resolve(dir.path(), "@quartz/ip/bacd").unwrap(); + assert!(resolved.ends_with("dep/ip/bacd.htcl"), "{resolved:?}"); + } + + #[test] + fn bare_named_dep_resolves_to_module_htcl() { + // `src @quartz` → `/module.htcl` (analogous to + // Rust's `use crate` resolving to `src/lib.rs`). + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(&dep_root).unwrap(); + fs::write(dep_root.join("module.htcl"), "# entry\n").unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root.clone()); + let resolved = resolver.resolve(dir.path(), "@quartz").unwrap(); + assert!(resolved.ends_with("dep/module.htcl"), "{resolved:?}"); + } + + #[test] + fn unknown_dep_errors_cleanly() { + let (dir, resolver) = fixture(); + let err = resolver.resolve(dir.path(), "@nope/foo").unwrap_err(); + assert!( + matches!(err, ResolveError::UnknownDependency { .. }), + "{err:?}" + ); + } + + #[test] + fn missing_file_errors() { + let (dir, resolver) = fixture(); + let err = resolver.resolve(dir.path(), "does/not/exist").unwrap_err(); + assert!(matches!(err, ResolveError::NotFound { .. }), "{err:?}"); + } + + #[test] + fn directory_with_module_htcl_resolves() { + // `src ip` where `ip/` is a directory with `module.htcl` + // inside → `ip/module.htcl`. The layout the metroid + // project switched to after reorganizing per-IP files + // into per-IP directories. + let dir = tempfile::tempdir().unwrap(); + let ip = dir.path().join("ip"); + fs::create_dir_all(&ip).unwrap(); + fs::write(ip.join("module.htcl"), "# entry\n").unwrap(); + let resolver = Resolver::new(); + let resolved = resolver.resolve(dir.path(), "ip").unwrap(); + assert!(resolved.ends_with("ip/module.htcl"), "{resolved:?}"); + } + + #[test] + fn sibling_htcl_wins_over_directory_module() { + // When both `foo.htcl` and `foo/module.htcl` exist, prefer + // the file. Predictable during a rename: users incrementally + // migrating a single-file module to a directory don't get + // a surprise resolution swap the moment the directory + // sprouts a `module.htcl`. + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("foo")).unwrap(); + fs::write(dir.path().join("foo.htcl"), "# file\n").unwrap(); + fs::write(dir.path().join("foo").join("module.htcl"), "# dir\n") + .unwrap(); + let resolver = Resolver::new(); + let resolved = resolver.resolve(dir.path(), "foo").unwrap(); + assert!(resolved.ends_with("foo.htcl"), "{resolved:?}"); + assert!(!resolved.ends_with("module.htcl"), "{resolved:?}"); + } + + #[test] + fn directory_without_module_htcl_errors_pointing_at_module() { + // Directory exists but lacks `module.htcl` — the error + // path should name the missing `module.htcl`, not the + // sibling `.htcl` the resolver also considered. That's + // what tells the user "add module.htcl here" instead of + // "create a sibling file." + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("ip")).unwrap(); + let resolver = Resolver::new(); + let err = resolver.resolve(dir.path(), "ip").unwrap_err(); + match err { + ResolveError::NotFound { path } => { + assert!(path.ends_with("ip/module.htcl"), "{path:?}"); + } + other => panic!("expected NotFound, got {other:?}"), + } + } + + #[test] + fn named_dep_subpath_can_be_directory_module() { + // `src @quartz/ip` where the dep has `ip/module.htcl` — + // same directory-as-module rule applies to subpaths of a + // named dep, not just to workspace-local paths. + let dir = tempfile::tempdir().unwrap(); + let dep_root = dir.path().join("dep"); + fs::create_dir_all(dep_root.join("ip")).unwrap(); + fs::write(dep_root.join("ip").join("module.htcl"), "# entry\n") + .unwrap(); + let resolver = Resolver::new().with_dep("quartz", dep_root); + let resolved = resolver.resolve(dir.path(), "@quartz/ip").unwrap(); + assert!(resolved.ends_with("dep/ip/module.htcl"), "{resolved:?}"); + } + + #[test] + fn with_dep_if_absent_leaves_existing_alone() { + // A user-declared dep of the same name must shadow the + // self-injection — same policy Cargo uses for the crate- + // self reference. Without this, a library that legitimately + // depends on an external `foo` couldn't also self-reference. + let existing = PathBuf::from("/tmp/existing"); + let new_path = PathBuf::from("/tmp/new"); + let resolver = Resolver::new() + .with_dep("foo", existing.clone()) + .with_dep_if_absent("foo", new_path); + assert_eq!(resolver.dep_root("foo"), Some(existing.as_path())); + } + + #[test] + fn with_dep_if_absent_registers_when_missing() { + let path = PathBuf::from("/tmp/self"); + let resolver = Resolver::new().with_dep_if_absent("self", path.clone()); + assert_eq!(resolver.dep_root("self"), Some(path.as_path())); + } + + #[test] + fn self_referential_workspace_resolves() { + // Simulate a library named `foo` that sources one of its + // own sibling modules via `src @foo/bar`. The resolver has + // `foo` self-injected to the workspace root, so `@foo/bar` + // resolves to `/bar.htcl`. + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("bar.htcl"), "## sib\n").unwrap(); + fs::write(dir.path().join("vw.toml"), "[workspace]\nname = \"foo\"\n") + .unwrap(); + let resolver = + Resolver::new().with_dep_if_absent("foo", dir.path().to_path_buf()); + let resolved = resolver.resolve(dir.path(), "@foo/bar").unwrap(); + assert!(resolved.ends_with("bar.htcl"), "{resolved:?}"); + } +} diff --git a/vw-htcl/src/type_parse.rs b/vw-htcl/src/type_parse.rs new file mode 100644 index 0000000..90e051d --- /dev/null +++ b/vw-htcl/src/type_parse.rs @@ -0,0 +1,457 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Mini-parser for htcl type expressions. +//! +//! Grammar: +//! +//! ```text +//! Type ::= Ident ('::' Ident | '<' Type (',' Type)* '>')? +//! Ident ::= [A-Za-z_] [A-Za-z0-9_]* +//! ``` +//! +//! The `Ident '::' Ident` form yields a [`TypeExpr::Qualified`], +//! used for the `Enum::Variant` annotations on overloaded handler +//! procs. The two forms (qualified vs generic) are mutually +//! exclusive — `Enum::Variant<…>` is a parse error. +//! +//! Whitespace is permitted between tokens but not within identifiers. +//! That's why type expressions with whitespace (`dict`) +//! must be brace-wrapped when used as a single htcl word — `dict` parses as four htcl words at the parent level, but +//! `{dict}` parses as one. The caller of [`parse`] is +//! responsible for that unwrap before handing us the type text. +//! +//! Spans returned are absolute source spans: the caller passes a +//! `base_offset` corresponding to the byte position of the first +//! character of `text` in the original source. + +use crate::ast::TypeExpr; +use crate::span::Span; + +/// One parse-error from the type parser. The caller renders these as +/// regular htcl parse-error diagnostics. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeParseError { + pub message: String, + pub span: Span, +} + +/// Parse `text` as a type expression, with absolute source positions +/// rooted at `base_offset`. Returns the parsed expression on success, +/// or the first error encountered. +pub fn parse(text: &str, base_offset: u32) -> Result { + let mut p = Parser::new(text, base_offset); + let ty = p.parse_type()?; + p.skip_ws(); + if !p.eof() { + return Err(TypeParseError { + message: format!( + "unexpected `{}` after type expression", + p.rest().chars().next().unwrap_or('\0') + ), + span: p.here_span(), + }); + } + Ok(ty) +} + +struct Parser<'a> { + text: &'a str, + bytes: &'a [u8], + pos: usize, + base: u32, +} + +impl<'a> Parser<'a> { + fn new(text: &'a str, base: u32) -> Self { + Self { + text, + bytes: text.as_bytes(), + pos: 0, + base, + } + } + + fn eof(&self) -> bool { + self.pos >= self.bytes.len() + } + + fn rest(&self) -> &str { + &self.text[self.pos..] + } + + fn skip_ws(&mut self) { + while self.pos < self.bytes.len() + && self.bytes[self.pos].is_ascii_whitespace() + { + self.pos += 1; + } + } + + fn here(&self) -> u32 { + self.base + self.pos as u32 + } + + /// Zero-width span at the current position — used for "unexpected + /// token" diagnostics where there's no real token to underline. + fn here_span(&self) -> Span { + let h = self.here(); + Span::new(h, h) + } + + fn span_from(&self, start: usize) -> Span { + Span::new(self.base + start as u32, self.base + self.pos as u32) + } + + /// Consume one bare identifier, returning its text and span. + /// Identifiers start with `[A-Za-z_]` and contain `[A-Za-z0-9_]`. + fn parse_ident(&mut self) -> Result<(String, Span), TypeParseError> { + self.skip_ws(); + let start = self.pos; + if self.eof() { + return Err(TypeParseError { + message: "expected type name, found end of input".into(), + span: self.here_span(), + }); + } + let first = self.bytes[self.pos]; + if !(first.is_ascii_alphabetic() || first == b'_') { + return Err(TypeParseError { + message: format!( + "expected type name, found `{}`", + first as char + ), + span: self.here_span(), + }); + } + self.pos += 1; + while self.pos < self.bytes.len() { + let c = self.bytes[self.pos]; + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + let name = self.text[start..self.pos].to_string(); + Ok((name, self.span_from(start))) + } + + fn parse_type(&mut self) -> Result { + let start = self.pos; + self.skip_ws(); + let ident_start = self.pos; + let (name, name_span) = self.parse_ident()?; + self.skip_ws(); + // Optional `::Variant` qualified-path suffix. Mutually + // exclusive with the `<…>` generic form — `E::V` is + // rejected below. + // + // Deeper chains (`A::B::C::D`) collapse into a `Named` type + // whose name is the whole colon-joined string. That's how + // generated wrappers can reference nested-namespace + // newtypes like `gtwiz_versal::intf0::gt_settings::Lr0Settings` + // without teaching the whole validator about + // multi-segment qualified paths (they never carry variant + // semantics — they're just deep newtype references). + if self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + self.pos += 2; // :: + let (variant, variant_span) = self.parse_ident()?; + self.skip_ws(); + // Third `::segment`? Keep going and produce a flat + // Named type with the whole joined path as its name. + if self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + let mut joined = format!("{name}::{variant}"); + while self.pos + 1 < self.bytes.len() + && self.bytes[self.pos] == b':' + && self.bytes[self.pos + 1] == b':' + { + self.pos += 2; + let (seg, _) = self.parse_ident()?; + joined.push_str("::"); + joined.push_str(&seg); + self.skip_ws(); + } + if !self.eof() && self.bytes[self.pos] == b'<' { + return Err(TypeParseError { + message: format!( + "nested-namespace type `{joined}` cannot take \ + generic arguments" + ), + span: self.here_span(), + }); + } + let span = self.span_from(start); + return Ok(TypeExpr::Named { name: joined, span }); + } + // Exactly two segments — the classic `Enum::Variant` + // shape used for overload dispatch. Preserve the + // Qualified form so the validator's variant-reference + // rules kick in. + // + // Reject `E::V<…>` — qualified names don't take generic + // args (their purpose is to name one variant of a + // declared enum, which has no type parameters in v1). + if !self.eof() && self.bytes[self.pos] == b'<' { + return Err(TypeParseError { + message: format!( + "qualified type `{name}::{variant}` cannot take \ + generic arguments" + ), + span: self.here_span(), + }); + } + return Ok(TypeExpr::Qualified { + namespace: name, + variant, + namespace_span: name_span, + variant_span, + span: self.span_from(start), + }); + } + // Optional `<...>` generic argument list. + if !self.eof() && self.bytes[self.pos] == b'<' { + self.pos += 1; // < + let mut args = Vec::new(); + // Allow empty? No — `list<>` is meaningless. Require + // at least one arg. + args.push(self.parse_type()?); + self.skip_ws(); + while !self.eof() && self.bytes[self.pos] == b',' { + self.pos += 1; // , + args.push(self.parse_type()?); + self.skip_ws(); + } + if self.eof() { + return Err(TypeParseError { + message: format!( + "unterminated generic type `{name}<…>`: \ + expected `>` or `,`", + ), + span: self.span_from(ident_start), + }); + } + if self.bytes[self.pos] != b'>' { + return Err(TypeParseError { + message: format!( + "expected `>` or `,` in generic type `{name}<…>`, \ + found `{}`", + self.bytes[self.pos] as char + ), + span: self.here_span(), + }); + } + self.pos += 1; // > + return Ok(TypeExpr::Generic { + name, + name_span, + args, + span: self.span_from(start), + }); + } + Ok(TypeExpr::Named { + name, + span: name_span, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(s: &str) -> TypeExpr { + parse(s, 0).unwrap_or_else(|e| panic!("parse failed: {e:?}")) + } + + #[test] + fn named_simple() { + let ty = p("string"); + match ty { + TypeExpr::Named { name, span } => { + assert_eq!(name, "string"); + assert_eq!(span, Span::new(0, 6)); + } + _ => panic!("expected Named"), + } + } + + #[test] + fn generic_single_arg() { + let ty = p("list"); + let TypeExpr::Generic { + name, args, span, .. + } = ty + else { + panic!("expected Generic"); + }; + assert_eq!(name, "list"); + assert_eq!(span, Span::new(0, 13)); + assert_eq!(args.len(), 1); + assert_eq!(args[0].name(), "bd_cell"); + } + + #[test] + fn generic_two_args() { + let ty = p("dict"); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!(); + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + assert_eq!(args[1].name(), "int"); + } + + #[test] + fn nested_generic() { + let ty = p("dict>"); + let TypeExpr::Generic { args, .. } = ty else { + panic!() + }; + assert_eq!(args.len(), 2); + assert_eq!(args[0].name(), "string"); + let TypeExpr::Generic { + name, args: inner, .. + } = &args[1] + else { + panic!("expected inner generic"); + }; + assert_eq!(name, "list"); + assert_eq!(inner[0].name(), "int"); + } + + #[test] + fn deeply_nested() { + let ty = p("list>"); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "list"); + let TypeExpr::Generic { + name: inner_name, + args: inner_args, + .. + } = &args[0] + else { + panic!(); + }; + assert_eq!(inner_name, "dict"); + assert_eq!(inner_args.len(), 2); + assert_eq!(inner_args[0].name(), "string"); + assert_eq!(inner_args[1].name(), "bd_cell"); + } + + #[test] + fn whitespace_between_tokens_is_fine() { + let ty = p(" dict < string , int > "); + let TypeExpr::Generic { name, args, .. } = ty else { + panic!() + }; + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + + #[test] + fn span_uses_base_offset() { + let ty = parse("bd_cell", 100).unwrap(); + let TypeExpr::Named { span, .. } = ty else { + panic!() + }; + assert_eq!(span, Span::new(100, 107)); + } + + #[test] + fn err_empty_input() { + let e = parse("", 0).unwrap_err(); + assert!(e.message.contains("expected type name")); + } + + #[test] + fn err_invalid_ident_start() { + let e = parse("", 0).unwrap_err(); + assert!( + e.message.contains("cannot take generic arguments"), + "{}", + e.message + ); + } + + #[test] + fn err_qualified_missing_variant() { + let e = parse("Property::", 0).unwrap_err(); + assert!(e.message.contains("expected type name"), "{}", e.message); + } + + #[test] + fn err_single_colon_not_qualified() { + // `Property:Scalar` (one colon) — not a qualified form. The + // first ident parses, then the trailing `:Scalar` is junk. + let e = parse("Property:Scalar", 0).unwrap_err(); + assert!(e.message.contains("unexpected")); + } +} diff --git a/vw-htcl/src/undefined.rs b/vw-htcl/src/undefined.rs new file mode 100644 index 0000000..acec5dc --- /dev/null +++ b/vw-htcl/src/undefined.rs @@ -0,0 +1,780 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Undefined-variable error pass. +//! +//! Emits a [`Diagnostic`] with severity [`Severity::Error`] for every +//! `$name` reference whose name isn't in the enclosing scope's decl +//! set. Same scope model as [`crate::unused`] (proc args + `set` + +//! `foreach` + `upvar`, with body-host bodies contributing decls to +//! the enclosing scope), inverted: uses that lack a matching decl. +//! +//! **What we catch.** The motivating case is the typo +//! `set _dcmac [ … ]; return $dcmac` from dcmac.htcl — obvious +//! misspelling, previously only caught at runtime by Vivado. +//! +//! **What we don't catch.** Full Tcl-style scoping (deep `upvar N` +//! traversal, `interp` / child interpreters, `global`/`variable` +//! cross-frame refs) is out of scope. Cross-file globals aren't +//! tracked either — the pass looks at one document at a time. +//! +//! **Escape hatches.** Scope-leak suppression (same policy as +//! `unused.rs`): a scope containing `eval $x` / `uplevel N $x` / +//! `apply $x` / `info exists $var` is opaque, so we emit no undef +//! errors for it. A small implicit-name whitelist (`env`, +//! `errorInfo`, `errorCode`, `tcl_platform`, `tcl_version`, `argv`, +//! `argv0`, `_`) covers Tcl's environment-provided names. + +use std::collections::{HashMap, HashSet}; + +use crate::ast::{ + Command, CommandKind, Document, Stmt, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::span::Span; +use crate::unused::{ + collect_decls, reparse_braced_body, scope_is_leaked, DeclSite, +}; +use crate::validate::{Diagnostic, Severity}; + +/// Names pre-defined in every Tcl scope. Referencing any of these +/// never produces an undefined-variable error. +const IMPLICITS: &[&str] = &[ + "env", + "errorInfo", + "errorCode", + "tcl_platform", + "tcl_version", + "tcl_pkgPath", + "tcl_library", + "tcl_patchLevel", + "argv", + "argv0", + "argc", + "_", +]; + +/// Collect the names of every variable defined at the document's +/// top level. Same collection rules as the undef pass — `set` +/// LHS, `foreach` iterator vars, `upvar` locals, `catch` result +/// vars, `regexp`/`regsub` capture vars, and any of the above +/// inside body-host `if`/`while`/… bodies that run in the top- +/// level frame. Used by the REPL to accumulate a name set across +/// batches so `set p …` in batch N-1 doesn't cause a false- +/// positive `undefined variable $p` in batch N. +pub fn top_level_var_names( + document: &Document, + source: &str, +) -> HashSet { + let mut decls: HashMap = HashMap::new(); + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + crate::unused::collect_decls(cmd, source, &mut decls); + } + decls.into_keys().collect() +} + +/// Companion to [`top_level_var_names`] that also returns the +/// inferred type of each top-level `set VAR ` binding when +/// the RHS's type is statically knowable. Used by the REPL to +/// carry variable-type context across batches so `putr $foo` in +/// batch N sees the type that batch N-1's `set foo […]` produced. +/// +/// Only the whole-word `[proc-call]`, `$var-copy`, and bare +/// `true`/`false` shapes are typed — everything else stays out +/// (matches [`crate::validate::value_type`]'s coverage). Missing +/// entries are fine: the caller merges these into an initial +/// `VarTypeTable` and falls back to plain `puts` when a name +/// isn't present. +pub fn top_level_var_types( + document: &Document, + sig_table: &HashMap, +) -> HashMap { + use crate::ast::CommandKind; + // Threaded var_table so a later `set y $x` picks up the type + // an earlier `set x [typed_proc]` recorded — matches the + // rewrite walker's own scope discipline for consistency. + let mut var_table = crate::validate::VarTypeTable::new(); + // Proc table for return-type INFERENCE on unannotated procs. + // A user proc like `proc configure_gtm {} { set cfg [typed]; …; return $cfg }` + // has no annotated return type — `value_type` alone would + // report None for `[configure_gtm]`. The proc-table lookup + // lets `value_type_with_procs` walk the body's last `return` + // to figure out the type flows out. Without this, `putr + // $_gtm` after `set _gtm [configure_gtm]` falls to plain + // puts and dumps the raw tagged tree. + let proc_table = crate::validate::build_proc_table(document); + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if !matches!(cmd.kind, CommandKind::Set) { + continue; + } + let Some(name_word) = cmd.words.get(1) else { + continue; + }; + let Some(value_word) = cmd.words.get(2) else { + continue; + }; + let Some(name) = name_word.as_text() else { + continue; + }; + if let Some(ty) = crate::validate::value_type_with_procs( + value_word, + sig_table, + &var_table, + Some(&proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + var_table +} + +/// Top-level entry. Walks the document as one scope (for top-level +/// `set`/`$var` references), then recurses into every proc body / +/// namespace-eval body as an independent scope. +pub fn validate_undefined_vars( + document: &Document, + source: &str, + diags: &mut Vec, +) { + validate_undefined_vars_with_extras( + document, + source, + &HashSet::new(), + diags, + ); +} + +/// Same as [`validate_undefined_vars`], with an extra pool of +/// top-level variable names the caller injects as "already +/// defined" — used by the REPL so a `set p …` in batch N-1 makes +/// `$p` in batch N legal. Only applies at the DOCUMENT top level; +/// proc bodies start with just their own args + `set`s (Tcl +/// semantics — top-level vars aren't visible inside a proc +/// without `global`/`upvar`, so leaking session state in would +/// mask real bugs). +pub fn validate_undefined_vars_with_extras( + document: &Document, + source: &str, + extra_top_level: &HashSet, + diags: &mut Vec, +) { + walk_scope(&document.stmts, source, extra_top_level, diags); +} + +/// One scope pass — collect decls, walk uses (with spans), emit +/// errors for each use that has no matching decl and isn't implicit. +/// The `extra_decls` param is only non-empty for the document top +/// level (see [`validate_undefined_vars_with_extras`]); recursive +/// scope walks pass an empty set. +fn walk_scope( + stmts: &[Stmt], + source: &str, + extra_decls: &HashSet, + diags: &mut Vec, +) { + let mut decls: HashMap = HashMap::new(); + for name in extra_decls { + decls.insert( + name.clone(), + DeclSite { + span: Span::new(0, 0), + kind: crate::unused::DeclKind::Set, + }, + ); + } + let mut use_sites: Vec<(String, Span)> = Vec::new(); + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + collect_decls(cmd, source, &mut decls); + collect_use_sites_in_command(cmd, source, &mut use_sites); + } + if !scope_is_leaked(stmts, source) { + emit_undefined(&decls, &use_sites, diags); + } + // Descend into fresh-frame children (proc bodies, namespace eval + // bodies) regardless — the outer scope's leak doesn't taint them. + // Extras only apply to the document top level; recursive walks + // pass an empty set (see [`validate_undefined_vars_with_extras`] + // docstring for the rationale). + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + descend_scopes(cmd, source, diags); + } +} + +/// Recurse into scope-establishing children of `cmd`. Nested procs +/// and `namespace eval` bodies each get their own `walk_scope`. +/// Mirrors `unused::descend_scopes` but with the undef pass's decl +/// seeding + use collection. +fn descend_scopes(cmd: &Command, source: &str, diags: &mut Vec) { + match &cmd.kind { + CommandKind::Proc(proc) => { + let mut decls: HashMap = HashMap::new(); + let mut use_sites: Vec<(String, Span)> = Vec::new(); + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + decls.insert( + arg.name.clone(), + DeclSite { + span: arg.name_span, + // The DeclKind values are private to + // unused.rs's warning-message dispatch; + // we never read them here, but the field + // must be set to something. Use ProcArg + // as the closest match — this DeclSite + // exists purely as a "name is defined" + // marker for our lookup. + kind: crate::unused::DeclKind::ProcArg, + }, + ); + } + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, &mut decls); + collect_use_sites_in_command(inner, source, &mut use_sites); + } + if !scope_is_leaked(&proc.body, source) { + emit_undefined(&decls, &use_sites, diags); + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + descend_scopes(inner, source, diags); + } + } + CommandKind::NamespaceEval(ns) => { + walk_scope(&ns.body, source, &HashSet::new(), diags); + } + _ => {} + } +} + +/// Walk `cmd.words` (and any body-host braced-body interiors) and +/// record every `WordPart::VarRef` as a (name, span) pair. +/// +/// Body-host bodies run in the enclosing frame, so their `$foo` +/// references count as uses of the current scope. Skip `proc` and +/// `namespace eval` — those open fresh scopes and `descend_scopes` +/// handles them. +fn collect_use_sites_in_command( + cmd: &Command, + source: &str, + use_sites: &mut Vec<(String, Span)>, +) { + // `set foo` (exactly 2 words) is a read of $foo — mirror the + // unused pass's convention. + if matches!(cmd.kind, CommandKind::Set) && cmd.words.len() == 2 { + if let Some(name) = cmd.words[1].as_text() { + use_sites.push((name.to_string(), cmd.words[1].span)); + } + } + for word in &cmd.words { + collect_use_sites_in_word(word, source, use_sites); + } + // Body-host recursion for uses. Same shape as `collect_decls`'s + // recursion — proc / namespace eval are excluded. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) + && !matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) + { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { continue }; + collect_use_sites_in_command(inner, source, use_sites); + } + } + } + } + } +} + +fn collect_use_sites_in_word( + word: &Word, + source: &str, + use_sites: &mut Vec<(String, Span)>, +) { + // Braced literals are opaque in Tcl — `puts {$foo}` prints the + // literal string `$foo`, not the value of `$foo`. The parser + // encodes this by making `Braced` words carry a single Text + // part with no VarRef sub-parts, so the loop below naturally + // skips them. + for part in &word.parts { + match part { + WordPart::VarRef { name, span, .. } => { + use_sites.push((name.clone(), *span)); + } + WordPart::CmdSubst { body, .. } => { + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_use_sites_in_command(inner, source, use_sites); + } + } + WordPart::Text { .. } | WordPart::Escape { .. } => {} + } + } + // Defensive: on the off-chance a Braced word carries VarRef + // sub-parts (parser change / future extension), skip them — + // Tcl braced-word semantics prevail. If future parsers surface + // sub-refs from a `Quoted` word, they'll flow through above. + if word.form == WordForm::Braced { + // No-op: the loop above already handled the (only) Text + // part. This block exists as documentation and a place to + // add a diagnostic if the invariant ever breaks. + } +} + +/// For each use-site whose name isn't defined and isn't implicit, +/// emit an `undefined variable` error. The message includes a +/// "did you mean" hint when a decl within edit-distance 2 exists. +fn emit_undefined( + decls: &HashMap, + use_sites: &[(String, Span)], + diags: &mut Vec, +) { + // Dedupe by span so a name referenced twice at the same span + // (shouldn't happen, but cheap insurance) only emits once. + let mut seen: HashSet<(String, u32, u32)> = HashSet::new(); + // Emit in source order for stable output. + let mut items: Vec<&(String, Span)> = use_sites.iter().collect(); + items.sort_by_key(|(_, sp)| (sp.start, sp.end)); + for (name, span) in items { + if !seen.insert((name.clone(), span.start, span.end)) { + continue; + } + // The base name for `arr(key)` subscripts — if `arr` is + // defined, the subscript form is a legal reference to it. + let base = match name.find('(') { + Some(idx) => &name[..idx], + None => name.as_str(), + }; + if decls.contains_key(base) || decls.contains_key(name) { + continue; + } + if is_implicit(base) || is_implicit(name) { + continue; + } + // Numeric names ($1, $2) — regex submatch refs and the + // like. Skip. + if name.parse::().is_ok() { + continue; + } + let hint = suggest(name, decls); + let message = match hint { + Some(sug) => { + format!("undefined variable `${name}`; did you mean `${sug}`?") + } + None => format!("undefined variable `${name}`"), + }; + diags.push(Diagnostic { + severity: Severity::Error, + message, + span: *span, + }); + } +} + +fn is_implicit(name: &str) -> bool { + if IMPLICITS.contains(&name) { + return true; + } + // Compiler-provided kwargs presence flags. The vw-htcl lowering + // injects a `::vw::kwargs` shim call at proc entry that sets a + // `__vw_kw__set` boolean for every optional kwarg, so + // proc bodies (both hand-written and generator-emitted) can + // check `${__vw_kw_foo_set}` to see whether the user passed a + // value. These never appear as source-level decls; treat as + // pre-defined in every scope. See vw-ip/src/generate.rs's + // `emit_dict_proc` for the emission side. + if let Some(rest) = name.strip_prefix("__vw_kw_") { + if rest.ends_with("_set") { + return true; + } + } + false +} + +/// Levenshtein-distance suggestion — returns the closest decl name +/// within distance 2, only for names of length ≥ 3 (below that, +/// every 2-char name is within distance 2 of every other, which +/// produces noisy hints). +fn suggest(name: &str, decls: &HashMap) -> Option { + if name.len() < 3 { + return None; + } + let mut best: Option<(usize, String)> = None; + for candidate in decls.keys() { + if candidate.len() < 3 { + continue; + } + let d = edit_distance(name, candidate); + if d > 2 { + continue; + } + if best.as_ref().is_none_or(|(bd, _)| d < *bd) { + best = Some((d, candidate.clone())); + } + } + best.map(|(_, s)| s) +} + +/// Iterative Levenshtein distance. Small strings only — no need +/// to optimize further for our decl-set sizes. +fn edit_distance(a: &str, b: &str) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let n = a.len(); + let m = b.len(); + if n == 0 { + return m; + } + if m == 0 { + return n; + } + let mut prev: Vec = (0..=m).collect(); + let mut curr: Vec = vec![0; m + 1]; + for i in 1..=n { + curr[0] = i; + for j in 1..=m { + let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + curr[j] = + (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost); + } + std::mem::swap(&mut prev, &mut curr); + } + prev[m] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn errors(src: &str) -> Vec { + let parsed = parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let mut out = Vec::new(); + validate_undefined_vars(&parsed.document, src, &mut out); + out.into_iter() + .filter(|d| d.severity == Severity::Error) + .collect() + } + + #[test] + fn dcmac_typo_flagged() { + // The motivating case: `set _dcmac …` then `return $dcmac`. + let src = "proc f {} { set _dcmac 1; return $dcmac }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$dcmac`"), + "{}", + e[0].message + ); + assert!( + e[0].message.contains("did you mean `$_dcmac`"), + "{}", + e[0].message + ); + } + + #[test] + fn defined_local_clean() { + let src = "proc f {} { set x 1; puts $x }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn proc_arg_clean() { + let src = "proc f {x} { puts $x }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn foreach_var_clean() { + let src = "proc f {xs} { foreach i $xs { puts $i } }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn foreach_iterator_available_after_loop() { + // Tcl semantics: foreach iterator persists after the loop. + let src = "proc f {xs} { foreach i $xs { }; puts $i }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn braced_ref_not_flagged() { + // `puts {$foo}` — braced content is literal in Tcl, so + // `$foo` here is just the string `$foo`, not a var ref. + let src = "proc f {} { puts {$foo} }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn if_else_union_clean() { + // Tcl runs `if`/`else` bodies in the enclosing frame, so a + // `set x` inside either branch defines `x` in the outer + // scope. Reading `$x` after the `if` is legal. + let src = "\ +proc f {c} { + if { $c } { set x 1 } else { set x 2 } + puts $x +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn underscore_prefix_no_alias() { + // `_foo` and `foo` are distinct names. The unused-var + // pass's `_`-prefix escape hatch does NOT create a $foo + // alias for a set _foo decl. + let src = "proc f {} { set _foo 1; puts $foo }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$foo`"), + "{}", + e[0].message + ); + } + + #[test] + fn implicit_env_clean() { + // `$env(HOME)` is a subscript on the `env` array, which is + // pre-defined by Tcl. + let src = "proc f {} { puts $env(HOME) }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn implicit_errorinfo_clean() { + let src = "proc f {} { puts $errorInfo }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn dynamic_eval_suppresses() { + // Scope leak — we can't tell what `eval $script` might + // reference, so suppress the whole scope's undef check. + let src = "proc f {} { eval $script; puts $mystery }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn top_level_flagged() { + let src = "set x 1\nputs $y\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$y`"), + "{}", + e[0].message + ); + } + + #[test] + fn top_level_defined_clean() { + // Matches the ~/sketch/metroid/project.htcl pattern: + // top-level `set proj [...]` then `... -proj $proj`. + let src = "set proj 1\nputs $proj\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn upvar_local_defined() { + let src = "proc f {} { upvar 1 remote local; puts $local }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn subscript_base_defined() { + // `$arr(key)` — `arr` is defined, so the array subscript + // reference is fine. + let src = "proc f {} { set arr 1; puts $arr(key) }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn cmd_subst_in_body() { + let src = "proc f {x} { set y [list $x]; puts $y }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn nested_scope_isolated() { + // Inner proc's `x` arg is distinct from outer scope. + // Outer `$outer_var` is undefined; inner uses `$x` cleanly. + let src = "\ +proc outer {} { + proc inner {x} { puts $x } + puts $outer_var +} +"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("undefined variable `$outer_var`"), + "{}", + e[0].message + ); + } + + #[test] + fn numeric_names_skipped() { + // `$1`, `$2` are typically regex submatch refs. Not flagged. + let src = "proc f {} { puts $1 }\n"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn typo_with_multiple_candidates() { + // Closest match by edit distance wins. + let src = "\ +proc f {} { + set apple 1 + set apricot 2 + puts $applet +} +"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!( + e[0].message.contains("did you mean `$apple`"), + "{}", + e[0].message + ); + } + + #[test] + fn port_htcl_repro_minimal() { + // Simplest form of the port.htcl false-positive: an `if + // {...} { continue }` in the foreach body between the + // `foreach` line and the `set val`. + let src = "\ +proc f {obj} { + foreach prop [list $obj] { + if {$prop eq \"\"} { continue } + set val 1 + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn port_htcl_repro() { + // The exact shape from ~/src/htcl/amd/vivado-cmd/port.htcl:116-124. + let src = "\ +proc f {obj} { + foreach prop [list $obj] { + if {![string match \"CONFIG.*\" $prop]} { continue } + if {[regexp {^CONFIG\\.foo$} $prop]} { continue } + set val [list $prop $obj] + if {$val eq \"\"} { continue } + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn dict_for_binds_key_and_value_vars() { + // `dict for {lib srcs} $deps { … }` — the two brace-list + // names bind for the body's lifetime, same as a `foreach + // {k v} $pairs { … }`. Before the fix, `$lib` / `$srcs` + // inside the body were flagged as undefined. + let src = "\ +proc f {deps} { + dict for {lib srcs} $deps { + puts $lib + puts $srcs + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn set_inside_foreach_body_defines_in_outer_scope() { + // Repro for the port.htcl false-positive: `foreach x [...] { + // set val [...]; puts $val }`. The `set val` is inside the + // foreach body, which runs in the enclosing frame per Tcl + // semantics, so `$val` on the next line is a legal ref. + let src = "\ +proc f {xs} { + foreach x $xs { + set val 1 + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn catch_result_var_is_a_decl() { + // `catch { … } n` — Tcl's catch binds the caught body's + // result into `n` (the enclosing scope's frame). Repro for + // lift.htcl:29 false-positive. + let src = "\ +proc f {raw} { + if {[catch {llength $raw} n]} { return 0 } + return $n +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn regexp_var_captures_are_decls() { + // `regexp {pattern} $s var1 var2 …` — every trailing var + // arg is a decl. Repro for props.htcl:121. + let src = "\ +proc f {s} { + if {[regexp {(.*)=(.*)} $s _all key val]} { + puts $key + puts $val + } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn vw_kwargs_flags_are_implicit() { + // The vw-htcl lowering injects `__vw_kw__set` boolean + // sentinels via a `::vw::kwargs` shim at proc entry — every + // optional kwarg gets one. These never appear as source- + // level decls, and generator output relies on them + // heavily. Treat as pre-defined in every scope. + let src = "\ +proc f {} { + if {${__vw_kw_config_c0_set}} { puts hi } +} +"; + assert!(errors(src).is_empty(), "{:?}", errors(src)); + } + + #[test] + fn short_names_no_suggestion_noise() { + // Names < 3 chars don't participate in the suggestion. + let src = "proc f {} { set ab 1; puts $xy }\n"; + let e = errors(src); + assert_eq!(e.len(), 1, "{:?}", e); + assert!(!e[0].message.contains("did you mean"), "{}", e[0].message); + } +} diff --git a/vw-htcl/src/unused.rs b/vw-htcl/src/unused.rs new file mode 100644 index 0000000..e716fca --- /dev/null +++ b/vw-htcl/src/unused.rs @@ -0,0 +1,1086 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Unused-variable warning pass. +//! +//! Emits a [`Diagnostic`] with severity [`Severity::Warning`] for every +//! local binding whose name never surfaces in the same scope's uses. +//! Slice 1 (current): proc args + `set` decls at both the top level +//! and inside proc bodies. No brace-body reparse yet (bodies of +//! `if`/`while`/`foreach`/etc. are opaque and can hide uses), so the +//! pass tolerates false-negatives but never emits false-positives. +//! Later slices add brace-body reparse and per-construct escape +//! hatches for `upvar`/`uplevel`/`eval`/`apply`/`info`. +//! +//! **Escape hatch.** A leading `_` on a name suppresses the warning +//! for that decl — `_ignored`, `_unused`, `_` alone all count. + +use std::collections::{HashMap, HashSet}; + +use crate::ast::{ + Command, CommandKind, Document, Stmt, Word, WordForm, WordPart, +}; +use crate::hover::is_body_host; +use crate::span::Span; +use crate::validate::{Diagnostic, Severity}; + +/// Kind of local binding — drives the diagnostic message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DeclKind { + ProcArg, + Set, + ForeachVar, + Upvar, +} + +/// Where a name was declared and by what construct. +#[derive(Clone, Copy, Debug)] +pub(crate) struct DeclSite { + pub(crate) span: Span, + pub(crate) kind: DeclKind, +} + +/// Top-level entry. Walks the document as one scope (for top-level +/// `set` decls), then recurses into every proc body / namespace-eval +/// body as its own independent scope. +pub fn validate_unused_vars( + document: &Document, + source: &str, + diags: &mut Vec, +) { + walk_scope(&document.stmts, source, diags); +} + +/// Collect decls + uses over a flat list of statements as one scope, +/// then emit warnings for decls whose names never appear as uses. +/// Recurses into `NamespaceEval.body` and each `Proc.body` as fresh +/// scopes. +/// +/// If the scope contains a dynamic-script construct we can't see +/// through (`eval $x`, `uplevel N $x`, `apply $x`) we suppress the +/// warnings for this scope but still descend into nested scopes — +/// those are unaffected. +fn walk_scope(stmts: &[Stmt], source: &str, diags: &mut Vec) { + let mut decls: HashMap = HashMap::new(); + let mut uses: HashSet = HashSet::new(); + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + collect_decls(cmd, source, &mut decls); + collect_uses_in_command(cmd, source, &mut uses); + } + if !scope_is_leaked(stmts, source) { + emit_unused(&decls, &uses, diags); + } + // Descend into nested scopes regardless — a leak in the outer + // scope doesn't taint an inner proc's locals. + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + descend_scopes(cmd, source, diags); + } +} + +/// True when this scope contains a construct that could reference +/// locals by dynamically-computed names. Presence of any of the +/// following counts as a leak: +/// +/// - `eval` with a non-literal script arg (`$x`, `"…$x…"`). +/// - `uplevel LEVEL` with a non-literal script arg (any LEVEL — +/// even LEVEL=0 with a dynamic body is unpeekable). +/// - `apply` with a non-literal envelope word (`apply $x …`). +/// - `info level`, `info vars`, `info exists` with a dynamic arg. +/// +/// Scans this scope's statements plus any brace-body interiors +/// that Slice 2's reparse would walk — same-frame constructs +/// (`if`/`while`/etc.) can leak from inside their bodies too. +pub(crate) fn scope_is_leaked(stmts: &[Stmt], source: &str) -> bool { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + if command_leaks(cmd) { + return true; + } + // Recurse into any brace-body interior — same frame, so + // a leak there leaks the outer scope. Only body-hosts + // (`if`/`while`/`foreach`/…) have such bodies. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(inner) = reparse_braced_body(word, source) { + if scope_is_leaked(&inner, source) { + return true; + } + } + } + } + } + } + false +} + +/// True when a single command is a scope-leak site. +fn command_leaks(cmd: &Command) -> bool { + let Some(head) = cmd.words.first().and_then(Word::as_text) else { + return false; + }; + match head { + "eval" | "uplevel" | "apply" => { + // Any non-literal arg → leak. Walk args, skipping the + // command name; if any word contains a VarRef or a + // CmdSubst it's dynamic. + cmd.words.iter().skip(1).any(word_is_dynamic) + } + "info" => { + // `info level`, `info vars`, `info exists` — all three + // are introspection over the current frame. When their + // arg is dynamic we can't tell which locals get named, + // so bail. Static forms (`info exists foo`) don't leak + // (they're a use of `foo`; `collect_uses_in_command` + // could later grow to record them, but for now the + // conservative treatment is to treat as leak only when + // dynamic). + let sub = cmd.words.get(1).and_then(Word::as_text); + match sub { + Some("level") | Some("vars") | Some("exists") => { + cmd.words.iter().skip(2).any(word_is_dynamic) + } + _ => false, + } + } + _ => false, + } +} + +/// True when `word` isn't a pure literal (contains a `$var` or +/// `[…]` substitution). +pub(crate) fn word_is_dynamic(word: &Word) -> bool { + word.parts.iter().any(|p| { + matches!(p, WordPart::VarRef { .. } | WordPart::CmdSubst { .. }) + }) +} + +/// Recurse into scope-establishing children of `cmd`. Nested procs +/// and `namespace eval` bodies each get their own `walk_scope` call. +fn descend_scopes(cmd: &Command, source: &str, diags: &mut Vec) { + match &cmd.kind { + CommandKind::Proc(proc) => { + // Fresh scope. Seed it with the proc's args before + // walking the body's statements. + let mut decls: HashMap = HashMap::new(); + let mut uses: HashSet = HashSet::new(); + if let Some(sig) = proc.signature.as_ref() { + for arg in &sig.args { + decls.insert( + arg.name.clone(), + DeclSite { + span: arg.name_span, + kind: DeclKind::ProcArg, + }, + ); + } + } + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, &mut decls); + collect_uses_in_command(inner, source, &mut uses); + } + if !scope_is_leaked(&proc.body, source) { + emit_unused(&decls, &uses, diags); + } + // And recurse into nested scopes inside the body. + for stmt in &proc.body { + let Stmt::Command(inner) = stmt else { continue }; + descend_scopes(inner, source, diags); + } + } + CommandKind::NamespaceEval(ns) => { + walk_scope(&ns.body, source, diags); + } + _ => {} + } +} + +/// If `cmd` binds a local, add it to `decls`. Recognizes: +/// - `set X value` (`CommandKind::Set` with `words.len() >= 3`) +/// - `foreach X list body` (Generic command with head `"foreach"`). +/// Both single-var (`words[1]` bare) and multi-var brace-list +/// (`words[1]` Braced containing whitespace-separated names) +/// forms are handled. The iterator var is declared in the +/// *enclosing* scope's frame per Tcl semantics — a `foreach x $list +/// {}` binding is visible after the loop returns. That means adding +/// the iterator to the same scope's decl map is correct. +/// +/// **Recursion into body-hosts.** Tcl's `if`/`while`/`for`/`foreach`/ +/// `catch`/`try` bodies run in the enclosing frame — a `set foo …` +/// inside `if { … } { … }` binds `foo` in the outer scope. So we +/// reparse each braced-body argument of a body-host and recurse. +/// Only `proc` and `namespace eval` bodies open fresh frames; those +/// are handled by [`descend_scopes`], not here. +pub(crate) fn collect_decls( + cmd: &Command, + source: &str, + decls: &mut HashMap, +) { + match &cmd.kind { + CommandKind::Set => { + // 2-word `set` is a read (`set foo` returns $foo). Only + // 3+-word forms are decls. + if cmd.words.len() < 3 { + return; + } + let target = &cmd.words[1]; + let Some(name) = target.as_text() else { + return; + }; + // First decl in the scope wins — Tcl reassignment + // doesn't create a new binding, and pointing at the + // original decl is what the user recognizes when + // hunting an unused local. + decls.entry(name.to_string()).or_insert(DeclSite { + span: target.span, + kind: DeclKind::Set, + }); + } + CommandKind::Generic => { + // Head-based recognition. A command whose first word + // isn't a plain identifier (e.g. `[cmd-subst]`) has + // no head we can dispatch on — skip this stage, but + // fall through to the body-host and cmd-subst recursion + // below (which walk the WORDS regardless). + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + match head { + "foreach" => collect_foreach_decls(cmd, decls), + "upvar" => collect_upvar_decls(cmd, decls), + "catch" => collect_catch_decls(cmd, decls), + "regexp" | "regsub" => collect_regexp_decls(cmd, decls), + // `dict for {kv} DICT BODY` — the varname list + // at words[2] binds one or two locals in the + // enclosing frame, same as `foreach`. + "dict" => collect_dict_for_decls(cmd, decls), + _ => {} + } + } + } + _ => {} + } + // Body-host recursion: `if`/`while`/`foreach`/`for`/`catch`/`try`/… + // bodies run in the enclosing frame. Reparse each braced-body arg + // and recurse — a `set foo …` inside binds `foo` in this scope. + // Skip `proc` and `namespace eval` (they open fresh scopes). + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) + && !matches!( + &cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) + { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, decls); + } + } + } + } + } + // Command-substitution recursion: `[set x 1]`, `if {[catch {…} n]} …`, + // and any other `[…]` embedded in a word runs in the enclosing + // frame. Its `set`/`catch`/`regexp`/… count as decls here. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_decls(inner, source, decls); + } + } + } + } +} + +/// Extract the *local* names from an `upvar` command. Syntax: +/// `upvar [LEVEL] remote local ?remote local ...?` +/// LEVEL is optional; when present it's a bare numeric or `#N` +/// prefix on the first arg. Rather than parse it precisely, we +/// probe: if the first arg after `upvar` looks like a level +/// (leading digit or `#`), skip it; then take pairs (remote, local). +/// +/// Every `local` becomes a decl. The `remote` names are opaque — +/// they refer to an outer frame we can't see. Dynamic-remote form +/// (`upvar $var local`) is fine: we can still see the *local* half +/// literally as a decl. +pub(crate) fn collect_upvar_decls( + cmd: &Command, + decls: &mut HashMap, +) { + let mut idx = 1; + // Skip the optional LEVEL: bare numeric or `#`-prefixed. + if let Some(w) = cmd.words.get(idx) { + if let Some(t) = w.as_text() { + if t.starts_with('#') + || t.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + idx += 1; + } + } + } + // Now consume (remote, local) pairs. + while idx + 1 < cmd.words.len() { + let local_word = &cmd.words[idx + 1]; + if let Some(name) = local_word.as_text() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: local_word.span, + kind: DeclKind::Upvar, + }); + } + idx += 2; + } +} + +/// Extract the result-var and options-var names from a `catch`. +/// +/// Syntax: `catch script ?resultVarName? ?optionsVarName?`. Both +/// trailing args (when literal identifiers) are decls in the +/// enclosing scope — catch runs the script in the current frame +/// and stores the return value into `resultVarName`. Dynamic +/// forms (`catch script $var`) are opaque; we skip them. +pub(crate) fn collect_catch_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // `catch script` (2 words) — no result var. + // `catch script name` (3 words) — name is result decl. + // `catch script name opts` (4 words) — both are decls. + for i in [2, 3] { + let Some(w) = cmd.words.get(i) else { break }; + let Some(name) = w.as_text() else { continue }; + decls.entry(name.to_string()).or_insert(DeclSite { + span: w.span, + kind: DeclKind::Set, + }); + } +} + +/// Extract capture-var names from a `regexp`/`regsub` command. +/// +/// `regexp ?switches? pattern string ?matchVar? ?subVar ...?` +/// Everything after the pattern + string that's a bare identifier +/// becomes a capture-var decl in the enclosing scope. Switches +/// (leading `-`) are skipped up to `--` or the first non-switch +/// word; the switch/pattern boundary heuristic here is: the first +/// non-`-` word is the pattern, the next is the string, and +/// everything else is a capture-var. That's the standard Tcl +/// shape; we skip precise switch-list parsing. +/// +/// `regsub` shares the same trailing-vars shape (the last arg is +/// the result-var). +pub(crate) fn collect_regexp_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // Skip the command word (index 0). Skip leading switches (any + // word beginning with `-` that isn't `--`). After the pattern + // + string, the rest are capture vars. + let mut i = 1; + while let Some(w) = cmd.words.get(i) { + let Some(t) = w.as_text() else { break }; + if t == "--" { + i += 1; + break; + } + if !t.starts_with('-') { + break; + } + i += 1; + } + // i is now the pattern arg. Skip it + the string arg. + i += 2; + // Remaining words are capture-var names (bare identifiers). + while let Some(w) = cmd.words.get(i) { + if let Some(name) = w.as_text() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: w.span, + kind: DeclKind::Set, + }); + } + i += 1; + } +} + +/// Extract the iterator variable(s) from a `foreach` command. +/// `foreach var $list {…}` — single var at `words[1]`. +/// `foreach {a b c} $list {…}` — multi-var brace list at `words[1]` +/// containing whitespace-separated names. +/// `foreach a $la b $lb {…}` — pairs form. We take every 2nd word +/// starting at 1 as an iterator target: words[1], words[3], … up to +/// `words.len() - 2` (last two words are the final list-value and +/// the body). +pub(crate) fn collect_foreach_decls( + cmd: &Command, + decls: &mut HashMap, +) { + if cmd.words.len() < 4 { + // `foreach var list body` minimum. Malformed: give up + // gracefully rather than emit a spurious decl. + return; + } + // The last word is the body; strip it, then every even-indexed + // remaining word (skipping the leading `foreach`) is an iter + // target. Odd-indexed remainders are the list values. + let body_idx = cmd.words.len() - 1; + let mut i = 1; + while i < body_idx { + let target = &cmd.words[i]; + add_foreach_target(target, decls); + i += 2; + } +} + +/// Extract the key/value binding names from `dict for {kv} DICT +/// BODY`. Only recognizes the `for` sub-command; other `dict` +/// forms (`dict get`, `dict set`, …) don't introduce bindings. +/// +/// The varname list at `words[2]` follows the same shape as +/// `foreach`'s first-arg target — either a bare word (rare; Tcl +/// requires exactly two names in the braced form for `dict for` +/// but the parser accepts anything) or a braced whitespace- +/// separated list. Both shapes flow through `add_foreach_target`. +pub(crate) fn collect_dict_for_decls( + cmd: &Command, + decls: &mut HashMap, +) { + // `dict for {kv} DICT BODY` needs at least 5 words. If the + // second word isn't `for`, this isn't the binding form — skip. + if cmd.words.len() < 5 { + return; + } + if cmd.words.get(1).and_then(Word::as_text) != Some("for") { + return; + } + add_foreach_target(&cmd.words[2], decls); +} + +fn add_foreach_target(target: &Word, decls: &mut HashMap) { + if target.form == WordForm::Braced { + // Multi-var brace list. The interior is a single Text part + // (the parser doesn't sub-split braced words). Whitespace- + // split and treat each token as a decl. + let Some(WordPart::Text { value, span }) = target.parts.first() else { + return; + }; + // Each token gets a fresh DeclSite whose span points at + // the containing braced word — good enough for a "the + // culprit is here" underline; sub-token spans would need + // extra parser wiring. + for name in value.split_whitespace() { + decls.entry(name.to_string()).or_insert(DeclSite { + span: *span, + kind: DeclKind::ForeachVar, + }); + } + return; + } + // Bare form: whole word is the iterator name. + let Some(name) = target.as_text() else { + return; + }; + decls.entry(name.to_string()).or_insert(DeclSite { + span: target.span, + kind: DeclKind::ForeachVar, + }); +} + +/// Walk `cmd.words` and every command substitution nested inside, +/// adding every `WordPart::VarRef` name to `uses`. If `cmd` is a +/// body-host construct (`if`/`while`/`foreach`/…), each `Braced` +/// argument is reparsed as a script fragment and its interior is +/// walked recursively — that reparse is what recovers false- +/// negatives from Slice 1 (variables used inside `if { $x > 0 } +/// { … }` etc.). +pub(crate) fn collect_uses_in_command( + cmd: &Command, + source: &str, + uses: &mut HashSet, +) { + // Scope-opening commands (`proc`, `namespace eval`) live in + // their own frame — a `$y` in a nested proc's body doesn't + // reference the outer scope's `y`. `descend_scopes` walks + // those bodies with a fresh decls/uses table; skip them here + // so the outer scope doesn't count their body-word contents + // as its own uses. + if matches!( + cmd.kind, + CommandKind::Proc(_) | CommandKind::NamespaceEval(_) + ) { + return; + } + // Special case: `set foo` (exactly 2 words) is a *read* of `foo`, + // not a decl. `collect_decls` correctly ignores this shape, but + // we also need to count it here as a use so a `set foo 1; set foo` + // doesn't warn `foo` as unused. + if matches!(cmd.kind, CommandKind::Set) && cmd.words.len() == 2 { + if let Some(name) = cmd.words[1].as_text() { + uses.insert(name.to_string()); + } + } + for word in &cmd.words { + collect_uses_in_word(word, source, uses); + } + // Body-host commands hide scripts inside braced words. Reparse + // each such word and walk its statements as if they were part + // of the current scope — Tcl runs them in the current frame + // (for `if`/`while`/`foreach`/`for`/`catch`/`try` bodies at + // least), so their VarRefs count as uses here. + if let Some(head) = cmd.words.first().and_then(Word::as_text) { + if is_body_host(head) { + for word in cmd.words.iter().skip(1) { + if let Some(stmts) = reparse_braced_body(word, source) { + for stmt in &stmts { + let Stmt::Command(inner) = stmt else { + continue; + }; + collect_uses_in_command(inner, source, uses); + } + } + } + } + } +} + +/// If `word` is a braced word, reparse its interior as a script +/// fragment (same recipe as `hover_in_braced_bodies`). Returns +/// `None` for non-braced words (`Bare`, `Quoted`) or when the +/// interior isn't a single text part. +pub(crate) fn reparse_braced_body( + word: &Word, + source: &str, +) -> Option> { + if word.form != WordForm::Braced { + return None; + } + let WordPart::Text { value, span } = word.parts.first()? else { + return None; + }; + // Body-host bodies (`if`/`while`/`foreach`/`for`/`catch`/`try` + // {…}) are Tcl SCRIPTS — newline is a statement separator, not + // whitespace. Reparse in `Mode::Toplevel` so each `set foo` / + // `puts $bar` / etc. lands as its own Command. Using BracketBody + // here would merge every statement into a single mega-command + // whose head is the first statement's head, causing both decl + // collection AND use collection to miss everything past the + // first statement (repros against port.htcl's `foreach` body). + let (mut stmts, mut errs) = crate::parser::parse_fragment( + value.as_str(), + crate::parser::Mode::Toplevel, + ); + let delta = span.start; + for s in &mut stmts { + crate::parser::shift_stmt(s, delta); + } + // Errors from reparse would surface as duplicate parser + // diagnostics; we discard them here since the top-level + // parser has already flagged real issues. The unused-var + // pass is best-effort. + crate::parser::populate_procs(&mut stmts, source, &mut errs); + Some(stmts) +} + +fn collect_uses_in_word(word: &Word, source: &str, uses: &mut HashSet) { + for part in &word.parts { + match part { + WordPart::VarRef { name, .. } => { + // `${foo(bar)}` lands here as a single name + // `"foo(bar)"`. We record the whole string. A + // decl `set foo(bar) …` would match; a decl + // `set foo …` won't. Rare enough to defer. + uses.insert(name.clone()); + // Also record the base name (before the `(`) so + // that `${arr(key)}` counts as a use of a decl + // `set arr …` — the array-vs-scalar distinction + // is Tcl-internal, not a decl-scope question. + if let Some(paren) = name.find('(') { + uses.insert(name[..paren].to_string()); + } + } + WordPart::CmdSubst { body, .. } => { + // Nested command substitution: its interior stmts + // run in the *current* frame (Tcl semantics), so + // their VarRefs count as uses of the outer scope. + for stmt in body { + let Stmt::Command(inner) = stmt else { continue }; + collect_uses_in_command(inner, source, uses); + } + } + WordPart::Text { value, .. } => { + // Braced words (`expr { $kind eq "..." }`, + // `if { $x == 1 } {...}`, condition bodies of + // `while`/`for`, etc.) are stored as opaque Text + // — the parser doesn't split their `$var` + // substrings into VarRefs because inside `{...}` + // Tcl doesn't perform variable substitution at + // parse time. But at runtime, `expr` (and the + // implicit-`expr` bodies of `if`/`while`/`for`) + // DO interpolate variables. Scan the text for + // `$IDENT` patterns and record each as a use so + // a `set kind ""; catch { set kind [...] }; + // expr { $kind == "x" }` shape doesn't warn + // `kind` as unused. + // + // Conservative false-positive rate is fine here: + // the worst case is a literal `$foo` in a text + // that ALSO happens to match a same-named local, + // suppressing a legitimate warning. Missing real + // uses is worse (that's a spurious warning users + // learn to ignore). + if word.form == WordForm::Braced { + scan_brace_var_refs(value, uses); + } + } + WordPart::Escape { .. } => {} + } + } +} + +/// Scan `text` for `$IDENT` and `${IDENT}` occurrences and record +/// each identifier as a use. Handles arrays (`$arr(key)` → both +/// `arr(key)` and `arr`), namespace qualifiers (`$ns::x`), and +/// escaped dollars (`\$foo` → skipped). Doesn't try to be Tcl- +/// precise — the goal is to catch the common variable-reference +/// shapes in expr and control-flow bodies. +fn scan_brace_var_refs(text: &str, uses: &mut HashSet) { + let bytes = text.as_bytes(); + let mut i = 0; + let is_ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b':'; + while i < bytes.len() { + if bytes[i] != b'$' { + i += 1; + continue; + } + // Skip escaped `\$`. + if i > 0 && bytes[i - 1] == b'\\' { + i += 1; + continue; + } + let name_start; + let name_end; + if i + 1 < bytes.len() && bytes[i + 1] == b'{' { + // `${...}` — read until matching `}`. + name_start = i + 2; + let mut j = name_start; + while j < bytes.len() && bytes[j] != b'}' { + j += 1; + } + name_end = j; + i = j + 1; + } else { + name_start = i + 1; + let mut j = name_start; + while j < bytes.len() && is_ident(bytes[j]) { + j += 1; + } + // Include an `(...)` array-index suffix if present. + if j < bytes.len() && bytes[j] == b'(' { + while j < bytes.len() && bytes[j] != b')' { + j += 1; + } + if j < bytes.len() { + j += 1; + } + } + name_end = j; + i = j; + } + if name_end > name_start { + let name = &text[name_start..name_end]; + uses.insert(name.to_string()); + if let Some(paren) = name.find('(') { + uses.insert(name[..paren].to_string()); + } + } + } +} + +/// Emit one warning per decl whose name isn't in `uses` and isn't +/// underscore-prefixed. +fn emit_unused( + decls: &HashMap, + uses: &HashSet, + diags: &mut Vec, +) { + // Sort by span so diagnostic order is stable across runs — + // HashMap iteration order isn't. + let mut items: Vec<(&String, &DeclSite)> = decls.iter().collect(); + items.sort_by_key(|(_, d)| d.span.start); + for (name, decl) in items { + if name.starts_with('_') { + continue; + } + if uses.contains(name) { + continue; + } + let message = match decl.kind { + DeclKind::ProcArg => format!("unused proc arg '{name}'"), + DeclKind::Set => format!("unused local '{name}'"), + DeclKind::ForeachVar => { + format!("unused foreach var '{name}'") + } + DeclKind::Upvar => { + format!("unused upvar binding '{name}'") + } + }; + diags.push(Diagnostic { + severity: Severity::Warning, + message, + span: decl.span, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn diags(src: &str) -> Vec { + let parsed = parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let mut out = Vec::new(); + validate_unused_vars(&parsed.document, src, &mut out); + out + } + + fn warning_messages(d: &[Diagnostic]) -> Vec { + d.iter() + .filter(|dd| dd.severity == Severity::Warning) + .map(|dd| dd.message.clone()) + .collect() + } + + #[test] + fn unused_proc_arg_is_flagged() { + let src = "proc f {x} { return 1 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused proc arg 'x'"]); + } + + #[test] + fn used_proc_arg_is_not_flagged() { + let src = "proc f {x} { return $x }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn underscore_prefix_suppresses_arg_warning() { + let src = "proc f {_ignored} { return 1 }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn unused_local_set_is_flagged() { + let src = "proc f {} { set y 1; return 2 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'y'"]); + } + + #[test] + fn used_local_set_is_not_flagged() { + let src = "proc f {} { set y 1; return $y }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn underscore_prefix_suppresses_local_warning() { + let src = "proc f {} { set _tmp 1; return 2 }\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn use_inside_command_substitution_counts() { + // `$x` at the top level of `[…]` runs in the same frame — + // counts as a use. The braced `{$x + 1}` argument of `expr` + // is opaque in Slice 1; use a form where `$x` sits as a + // direct word so Slice 1's walker sees it. + let src = "proc f {x} { return [list $x] }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn nested_proc_scope_does_not_leak() { + // Outer `y` is unused. Inner proc uses its own `y` — that + // shouldn't count as a use of the outer. + let src = "\ +proc outer {} { + set y 1 + proc inner {y} { return $y } + return 2 +} +"; + let msgs = warning_messages(&diags(src)); + // Both `y`s should be OK now — outer 'y' unused (warned), + // inner 'y' used (not warned). + assert_eq!(msgs, vec!["unused local 'y'"], "{:?}", diags(src)); + } + + #[test] + fn top_level_unused_set_is_flagged() { + let src = "set foo 1\nputs hello\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'foo'"]); + } + + #[test] + fn top_level_used_set_is_not_flagged() { + let src = "set foo 1\nputs $foo\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn top_level_underscore_prefix_suppresses() { + let src = "set _bar 1\nputs hi\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn two_word_set_is_read_not_decl() { + // `set foo` (2 words) *reads* $foo. It's a use, not a decl. + // So the declared `foo` from earlier IS used by the bare + // `set foo` — no warning. + let src = "set foo 1\nset foo\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn subscript_reference_counts_as_use_of_base() { + // `$arr(key)` should count as a use of `arr`. + let src = "proc f {} { set arr 1; return $arr(key) }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn arg_used_in_namespace_eval_body_counts() { + let src = "\ +namespace eval ns { + proc f {x} { return $x } +} +"; + assert!(diags(src).is_empty()); + } + + // ---------- Slice 2 tests ---------- + + #[test] + fn use_inside_if_body_is_reached() { + let src = "proc f {x} { if {1} { return $x } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_inside_while_body_is_reached() { + let src = "proc f {x} { while {0} { puts $x } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_inside_nested_if_else_is_reached() { + let src = "\ +proc f {x y} { + if {1} { + return $x + } else { + return $y + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_iterator_used_in_body_is_not_flagged() { + let src = "proc f {} { foreach z {1 2 3} { puts $z } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_iterator_unused_is_flagged() { + let src = "proc f {} { foreach z {1 2 3} { puts hi } }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused foreach var 'z'"]); + } + + #[test] + fn foreach_multi_var_all_used() { + let src = "\ +proc f {} { + foreach {a b} {1 2 3 4} { + puts $a + puts $b + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn foreach_multi_var_partial_unused() { + let src = "\ +proc f {} { + foreach {a b} {1 2 3 4} { + puts $a + } +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused foreach var 'b'"]); + } + + #[test] + fn foreach_pairs_form_all_used() { + let src = "\ +proc f {} { + foreach a {1 2} b {3 4} { + puts $a + puts $b + } +} +"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg_is_reached() { + // The braced `{$x > 0}` is a body-host (`if`) arg — reparse + // catches the `$x`. + let src = "proc f {x} { if {$x > 0} { puts hi } }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + // ---------- Slice 3 tests ---------- + + #[test] + fn upvar_local_used_is_not_flagged() { + let src = "proc f {} { upvar 1 remote local; return $local }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn upvar_local_unused_is_flagged() { + let src = "proc f {} { upvar 1 remote local; return 1 }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused upvar binding 'local'"]); + } + + #[test] + fn upvar_multi_pair_partial_unused() { + let src = "\ +proc f {} { + upvar 1 remoteA localA remoteB localB + return $localA +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused upvar binding 'localB'"]); + } + + #[test] + fn dynamic_eval_leaks_scope() { + // Proc contains `eval $script` — we can't see the script, + // so the unused local `q` may or may not actually be + // referenced. Conservatively: no warning. + let src = "proc f {} { set q 1; eval $script }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn literal_eval_does_not_leak() { + // `eval { puts hi }` — literal body, no dynamic ref. Unused + // `q` should still warn. + let src = "proc f {} { set q 1; eval { puts hi } }\n"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused local 'q'"], "{:?}", diags(src)); + } + + #[test] + fn dynamic_uplevel_leaks_scope() { + let src = "proc f {} { set q 1; uplevel 1 $script }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn dynamic_apply_leaks_scope() { + let src = "proc f {} { set q 1; apply $lambda 42 }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn dynamic_info_exists_leaks_scope() { + let src = "proc f {} { set q 1; info exists $name }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn nested_proc_scope_not_tainted_by_outer_leak() { + // Outer scope has `eval $script` (leaked). Inner proc + // has an unused arg `x` — that should still warn since the + // inner scope is a fresh frame and doesn't inherit the + // leak. + let src = "\ +proc outer {} { + eval $script + proc inner {x} { return 1 } +} +"; + let msgs = warning_messages(&diags(src)); + assert_eq!(msgs, vec!["unused proc arg 'x'"]); + } + + #[test] + fn use_via_expr_arg_of_expr_command() { + // Bare-form `expr $x + 1` — the `$x` word is a proper + // VarRef part, no brace-scanning needed. + let src = "proc f {x} { return [expr $x + 1] }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg() { + // Braced form `expr { $x + 1 }` — the `$x` inside `{...}` + // is stored as opaque text by the parser (Tcl doesn't + // substitute inside braces at parse time), but at runtime + // `expr` interpolates it. Regression against the + // `lift::vivado_property` shape where a `set kind ""` + // + `catch { set kind [...] }` + `expr { $kind eq ... }` + // was falsely flagged as unused. Scan the braced text + // for `$IDENT` patterns and count them as uses. + let src = "proc f {} {\n\ + set x 1\n\ + return [expr {$x + 1}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_expr_braced_arg_with_string_ops() { + // Real-world shape: guard-init + reassign-in-catch + read + // in expr braces. Matches the lift.htcl `vivado_property` + // proc that was warning `unused local 'kind'`. + let src = "proc f {} {\n\ + set kind \"\"\n\ + catch { set kind hello }\n\ + return [expr {$kind eq \"string\" || $kind eq \"bool\"}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } + + #[test] + fn use_via_braced_var_with_namespace_qualifier() { + let src = "proc f {} {\n\ + set ns::x 1\n\ + return [expr {$ns::x + 1}]\n\ + }\n"; + assert!(diags(src).is_empty(), "{:?}", diags(src)); + } +} diff --git a/vw-htcl/src/validate.rs b/vw-htcl/src/validate.rs new file mode 100644 index 0000000..6c4f775 --- /dev/null +++ b/vw-htcl/src/validate.rs @@ -0,0 +1,5630 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Signature-aware call-site validation. +//! +//! Builds a {proc_name → ProcSignature} table from the top-level +//! procs in a document, then walks every call site in the same +//! document and checks the keyword arguments against the declared +//! signature. Diagnostics are language-neutral; downstream (the LSP, +//! `vw check`) maps them to the appropriate display form. + +use std::collections::HashMap; + +use crate::ast::{ + Attribute, AttributeValue, Command, CommandKind, Document, EnumDecl, + OverloadInfo, OverloadVariant, Proc, ProcArg, ProcSignature, Stmt, + TypeDecl, TypeExpr, Word, WordPart, +}; +use crate::span::Span; + +/// Side-table produced alongside the signature table by +/// [`build_signature_table_with_overloads`]. Maps each public proc +/// name that resolves to an enum-overload set to its [`OverloadInfo`]. +/// Names not in this map are regular (non-overloaded) procs. +pub type OverloadTable = HashMap; + +/// Mangle a specialization's internal name. The `__` prefix is +/// reserved (the validator rejects user procs whose names start +/// with `__`) so mangled names don't collide with anything +/// user-written. +/// +/// For namespaced public names (`Property::as_nested`), the +/// prefix goes on the LEAF, not the whole name — otherwise the +/// mangled form (`__Property::as_nested__Nested`) puts the proc +/// in a fictional `__Property` namespace Tcl hasn't created, and +/// `proc` errors with "unknown namespace." Keeping the leaf-only +/// prefix (`Property::__as_nested__Nested`) places the +/// specialization inside the SAME namespace as its public +/// dispatcher, which the enum prelude or user `namespace eval` +/// already declared. +pub fn mangle_specialization(public_name: &str, variant: &str) -> String { + match public_name.rsplit_once("::") { + Some((ns, leaf)) => format!("{ns}::__{leaf}__{variant}"), + None => format!("__{public_name}__{variant}"), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Severity { + Error, + Warning, +} + +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub severity: Severity, + pub message: String, + pub span: Span, +} + +pub fn validate(document: &Document, source: &str) -> Vec { + validate_with_signatures(document, source, &HashMap::new()) +} + +/// Same as [`validate`], but resolves unknown calls against an +/// additional pool of signatures supplied by the caller — used by +/// the REPL to make procs declared in earlier session batches +/// visible to a new batch without re-parsing the whole prelude. +/// +/// Merge rules: +/// +/// - The document's own signatures shadow `extra`. Redefining a +/// proc in `document` overrides the prior version (Tcl +/// semantics — a second `proc` redefines). +/// - Duplicate-definition diagnostics only fire for collisions +/// **within** `document`. A new batch that re-`src`s a wrapper +/// already loaded earlier shouldn't warn on every input. +pub fn validate_with_signatures<'doc>( + document: &'doc Document, + source: &str, + extra: &HashMap, +) -> Vec { + validate_with_extras(document, source, extra, &HashMap::new()) +} + +/// Full validation entry point: same as [`validate_with_signatures`] +/// but also takes a pool of newtype declarations from prior session +/// batches. Lets the REPL drop a `proc bd_cell::repr` in batch N +/// without re-tripping "type bd_cell missing repr" diagnostics for +/// the `type bd_cell = string` declaration in batch N-1. +pub fn validate_with_extras<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, +) -> Vec { + validate_with_all_extras( + document, + source, + extra_sigs, + extra_types, + &HashMap::new(), + ) +} + +/// Full validation entry point. Accepts a prior-batch pool of +/// signatures, type declarations, AND enum declarations, so the +/// REPL can split an `enum E = …` decl across batches from the +/// procs that dispatch on it. +pub fn validate_with_all_extras<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, + extra_enums: &HashMap, +) -> Vec { + validate_with_all_extras_and_vars( + document, + source, + extra_sigs, + extra_types, + extra_enums, + &std::collections::HashSet::new(), + &std::collections::HashSet::new(), + ) +} + +/// Same as [`validate_with_all_extras`], plus: +/// +/// - `extra_top_level_vars` — top-level variable names known to +/// be defined in prior batches. The undef-variable pass merges +/// these into its top-level decl set, so a `set p …` in REPL +/// batch N-1 makes `$p` in batch N legal. Proc-body scopes +/// ignore the pool (Tcl locals don't inherit top-level scope), +/// so this only affects the document's own top-level statements. +/// - `extra_dep_names` — workspace-dependency names the caller +/// registered with its `vw_htcl::Resolver`. Each `src @` +/// statement in the document is checked against this pool; a +/// name that's not in the set fires an `Error` diagnostic +/// spanned to the `@` text. Empty set → the check +/// no-ops (unit tests and non-workspace-aware callers). +pub fn validate_with_all_extras_and_vars<'doc>( + document: &'doc Document, + source: &str, + extra_sigs: &HashMap, + extra_types: &HashMap, + extra_enums: &HashMap, + extra_top_level_vars: &std::collections::HashSet, + extra_dep_names: &std::collections::HashSet, +) -> Vec { + let mut diags = Vec::new(); + // Type-table FIRST — its keys feed the overloaded-proc-arm + // detection in `build_signature_table_with_overloads` so a proc + // whose first arg is a newtype-Qualified name (e.g. + // `-config: versal_cips::PsPmcConfig` inside `namespace eval + // versal_cips`) isn't misclassified as an enum-overload arm. + // Duplicate-decl diagnostics from this pass are held aside and + // re-emitted after signature collection so ordering matches the + // pre-refactor rendering. + let mut type_prescan_diags = Vec::new(); + let mut type_table = + build_type_decl_table(document, &mut type_prescan_diags); + for (name, td) in extra_types { + type_table.entry(name.clone()).or_insert(*td); + } + let newtype_qualified_names: std::collections::HashSet = + type_table.keys().cloned().collect(); + let (mut table, _overloads) = build_signature_table_with_overloads( + document, + &newtype_qualified_names, + &mut diags, + ); + for (name, sig) in extra_sigs { + table.entry(name.clone()).or_insert(*sig); + } + diags.extend(type_prescan_diags); + let mut enum_table = build_enum_decl_table(document, &mut diags); + for (name, ed) in extra_enums { + enum_table.entry(name.clone()).or_insert(*ed); + } + validate_type_decl_triplets(&type_table, &table, &mut diags); + validate_enum_decls(&enum_table, &type_table, &mut diags); + let newtype_names: std::collections::HashSet = + type_table.keys().cloned().collect(); + validate_qualified_positions(document, &newtype_names, &mut diags); + let mut var_table = VarTypeTable::new(); + let proc_table = build_proc_table(document); + // Precompute per-signature arg-name → arg indexes so + // `validate_command` can do O(1) flag lookups instead of the + // O(N_args) linear scan `ProcSignature::find` does. Dcmac's + // auto-generated `ps_pmc_config` (and friends) carry ~890 + // args; a bare `sig.find("boot_mode")` scans all of them, and + // `validate_command` invokes `find` multiple times per keyword + // argument (parse loop + requires + conflicts). Times thousands + // of call sites this compounds into minutes of CPU. Building the + // index once per sig — even for sigs never touched — is cheap + // (linear in total-args-in-workspace) and turns per-call-site + // work from O(K × N_args) into O(K). + let sig_env = SigEnv::build(&table); + validate_stmts( + &document.stmts, + source, + &table, + &sig_env, + &proc_table, + &newtype_qualified_names, + &mut var_table, + &mut diags, + ); + crate::undefined::validate_undefined_vars_with_extras( + document, + source, + extra_top_level_vars, + &mut diags, + ); + crate::unused::validate_unused_vars(document, source, &mut diags); + validate_src_imports(document, extra_dep_names, &mut diags); + validate_test_attributes(document, &mut diags); + diags +} + +/// `@test` semantic checks. Fires warnings (not errors) so +/// misused tags surface in the LSP + `vw check` without blocking +/// execution. Rules: +/// +/// - `@test(X)` where X isn't the literal ident `dedicated-eda` +/// (the only recognized value today). +/// - `@test` on a proc with a non-empty parameter list — tests +/// are zero-arg for the MVP runner. +/// - `@test` on a nested proc (declared inside another proc's +/// body) — only top-level `@test` procs are discoverable by +/// `vw test`. +/// +/// Doesn't check for `@test` on non-proc statements — that's +/// caught at parse time with a spanned error. +fn validate_test_attributes(document: &Document, diags: &mut Vec) { + walk_procs_for_test_check( + &document.stmts, + /*inside_proc=*/ false, + diags, + ); +} + +fn walk_procs_for_test_check( + stmts: &[Stmt], + inside_proc: bool, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(attr) = proc.attribute("test") { + if inside_proc { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test` on a nested proc — only \ + top-level `@test`-annotated procs \ + are discoverable by `vw test`" + .into(), + span: attr.span, + }); + } + let mut has_dedicated = false; + let mut target_key_present = false; + let mut variant_key_present = false; + for value in &attr.values { + match value { + crate::ast::AttributeValue::Ident { + value: v, + .. + } if v == "dedicated-eda" => { + has_dedicated = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } if key == "target" => { + target_key_present = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } if key == "variant" => { + variant_key_present = true; + } + crate::ast::AttributeValue::Keyed { + key, .. + } => { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "`@test(...)` — unrecognized key \ + `{key}` (recognized: `target=`, \ + `variant=`)" + ), + span: attr.span, + }); + } + _ => { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "`@test(…)` value must be the \ + `dedicated-eda` marker, \ + `target=`, or `variant=`; \ + got `{}`", + render_attribute_value(value), + ), + span: attr.span, + }); + } + } + } + if target_key_present && !has_dedicated { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(target=…)` requires the \ + `dedicated-eda` marker — shared-bucket \ + tests cannot override the \ + auto-project's `-part`" + .into(), + span: attr.span, + }); + } + if variant_key_present && !has_dedicated { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(variant=…)` requires the \ + `dedicated-eda` marker — shared-bucket \ + tests cannot switch design surfaces" + .into(), + span: attr.span, + }); + } + if target_key_present && variant_key_present { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test(...)` — pick one of \ + `target=` or `variant=`, \ + not both (variants own their parts)" + .into(), + span: attr.span, + }); + } + if let Some(sig) = &proc.signature { + if !sig.args.is_empty() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: "`@test` procs must take zero \ + arguments — parameterized tests \ + aren't supported yet" + .into(), + span: attr.span, + }); + } + } + } + walk_procs_for_test_check(&proc.body, true, diags); + } + CommandKind::NamespaceEval(ns) => { + walk_procs_for_test_check(&ns.body, inside_proc, diags); + } + _ => {} + } + } +} + +fn render_attribute_value(v: &crate::ast::AttributeValue) -> String { + v.to_tcl_literal() +} + +/// Walk every top-level `src @` statement in `document` and +/// emit an Error diagnostic for any `` not present in +/// `known_deps`. Relative and absolute path imports (non-`@` +/// forms) are skipped — those get their existence checked +/// downstream by the loader's filesystem probe. +fn validate_src_imports( + document: &Document, + known_deps: &std::collections::HashSet, + diags: &mut Vec, +) { + // The empty-set case covers unit tests (no workspace) and + // downstream callers that don't hook up a Resolver. Short- + // circuit rather than walking every statement for nothing. + if known_deps.is_empty() { + return; + } + for stmt in &document.stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let CommandKind::Src(src) = &cmd.kind else { + continue; + }; + // Missing path (contains `$var` / `[cmd]` substitution) — + // handled by other passes; don't double-flag here. + let Some(path) = src.path.as_deref() else { + continue; + }; + let classified = crate::src_path::classify(path); + let crate::src_path::PathKind::Named { name, subpath } = + classified.kind + else { + continue; + }; + if known_deps.contains(&name) { + continue; + } + // Message mirrors `ResolveError::UnknownDependency`'s text + // in `src_path.rs` — same hint keeps the CLI hard-abort + // path (which still fires) and the analyzer diagnostic + // pointing at the same fix. + let subpath_hint = if subpath.is_empty() { + String::new() + } else { + format!("/{subpath}") + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "unknown src module `{name}` in `src @{name}{subpath_hint}`; \ + add a `[dependencies.{name}]` entry to your workspace's \ + vw.toml or run `vw add` to fetch it" + ), + span: src.path_span, + }); + } +} + +/// Validate every command in `stmts`, descending into proc bodies so +/// that calls nested inside a proc are checked just like top-level +/// ones. The signature table is document-wide, so a call resolves to +/// its (top-level) proc at any depth. +/// Variable-type table keyed by name. Populated as `validate_stmts` +/// walks `set VAR ` and proc-parameter bindings; consulted by +/// `value_type` when it hits a `$var` reference at a call site. +/// +/// Nominal / strict: entries hold the DECLARED `TypeExpr` (from a +/// proc's `return_type` or a `ProcArg.type_annotation`) without any +/// alias-walking. Comparing two entries via [`types_match`] gives +/// newtype identity — `Quad0Ch1Props` and `Quad1Ch0Props` are +/// distinct even though both alias to `Properties`. +/// +/// Scope discipline: each proc body owns its own table (created in +/// the `CommandKind::Proc` arm of `validate_stmts`). Nested +/// `namespace eval` blocks share the enclosing table (matches Tcl +/// semantics — `namespace eval` creates a namespace but doesn't +/// open a new local-variable scope). Bodies of `[ … ]` command +/// substitutions also share the enclosing table so `set` inside +/// brackets is visible outside. +pub(crate) type VarTypeTable = HashMap; + +/// Infer the type of a value word — the argument on the right of a +/// `-flag` at a call site, or the RHS of a `set VAR`. +/// +/// Covers the two forms the call-site type-check actually needs: +/// +/// - A whole-word command substitution `[proc-call …]` returns the +/// called proc's `return_type`. Multi-command bodies (`[a; b]`) +/// take the LAST command's return type — matches Tcl's "value of +/// the last command wins" for `[…]` substitution. +/// - A whole-word variable reference `$foo` / `${foo}` returns the +/// type recorded in `var_table` (from a prior `set` or a proc +/// parameter with a `type_annotation`). +/// +/// Anything else — literals, quoted strings, mixed compounds like +/// `prefix-$var` — returns `None`. Callers treat `None` as "unknown +/// type, skip the check" (gradual typing). We don't error on what +/// we can't infer. +pub(crate) fn value_type( + word: &crate::ast::Word, + sig_table: &HashMap, + var_table: &VarTypeTable, +) -> Option { + value_type_with_procs(word, sig_table, var_table, None) +} + +/// Companion to [`value_type`] that also has access to a proc +/// table for return-type inference on unannotated procs. When a +/// `[proc-call]` word hits a signature whose `return_type` is +/// `None`, the caller can supply the corresponding [`Proc`] node +/// via `proc_table` and this function walks the body's last +/// `return` statement to infer the type — handles the common +/// pattern where a user proc doesn't declare a return type but +/// its body ends with `return $x` or `return [typed_proc]`. +pub(crate) fn value_type_with_procs( + word: &crate::ast::Word, + sig_table: &HashMap, + var_table: &VarTypeTable, + proc_table: Option<&HashMap>, +) -> Option { + use crate::ast::{Stmt, TypeExpr, WordPart}; + match word.parts.as_slice() { + [WordPart::VarRef { name, .. }] => var_table.get(name).cloned(), + [WordPart::CmdSubst { body, .. }] => { + let last_cmd = body.iter().rev().find_map(|s| match s { + Stmt::Command(c) => Some(c), + _ => None, + })?; + let call_name = last_cmd.words.first()?.as_text()?; + let sig = sig_table.get(call_name)?; + // Annotated return type wins. + if let Some(ty) = &sig.return_type { + return Some(ty.clone()); + } + // Fallback: walk the proc body to infer. Only fires + // when a proc_table is supplied — top-level callers + // (per-batch var-type builders, putr rewrite) pass one + // through; internal callers that just want fast + // annotation-based lookup pass `None`. + let procs = proc_table?; + let proc = procs.get(call_name)?; + infer_return_type_from_body(proc, sig_table, procs) + } + // Bare `true` / `false` literals — the ONLY textual values + // whose type we infer, and only because they're the + // canonical HTCL bool literals. Everything else (bare + // words, quoted strings, mixed compounds) stays untyped + // (gradual typing). Position matters: this makes the + // check symmetric so `set flag true` binds `flag: bool` + // and a subsequent `-slot $flag` at a `bool` arg matches. + [WordPart::Text { value, .. }] + if value == "true" || value == "false" => + { + Some(TypeExpr::Named { + name: "bool".into(), + span: word.span, + }) + } + _ => None, + } +} + +/// Validate that every `return X` statement in `proc`'s body +/// produces a value whose type matches the proc's declared +/// return type. Only fires when the proc has a `return_type` +/// annotation (untyped procs skip the check entirely). +/// +/// Descends into the braced bodies of `if`/`elseif`/`else`/ +/// `while`/`for`/`foreach`/`catch` — a `return` buried inside +/// an early-exit branch still gets checked. Nested control +/// blocks recurse through the same walker so an arbitrary +/// depth of `if { if { return X } }` still catches wrong-typed +/// returns. +/// +/// Bare `return` (no argument) in an annotated proc is a hard +/// error — the annotation is a promise to produce a value. +pub(crate) fn validate_proc_returns( + proc: &crate::ast::Proc, + source: &str, + sig_table: &HashMap, + proc_table: &HashMap, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + // Newtype-triplet exemption: `T::from`, `T::to`, `T::repr`, + // `T::empty` are compiler-emitted (or generator-emitted) + // identity conversions between a newtype and its underlying. + // Under strict nominal identity, `return $v` where `$v: string` + // in a proc returning `T` would flag as a mismatch — but + // that's the WHOLE POINT of the from/to/repr layer: cross + // the newtype boundary via `return $v` (Tcl-level identity). + // Skip the check when the proc name is `T::` for a + // declared newtype `T` and a triplet suffix. Applies to both + // annotated and unannotated forms — hand-written triplets + // often omit the annotation and rely on the identity-shape. + if let Some(name) = proc.name.as_deref() { + if is_newtype_triplet_name(name, newtype_names) { + return; + } + } + let Some(declared) = &proc.return_type else { + // No return-type annotation → the proc is implicitly a + // side-effect-only op. `return X` in a side-effect proc is + // a structural mismatch: the caller has nothing to receive + // it, and the missing annotation tells readers the same. Flag + // every `return X` with a value. Bare `return` is fine. + walk_returns_without_annotation(&proc.body, source, diags); + return; + }; + // Enum-overload-arm exemption: `proc f {v: E::A} string { … }` + // is the specialization shape for the overload dispatcher — + // `v` is the enum variant's payload, which at Tcl runtime is + // just the underlying type's raw value (a `string` for + // `E::A: string`). Returning it as its underlying is + // structurally identity, same rationale as the newtype + // triplet. Detect by first-arg type being `Qualified` — + // that's the only shape overload arms use. + if let Some(sig) = &proc.signature { + if let Some(first) = sig.args.first() { + if matches!( + first.type_annotation, + Some(crate::ast::TypeExpr::Qualified { .. }) + ) { + return; + } + } + } + // Seed a var table with typed parameters. Walker updates it + // as it visits `set` bindings so downstream `return $VAR` + // resolves via the same scope-aware inference the outer + // arg-type check uses. + let mut local_vars = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + local_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + walk_returns( + &proc.body, + source, + sig_table, + proc_table, + &mut local_vars, + declared, + diags, + ); + + // Must-return: an annotated proc that isn't `unit` must + // reach a `return` on every path (or end with a + // last-expression whose type matches the annotation, per + // Tcl's implicit-return rule). Runs AFTER walk_returns so + // the per-return type errors surface first if both fire. + let is_unit = matches!( + declared, + crate::ast::TypeExpr::Named { name, .. } if name == "unit" + ); + if !is_unit { + // walk_returns mutated local_vars — snapshot a fresh + // seed for the must-return pass so we start from the + // proc's typed parameters, matching the walker's own + // starting state. + let mut fresh_vars = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + fresh_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + if !paths_always_return( + &proc.body, + source, + declared, + sig_table, + proc_table, + &mut fresh_vars, + ) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc annotated `{}` may fall through without \ + returning a value of the right type; every code \ + path must end with `return $X` (or a final \ + expression whose type matches)", + render_type_inline(declared), + ), + span: proc.name_span, + }); + } + } +} + +/// True when `proc_name` matches the newtype-triplet pattern +/// `::` where `T` is a declared newtype and `suffix` +/// is one of `from`, `to`, `repr`, `empty`. These procs are +/// identity conversions across the newtype boundary, so their +/// `return $v` bodies would trip the strict-nominal check by +/// design; the check exempts them. +fn is_newtype_triplet_name( + proc_name: &str, + newtype_names: &std::collections::HashSet, +) -> bool { + for suffix in ["from", "to", "repr", "empty"] { + let marker = format!("::{suffix}"); + if let Some(prefix) = proc_name.strip_suffix(&marker) { + if newtype_names.contains(prefix) { + return true; + } + } + } + false +} + +/// Recursive walker used by [`validate_proc_returns`]. Tracks +/// `set` bindings, records `return X` statements it finds, and +/// descends into parsed control-flow bodies. +fn walk_returns( + stmts: &[crate::ast::Stmt], + source: &str, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &mut VarTypeTable, + declared: &crate::ast::TypeExpr, + diags: &mut Vec, +) { + use crate::ast::{Stmt, WordForm}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Track `set VAR ` bindings the same way the + // rewrite walker does — later `return $VAR` needs to see + // the type. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + var_table, + Some(proc_table), + ) { + var_table.insert(name.to_string(), ty); + } + } + } + } + // `return X` — the actual check. + let head_text = cmd.words.first().and_then(|w| w.as_text()); + if head_text == Some("return") { + check_return( + cmd, sig_table, proc_table, var_table, declared, diags, + ); + } + // Descend into control-flow braced bodies. Heuristic: + // for known control-flow heads, parse every WordForm:: + // Braced word as a candidate body. Condition-shaped + // braces (like `if {$x == 1}`) parse without errors but + // don't contain `return` calls, so they contribute + // nothing — a benign no-op. `for INIT COND NEXT BODY`'s + // INIT and NEXT can hold `set` calls that would affect + // var_table if we tracked them; we don't, since the + // walker isn't a live-execution simulator and Tcl's + // control-flow scope semantics are already muddy. + if matches!( + head_text, + Some( + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + ) + ) { + for word in cmd.words.iter().skip(1) { + if word.form != WordForm::Braced { + continue; + } + // The word's span covers `{...}` including the + // outer braces. Strip 1 byte from each end for + // the interior text; parse as a fragment; shift + // spans by the interior start. + let word_start = word.span.start as usize; + let word_end = word.span.end as usize; + if word_end <= word_start + 2 { + // `{}` — empty body, nothing to check. + continue; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = crate::parser::parse_fragment( + body_text, + crate::parser::Mode::Toplevel, + ); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + // Populate procs INSIDE the parsed body so nested + // structures behave — mostly irrelevant for + // returns but keeps recursion consistent. + crate::parser::populate_procs( + &mut body_stmts, + source, + &mut Vec::new(), + ); + walk_returns( + &body_stmts, + source, + sig_table, + proc_table, + var_table, + declared, + diags, + ); + } + } + } +} + +/// Walk `stmts` looking for `return X` statements with a value in a +/// proc that has NO declared return type. Every such `return X` is +/// an error — the proc's shape declares "side effects only," and a +/// value-carrying return contradicts that. +/// +/// Descends into control-flow braced bodies the same way +/// [`walk_returns`] does — a value-return buried in an `if`/`else` +/// branch is caught. Bare `return` is fine (side-effect procs +/// naturally use it for early exits). +fn walk_returns_without_annotation( + stmts: &[crate::ast::Stmt], + source: &str, + diags: &mut Vec, +) { + use crate::ast::{Stmt, WordForm}; + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + let head_text = cmd.words.first().and_then(|w| w.as_text()); + if head_text == Some("return") && cmd.words.len() >= 2 { + diags.push(Diagnostic { + severity: Severity::Error, + message: "`return` with a value in a proc that has no \ + declared return type — add a return-type \ + annotation (`proc NAME { args } TYPE { ... }`) \ + or drop the returned value" + .to_string(), + span: cmd.span, + }); + } + if matches!( + head_text, + Some( + "if" | "elseif" + | "else" + | "while" + | "for" + | "foreach" + | "catch" + ) + ) { + for word in cmd.words.iter().skip(1) { + if word.form != WordForm::Braced { + continue; + } + let word_start = word.span.start as usize; + let word_end = word.span.end as usize; + if word_end <= word_start + 2 { + continue; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = crate::parser::parse_fragment( + body_text, + crate::parser::Mode::Toplevel, + ); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + crate::parser::populate_procs( + &mut body_stmts, + source, + &mut Vec::new(), + ); + walk_returns_without_annotation(&body_stmts, source, diags); + } + } + } +} + +/// Check a single `return X` (or bare `return`) against the +/// declared return type. Emits diagnostics directly. +fn check_return( + cmd: &crate::ast::Command, + sig_table: &HashMap, + proc_table: &HashMap, + var_table: &VarTypeTable, + declared: &crate::ast::TypeExpr, + diags: &mut Vec, +) { + use crate::ast::{TypeExpr, WordForm}; + // `return` alone — bare — violates the annotation's promise + // to produce a value UNLESS the declared type is `unit`, + // which means "no meaningful value" and matches bare-return + // semantics. Common in side-effecting procs that early-out. + let Some(arg) = cmd.words.get(1) else { + let is_unit = matches!( + declared, + TypeExpr::Named { name, .. } if name == "unit" + ); + if !is_unit { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "bare `return` in proc annotated `{}` — the return \ + type requires a value; use `return $X`", + render_type_inline(declared), + ), + span: cmd.span, + }); + } + return; + }; + // `return NAME` — bareword form — where NAME matches a variable + // in scope is almost always a missing `$` (the user meant + // `return $NAME`). Without the sigil, Tcl returns the literal + // string "NAME", which passes runtime typing but silently + // subverts the annotation. `true`/`false` are the canonical bool + // literals and integer literals are legitimate bare returns for + // `int`-annotated procs, so exempt those. + if arg.form == WordForm::Bare { + if let Some(lit) = arg.as_text() { + let looks_like_literal = + lit == "true" || lit == "false" || lit.parse::().is_ok(); + if !looks_like_literal && var_table.get(lit).is_some() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "return of bare identifier `{lit}` — a variable \ + named `{lit}` is in scope but not dereferenced; \ + did you mean `return ${lit}`? (without `$` this \ + returns the literal string \"{lit}\")", + ), + span: cmd.span, + }); + return; + } + } + } + // Try to infer the returned expression's type. If we can't, + // fall through to a bare-literal check below — bare text at a + // structural type (Qualified newtype) can't be right regardless + // of whether the value_type pass could deduce it. + let inferred = + value_type_with_procs(arg, sig_table, var_table, Some(proc_table)); + if let Some(actual) = inferred { + if !types_match(declared, &actual) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "return type mismatch: proc declared `{}`, but this \ + `return` produces `{}`", + render_type_inline(declared), + render_type_inline(&actual), + ), + span: cmd.span, + }); + } + return; + } + // Uninferrable return + declared newtype (`foo::Bar`): a + // qualified type never comes from a bare text literal — the + // only paths to a valid value are `foo::Bar::from …`, a proc + // whose return_type is `foo::Bar`, or a `$var` bound to one. + // A bare text word at this position is a strong signal of a + // missing sigil or a wrong-shape return. + if matches!(declared, TypeExpr::Qualified { .. }) + && arg.form == WordForm::Bare + { + if let Some(lit) = arg.as_text() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "return type mismatch: proc declared `{}`, but this \ + `return` produces the literal string \"{lit}\" — a \ + `{}` value can't be a bare word", + render_type_inline(declared), + render_type_inline(declared), + ), + span: cmd.span, + }); + } + } +} + +/// Whether every code path through `stmts` reaches an explicit +/// `return`, OR ends with a final statement whose result type +/// matches `declared` (Tcl's implicit-last-expression return). +/// +/// Empty `stmts` → false. This is the primary failure case for +/// the must-return check: a proc annotated with a non-`unit` +/// return type whose body is completely empty (or ends with a +/// side-effecting `puts`) has no path that produces a value. +/// +/// Descends into control-flow braced bodies via +/// `parser::parse_fragment` + `shift_stmt` — the same reparse +/// dance `walk_returns` already uses (see the identical pattern +/// at line ~570). +/// +/// Coverage of control commands: +/// +/// - `return` (any form) → this path terminates. +/// - `set VAR X` → records VAR's inferred type in `local_vars` +/// so a downstream implicit-last-expression `$VAR` gets typed. +/// - `if COND BODY [elseif COND BODY]* [else BODY]` → terminates +/// iff there IS an `else` AND every branch body terminates. +/// - `while` / `for` / `foreach` → conservative false (body may +/// not execute at runtime). +/// - `switch X { pat body pat body … [default body] }` → +/// terminates iff a `default` arm exists AND every arm's body +/// terminates. Fallthrough arms (`pat -`) inherit the next +/// arm's body. +/// - `catch` → conservative false (body may error out). +/// - Anything else → non-terminating individually; keep scanning. +fn paths_always_return( + stmts: &[Stmt], + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &mut VarTypeTable, +) -> bool { + if stmts.is_empty() { + return false; + } + let mut last_cmd_index: Option = None; + for (i, stmt) in stmts.iter().enumerate() { + let Stmt::Command(cmd) = stmt else { continue }; + let head_text = cmd.words.first().and_then(|w| w.as_text()); + + // Track `set VAR ` bindings so a trailing `$VAR` + // (implicit last-expression) sees the right type. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + local_vars, + Some(proc_table), + ) { + local_vars.insert(name.to_string(), ty); + } + } + } + } + last_cmd_index = Some(i); + + // Explicit `return` — the path terminates here. + if head_text == Some("return") { + return true; + } + // `error` — unwinds the stack, so control never falls + // through. Counts as terminating for the must-return + // analysis (the proc can't reach its end after this). + if head_text == Some("error") { + return true; + } + // `if` — check if all branches terminate AND an `else` exists. + if head_text == Some("if") + && if_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `switch` — check for `default` arm and all-arm termination. + if head_text == Some("switch") + && switch_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `try { body } [on ... handler]* [finally script]` — + // terminates iff the body AND every handler + // path-terminate. This is what the generator's + // wrap-body pattern emits (`try { return X } on error + // { error "prefix.$msg" }`). + if head_text == Some("try") + && try_command_terminates( + cmd, source, declared, sig_table, proc_table, local_vars, + ) + { + return true; + } + // `while` / `for` / `foreach` / `catch` — never + // guaranteed to run their body, so they can't be sole + // terminators. Keep scanning. + } + + // Implicit-last-expression rule: if we've made it here + // without hitting a `return`, look at the last command's + // last word. If that word's type matches `declared`, this + // path counts as terminating. + let Some(last_idx) = last_cmd_index else { + return false; + }; + let Stmt::Command(last_cmd) = &stmts[last_idx] else { + return false; + }; + let expr_word = if last_cmd.words.len() == 1 { + last_cmd.words.first() + } else { + // A multi-word command (e.g. `set _ [...]; $_`) parses + // as a single `$_` command with one word — but a + // multi-word command like `puts $x` has 2 words. Use + // the FIRST word only when there's a single word (a + // pure `$var` or `[proc-call]` expression); otherwise + // treat the last statement as an expression via its + // *head* word only when that head IS a proc-call — + // handled below. + last_cmd.words.first() + }; + let Some(expr_word) = expr_word else { + return false; + }; + // For a bare command like `some_typed_proc arg1 arg2`, we + // want to check the CALL's return type. `value_type_with_procs` + // won't help directly on the head word (it's a bare-text + // word, not a CmdSubst). But we CAN look up the head in the + // sig_table. If the head is a known proc AND its return + // type matches `declared`, treat as implicit return. + if let Some(head) = last_cmd.words.first().and_then(|w| w.as_text()) { + // `extern::name` is the caller's opt-out: "this is a raw + // Tcl proc; I'm not declaring its type, trust me." Same + // policy as `validate_command`'s check at line 1946. + // Trailing extern call = trust for the must-return check. + if crate::lower::is_extern_call(head) { + return true; + } + if let Some(sig) = sig_table.get(head) { + if let Some(ret_ty) = &sig.return_type { + if types_match(declared, ret_ty) { + return true; + } + } + } + } + // Otherwise: try value_type_with_procs on the expression + // word itself (handles `$var` and `[proc-call]` shapes). + if let Some(ty) = value_type_with_procs( + expr_word, + sig_table, + local_vars, + Some(proc_table), + ) { + if types_match(declared, &ty) { + return true; + } + } + false +} + +/// Does a single `if COND BODY [elseif COND BODY]* [else BODY]` +/// command terminate? Yes iff every branch body terminates AND +/// there IS an `else` (a chain with no `else` may fall through). +/// +/// Word layout, positions counted from 0: +/// - [0] = "if" +/// - [1] = condition +/// - [2] = body +/// - [3] = "elseif" | "else" (or end) +/// - [4] = condition (after elseif) | body (after else) +/// - … +fn if_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + let mut i = 1; + let mut has_else = false; + let mut branch_bodies: Vec<&crate::ast::Word> = Vec::new(); + while i < cmd.words.len() { + // Skip the condition word. + if i + 1 >= cmd.words.len() { + return false; + } + branch_bodies.push(&cmd.words[i + 1]); + i += 2; + if i >= cmd.words.len() { + break; + } + match cmd.words[i].as_text() { + Some("elseif") => { + i += 1; + // Loop continues with i at condition position. + } + Some("else") => { + has_else = true; + i += 1; + if i >= cmd.words.len() { + return false; + } + branch_bodies.push(&cmd.words[i]); + break; + } + _ => { + // Something unexpected — conservative: don't + // treat as terminating. + return false; + } + } + } + if !has_else { + return false; + } + for body_word in branch_bodies { + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + } + true +} + +/// Does a `switch X { pat body … [default body] }` terminate? +/// Yes iff there's a `default` arm AND every arm's body +/// terminates. Fallthrough arms (`pat -` where the body word is +/// literally `-`) inherit the next arm's body. +fn switch_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + // Find the switch body — the LAST braced word in the command + // (the argument list before it is `[options] value`, which + // we don't parse). + let body_word = cmd + .words + .iter() + .rev() + .find(|w| w.form == crate::ast::WordForm::Braced); + let Some(body_word) = body_word else { + return false; + }; + let word_start = body_word.span.start as usize; + let word_end = body_word.span.end as usize; + if word_end <= word_start + 2 { + return false; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut arm_stmts, _errs) = + crate::parser::parse_fragment(body_text, crate::parser::Mode::Toplevel); + for s in &mut arm_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + // Each arm parses as one command with words = [pat, body]. + // Collect them as pairs, resolving fallthrough (`pat -`) + // arms to the next arm's body. + let mut pairs: Vec<(String, &crate::ast::Word)> = Vec::new(); + let mut pending_pats: Vec = Vec::new(); + for stmt in &arm_stmts { + let Stmt::Command(arm_cmd) = stmt else { + continue; + }; + if arm_cmd.words.len() < 2 { + return false; + } + let pat = match arm_cmd.words[0].as_text() { + Some(p) => p.to_string(), + None => return false, + }; + let body_arg = &arm_cmd.words[1]; + if body_arg.as_text() == Some("-") { + // Fallthrough: this pattern inherits the next + // resolved arm's body. + pending_pats.push(pat); + continue; + } + // Resolve pending fallthroughs to this arm's body too. + for pending in pending_pats.drain(..) { + pairs.push((pending, body_arg)); + } + pairs.push((pat, body_arg)); + } + if !pending_pats.is_empty() { + // Trailing `pat -` without a resolving arm — malformed. + return false; + } + // Must have a `default` arm. + let has_default = pairs.iter().any(|(p, _)| p == "default"); + if !has_default { + return false; + } + for (_pat, body_word) in &pairs { + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + } + true +} + +/// Does a `try BODY [on CODE VAR HANDLER]* [trap PATS VAR HANDLER]* +/// [finally SCRIPT]` terminate? Yes iff BODY terminates AND every +/// handler body terminates. `finally` doesn't participate in +/// termination (it runs REGARDLESS of what the body/handlers did, +/// so it can't turn a non-terminating structure into a terminating +/// one — but it also doesn't invalidate one). +fn try_command_terminates( + cmd: &crate::ast::Command, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + // Body is the first argument (word[1]). + let Some(body_word) = cmd.words.get(1) else { + return false; + }; + if !branch_body_terminates( + body_word, source, declared, sig_table, proc_table, local_vars, + ) { + return false; + } + // Walk the remaining words looking for handler bodies. Each + // `on CODE VAR HANDLER` or `trap PATS VAR HANDLER` clause + // occupies 4 words; each `finally SCRIPT` occupies 2. + let mut i = 2; + while i < cmd.words.len() { + let head = cmd.words[i].as_text(); + match head { + Some("on") | Some("trap") => { + // handler body is at position i+3. + let Some(handler_body) = cmd.words.get(i + 3) else { + return false; + }; + if !branch_body_terminates( + handler_body, + source, + declared, + sig_table, + proc_table, + local_vars, + ) { + return false; + } + i += 4; + } + Some("finally") => { + // Finally script doesn't affect the terminator + // analysis — skip past its word. + i += 2; + } + _ => { + // Malformed / something we don't recognize; + // conservative false. + return false; + } + } + } + true +} + +/// Does the branch body word (a `WordForm::Braced` script) +/// terminate? Reparses the interior as a fragment and delegates +/// to `paths_always_return`. Non-braced body words (unusual — +/// only shows up when the parser hits malformed input) → false. +fn branch_body_terminates( + body_word: &crate::ast::Word, + source: &str, + declared: &crate::ast::TypeExpr, + sig_table: &HashMap, + proc_table: &HashMap, + local_vars: &VarTypeTable, +) -> bool { + if body_word.form != crate::ast::WordForm::Braced { + return false; + } + let word_start = body_word.span.start as usize; + let word_end = body_word.span.end as usize; + if word_end <= word_start + 2 { + return false; + } + let interior_start = word_start + 1; + let interior_end = word_end - 1; + let body_text = &source[interior_start..interior_end]; + let (mut body_stmts, _errs) = + crate::parser::parse_fragment(body_text, crate::parser::Mode::Toplevel); + for s in &mut body_stmts { + crate::parser::shift_stmt(s, interior_start as u32); + } + crate::parser::populate_procs(&mut body_stmts, source, &mut Vec::new()); + // Branch bodies share the outer scope's var table (Tcl + // control-flow doesn't create a new frame). + let mut branch_vars = local_vars.clone(); + paths_always_return( + &body_stmts, + source, + declared, + sig_table, + proc_table, + &mut branch_vars, + ) +} + +/// Walk `proc`'s body left-to-right, tracking `set VAR ` +/// bindings via [`value_type_with_procs`], then find the last +/// `return X` statement and resolve `X`'s type. `None` when the +/// body doesn't end with an inferrable return. +/// +/// Recursion depth is bounded implicitly by the finite proc-set +/// in a document: we don't cache visited procs here since v1 +/// documents don't hit deep recursion in practice. If a real +/// program starts driving this in circles, add a `HashSet<&str>` +/// guard on the proc name. +fn infer_return_type_from_body( + proc: &crate::ast::Proc, + sig_table: &HashMap, + proc_table: &HashMap, +) -> Option { + use crate::ast::{CommandKind, Stmt}; + let mut local_vars = VarTypeTable::new(); + // Seed the local var table with the proc's typed parameters — + // so a body like `proc pass_through {x: MyType} { return $x }` + // resolves through `$x` to `MyType`. + if let Some(sig) = &proc.signature { + for arg in &sig.args { + if let Some(ty) = &arg.type_annotation { + local_vars.insert(arg.name.clone(), ty.clone()); + } + } + } + // Walk `set` bindings in body order. + let mut return_word: Option<&crate::ast::Word> = None; + for stmt in &proc.body { + let Stmt::Command(cmd) = stmt else { continue }; + // Track `set VAR `. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type_with_procs( + value_word, + sig_table, + &local_vars, + Some(proc_table), + ) { + local_vars.insert(name.to_string(), ty); + } + } + } + } + // Track the last `return ` we see. Body execution + // ordinarily halts at `return`, but syntactically it's + // valid to have more code after (dead code); take the + // LAST occurrence since that's what the user's intent + // most likely reflects when reading the body. + if cmd.words.first().and_then(|w| w.as_text()) == Some("return") { + return_word = cmd.words.get(1); + } + } + let word = return_word?; + value_type_with_procs(word, sig_table, &local_vars, Some(proc_table)) +} + +/// Build a name → `Proc` lookup for return-type inference. Walks +/// top-level statements plus namespace-eval bodies (using the +/// namespace as a `::` prefix, matching how +/// [`build_signature_table`] qualifies proc names). +pub(crate) fn build_proc_table( + document: &crate::ast::Document, +) -> HashMap { + let mut out = HashMap::new(); + collect_procs(&document.stmts, "", &mut out); + out +} + +fn collect_procs<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + out: &mut HashMap, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + let qualified = qualify(prefix, name); + // Later `proc` shadows earlier — same as sig_table. + out.insert(qualified, proc); + } + } + CommandKind::NamespaceEval(ns) => { + let ns_name = ns.name.as_deref().unwrap_or(""); + let new_prefix = qualify(prefix, ns_name); + collect_procs(&ns.body, &new_prefix, out); + } + _ => {} + } + } +} + +/// True when a `TypeExpr` names the HTCL `bool` primitive. Kept +/// as a helper (rather than inlined) so the check has a single +/// point of change if we ever alias `bool` under a namespace +/// (e.g. `htcl::bool`). +fn is_bool_type(ty: &crate::ast::TypeExpr) -> bool { + matches!( + ty, + crate::ast::TypeExpr::Named { name, .. } if name == "bool" + ) +} + +/// Precomputed per-run lookups shared across every `validate_command` +/// invocation. Building once amortises the O(N_procs)- and +/// O(N_args)-per-proc setup that would otherwise run on every call +/// site. Populated at the top of `validate_with_all_extras_and_vars` +/// and threaded through `validate_stmts`. +pub(crate) struct SigEnv<'a> { + /// Per-signature `arg_name -> &ProcArg`. Replaces + /// `ProcSignature::find`'s O(N_args) linear scan in the + /// keyword-arg parse loop (dcmac wrappers carry ~890 args). + pub arg_indexes: HashMap>, + /// `bare_name -> [qualified proc names ending ::bare_name]`. + /// Lets the unknown-call path find a namespaced homonym in + /// O(1) instead of a linear scan of the whole signature + /// table on every builtin call. + pub suffix_index: HashMap<&'a str, Vec<&'a str>>, + /// Full list of qualified proc names, for the fuzzy sweep's + /// primary candidate set. Owned as `&str` slices into + /// `table`'s keys. + pub qualified_names: Vec<&'a str>, + /// Bare-name projection of `qualified_names` (last `::` + /// suffix, or the whole name if unqualified). The fuzzy + /// sweep used to compute this Vec on EVERY unknown positional + /// call — with a table of ~5000 wrapper procs and hundreds of + /// such call sites in a joined-source dump, that single + /// per-call alloc dominated `validate_stmts` (~100s on the + /// metroid workspace). Precomputed here once. + pub bare_names: Vec, +} + +impl<'a> SigEnv<'a> { + fn build(table: &'a HashMap) -> Self { + let arg_indexes: HashMap> = table + .iter() + .map(|(name, sig)| { + let idx: HashMap = + sig.args.iter().map(|a| (a.name.clone(), a)).collect(); + (name.clone(), idx) + }) + .collect(); + let mut suffix_index: HashMap<&str, Vec<&str>> = HashMap::new(); + for k in table.keys() { + if let Some((_, suffix)) = k.rsplit_once("::") { + suffix_index.entry(suffix).or_default().push(k.as_str()); + } + } + let qualified_names: Vec<&str> = + table.keys().map(String::as_str).collect(); + let bare_names: Vec = qualified_names + .iter() + .map(|k| { + k.rsplit_once("::") + .map(|(_, suffix)| suffix.to_string()) + .unwrap_or_else(|| (*k).to_string()) + }) + .collect(); + Self { + arg_indexes, + suffix_index, + qualified_names, + bare_names, + } + } +} + +#[allow(clippy::too_many_arguments)] +fn validate_stmts( + stmts: &[Stmt], + source: &str, + table: &HashMap, + env: &SigEnv, + proc_table: &HashMap, + newtype_names: &std::collections::HashSet, + var_table: &mut VarTypeTable, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + // Bind `set VAR ` into the var table BEFORE + // recursing — so downstream `$VAR` references in the same + // scope see the type. `validate_command` bails on + // `CommandKind::Set` (not a "call"), so this is the only + // place set-binding is observed. + if matches!(cmd.kind, CommandKind::Set) { + if let (Some(name_word), Some(value_word)) = + (cmd.words.get(1), cmd.words.get(2)) + { + if let Some(name) = name_word.as_text() { + if let Some(ty) = value_type(value_word, table, var_table) { + var_table.insert(name.to_string(), ty); + } + } + } + } + validate_command(cmd, source, table, env, var_table, diags); + match &cmd.kind { + CommandKind::Proc(proc) => { + // Return-type check: fires only when the proc has + // an annotated return type. Every `return X` in + // the body (including inside control-flow braced + // bodies) must produce a value whose inferred + // type matches the annotation. + validate_proc_returns( + proc, + source, + table, + proc_table, + newtype_names, + diags, + ); + // Fresh scope per proc body. Seed with typed + // parameters so `-slot $arg` inside the body knows + // `arg`'s declared type without the caller having + // to `set` it locally. + let mut proc_scope = VarTypeTable::new(); + if let Some(sig) = &proc.signature { + for a in &sig.args { + if let Some(ty) = &a.type_annotation { + proc_scope.insert(a.name.clone(), ty.clone()); + } + } + } + validate_stmts( + &proc.body, + source, + table, + env, + proc_table, + newtype_names, + &mut proc_scope, + diags, + ); + } + CommandKind::NamespaceEval(ns) => { + // Calls inside the namespace body are validated the + // same way; the signature-table is document-wide so + // a call to `project::set_target_language` from + // anywhere resolves to the same entry. (Bare, + // sibling-relative calls inside a namespace body + // aren't auto-qualified yet — write the qualified + // name explicitly.) Var scope is shared with the + // enclosing frame, matching Tcl's rule that + // `namespace eval` creates a namespace but not a + // fresh local-variable scope. + validate_stmts( + &ns.body, + source, + table, + env, + proc_table, + newtype_names, + var_table, + diags, + ); + } + _ => {} + } + // Also descend into any `[ … ]` command substitutions on this + // command's words so calls written inline get validated the + // same as top-level ones. Var-table shared with the + // enclosing scope — a `set X …` inside `[…]` is visible + // outside, per Tcl. + for word in &cmd.words { + for part in &word.parts { + if let WordPart::CmdSubst { body, .. } = part { + validate_stmts( + body, + source, + table, + env, + proc_table, + newtype_names, + var_table, + diags, + ); + } + } + } + } +} + +/// Build a name → signature map from every proc declaration in +/// the document, including those nested inside `namespace eval` +/// blocks (which register under `::`, matching Tcl's +/// namespace semantics). Duplicate names raise a diagnostic and the +/// later declaration wins, again matching Tcl (a second `proc` +/// redefines). +pub fn build_signature_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let (table, _overloads) = build_signature_table_with_overloads( + document, + &std::collections::HashSet::new(), + diags, + ); + table +} + +/// Same as [`build_signature_table`] but also returns the +/// [`OverloadTable`] side-map. Callers that need to know whether a +/// given proc name resolves through enum-overload dispatch (codegen, +/// hover, signature help) consult this. +pub fn build_signature_table_with_overloads<'doc>( + document: &'doc Document, + newtype_qualified_names: &std::collections::HashSet, + diags: &mut Vec, +) -> (HashMap, OverloadTable) { + // First pass: collect every proc decl per qualified name, + // preserving order so a "first wins" / "last wins" choice is + // unambiguous when we have to make one. Multi-decl entries are + // candidate overload sets; single-decl entries are normal + // procs. + let mut multi: HashMap> = + HashMap::new(); + collect_signatures_multi( + &document.stmts, + "", + newtype_qualified_names, + &mut multi, + diags, + ); + + let mut table: HashMap = HashMap::new(); + let mut overloads: OverloadTable = HashMap::new(); + + for (qualified, decls) in multi { + match decls.len() { + 0 => { /* impossible */ } + 1 => { + let (proc, sig) = decls[0]; + check_reserved_proc_name(&qualified, proc.name_span, diags); + table.insert(qualified, sig); + } + _ => { + // Multi-decl: classify as enum-overload OR emit + // hard error for ad-hoc overloading. + match classify_overload_set(&qualified, &decls, diags) { + Some(info) => { + // Each specialization registers under its + // mangled name so analyzer drill-down + the + // dispatcher's runtime switch can find it. + for v in &info.variants { + // Find the decl whose first arg is this + // variant. We computed the mangled name + // from it during classify, so the order + // matches by construction. + for (_proc, sig) in &decls { + let Some(first) = sig.args.first() else { + continue; + }; + if matches!( + &first.type_annotation, + Some(TypeExpr::Qualified { variant, .. }) + if variant == &v.variant_name + ) { + // Mangled names are compiler- + // generated — they're allowed to + // start with `__` (that's the + // whole point). Skip the + // reserved-name check here. + table.insert( + v.mangled_proc_name.clone(), + sig, + ); + } + } + } + // Public name resolves to the first overload's + // sig as a representative. Analyzer / callers + // that want the "true" public interface + // consult `overloads`. + let (proc, sig) = decls[0]; + check_reserved_proc_name( + &qualified, + proc.name_span, + diags, + ); + table.insert(qualified.clone(), sig); + overloads.insert(qualified, info); + } + None => { + // classify_overload_set already emitted the + // diagnostic; for table consistency, fall + // back to "last wins" so downstream + // validation keeps working. Check the + // reserved prefix on each. + let (proc, sig) = *decls.last().unwrap(); + check_reserved_proc_name( + &qualified, + proc.name_span, + diags, + ); + table.insert(qualified, sig); + } + } + } + } + } + + (table, overloads) +} + +/// User procs whose qualified name starts with `__` would collide +/// with the compiler's overload-specialization mangling +/// (`____`). Reject them up front. +fn check_reserved_proc_name( + qualified: &str, + name_span: Span, + diags: &mut Vec, +) { + // Look at the last segment after the final `::`. Tcl's + // namespace separator is part of the qualified name, so e.g. + // `vivado_cmd::__foo` has its "leaf" name as `__foo` — the + // collision risk is on the leaf, not the prefix. + let leaf = qualified.rsplit("::").next().unwrap_or(qualified); + if leaf.starts_with("__") { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc name `{qualified}` is reserved: names starting with \ + `__` are used by the compiler for overload-specialization \ + mangling (e.g. `__handle_prop__Scalar`). Rename to avoid \ + collisions." + ), + span: name_span, + }); + } +} + +fn collect_signatures_multi<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + newtype_qualified_names: &std::collections::HashSet, + multi: &mut HashMap>, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + // v1 restriction: enum-overloaded procs must be + // declared at the top level. Inside a `namespace + // eval` block, the REPL's batch-prepare layer + // doesn't re-route to the mangled-name + dispatcher + // pipeline, so an overload arm inside a namespace + // would silently lose its dispatch semantics. + // + // Only ENUM-Qualified first args (`Foo::Variant` where + // `Foo` is a declared enum) count as overload-arm + // shape. A `Qualified` type that resolves to a + // declared NEWTYPE (e.g. `-config: versal_cips::Config` + // in a `namespace eval versal_cips { proc create … }` + // block) is just a typed newtype ref — legal + // everywhere. The newtype-set peeked in by the caller + // disambiguates. + let overload_shape_first = sig + .args + .first() + .and_then(|a| a.type_annotation.as_ref()) + .and_then(|t| match t { + TypeExpr::Qualified { + namespace, variant, .. + } => Some(format!("{namespace}::{variant}")), + _ => None, + }) + .filter(|qname| !newtype_qualified_names.contains(qname)) + .is_some(); + if overload_shape_first && !prefix.is_empty() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overloaded proc `{name}` is declared inside \ + `namespace eval {prefix}` — v1 enum-overloads \ + must be declared at the top level. Move the \ + overload arms out of the namespace block." + ), + span: proc.name_span, + }); + continue; + } + let qualified = qualify(prefix, name); + multi.entry(qualified).or_default().push((proc, sig)); + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + // `extern` is reserved by htcl's lowering as the + // prefix for runtime-Tcl-proc disambiguation + // (`extern::foo` → `__vw_extern_foo`). A user- + // defined namespace named `extern` would silently + // collide with that rewrite at call sites; reject + // it up front. + if name == "extern" { + diags.push(Diagnostic { + severity: Severity::Error, + message: "`extern` is a reserved namespace name in \ + htcl (used for runtime-Tcl-proc \ + disambiguation); pick a different name" + .into(), + span: ns.name_span, + }); + continue; + } + let nested = qualify(prefix, name); + collect_signatures_multi( + &ns.body, + &nested, + newtype_qualified_names, + multi, + diags, + ); + } + _ => {} + } + } +} + +/// Classify a multi-decl proc-name set. Returns `Some(OverloadInfo)` +/// if every member's first arg is a distinct variant of the same +/// enum AND the tail args / return type agree; returns `None` and +/// emits a diagnostic if it's not a valid overload (ad-hoc +/// overloading, missing variant, tail mismatch, etc.). +fn classify_overload_set<'doc>( + public_name: &str, + decls: &[(&'doc Proc, &'doc ProcSignature)], + diags: &mut Vec, +) -> Option { + // Each decl's first arg must be `Qualified { namespace: E, variant: V }`. + // Collect (enum_name, variant_name, dispatch_arg_span) per decl. + let mut dispatch_infos: Vec<(String, String, Span, &Proc, &ProcSignature)> = + Vec::with_capacity(decls.len()); + for (proc, sig) in decls { + let Some(first) = sig.args.first() else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc `{public_name}` is declared multiple times; for \ + this to be a valid enum-overload set, every \ + declaration's first argument must be annotated with \ + a qualified variant type like `E::V`. This one has \ + no arguments." + ), + span: proc.name_span, + }); + return None; + }; + match &first.type_annotation { + Some(TypeExpr::Qualified { + namespace, variant, .. + }) => { + dispatch_infos.push(( + namespace.clone(), + variant.clone(), + first.name_span, + proc, + sig, + )); + } + _ => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "proc `{public_name}` is declared multiple times \ + with first-arg types that aren't all variants \ + of a common enum; ad-hoc overloading on arbitrary \ + types is not supported. Use an enum or rename \ + one of the procs." + ), + span: first.name_span, + }); + return None; + } + } + } + // All overloads must dispatch on the same enum. + let enum_name = dispatch_infos[0].0.clone(); + for (ns, _, sp, _, _) in &dispatch_infos[1..] { + if ns != &enum_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload set for proc `{public_name}` mixes enums: \ + `{enum_name}` and `{ns}`. All overloads in a set \ + must dispatch on the same enum." + ), + span: *sp, + }); + return None; + } + } + // Variants must be distinct. + { + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for (_, v, sp, _, _) in &dispatch_infos { + if !seen.insert(v.as_str()) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload set for proc `{public_name}` has two \ + arms dispatching on the same variant \ + `{enum_name}::{v}`. Each variant must have at \ + most one arm." + ), + span: *sp, + }); + return None; + } + } + } + // Tail-arg agreement: v1 restricts every arm to exactly one + // arg (the dispatched variant). Multi-arg overloads are + // future work — kwargs / specialization-binding interactions + // get hairy and the property-display motivating case doesn't + // need them. + let (_, _, _, first_proc, first_sig) = &dispatch_infos[0]; + if first_sig.args.len() != 1 { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` declares {} args; v1 \ + enum-overloads support exactly ONE arg (the dispatched \ + variant). Additional tail args are future work — model \ + the tail as a payload field on the enum variant for now.", + first_sig.args.len() + ), + span: first_sig.span, + }); + return None; + } + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + if sig.args.len() != 1 { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` declares \ + {} args; v1 enum-overloads support exactly ONE arg \ + (the dispatched variant).", + sig.args.len() + ), + span: sig.span, + }); + return None; + } + } + // Return-type agreement: every annotated return type must match. + // Mixed annotated/unannotated → error. + let first_ret = first_sig.return_type.as_ref(); + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + match (first_ret, sig.return_type.as_ref()) { + (None, None) => {} + (Some(a), Some(b)) if types_match(a, b) => {} + _ => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` \ + declares a different return type than the other \ + arms. All arms must agree on the return type \ + (annotate every arm with the same type, or none)." + ), + span: sig.span, + }); + return None; + } + } + } + // Arg-name agreement: every arm must use the same first-arg + // name so the dispatcher can pass the payload via kwargs as + // `- `. Cheaper than per-arm dispatch + // tracking and matches user convention (everyone writes `v`). + let dispatch_arg_name = first_sig.args[0].name.clone(); + for (ns, v, _, _, sig) in &dispatch_infos[1..] { + if sig.args[0].name != dispatch_arg_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "overload arm `{public_name}` for `{ns}::{v}` names its \ + dispatch arg `{}`; other arms name it `{dispatch_arg_name}`. \ + All arms must use the same arg name (convention: `v`).", + sig.args[0].name + ), + span: sig.args[0].name_span, + }); + return None; + } + } + // Build the OverloadInfo. Variant order matches source order + // of the overloads. + let variants = dispatch_infos + .iter() + .map(|(_, v, sp, _, _)| OverloadVariant { + variant_name: v.clone(), + mangled_proc_name: mangle_specialization(public_name, v), + dispatch_arg_span: *sp, + }) + .collect(); + Some(OverloadInfo { + public_name: public_name.to_string(), + enum_name, + dispatch_arg_name, + variants, + anchor_span: first_proc.name_span, + }) +} + +// `tails_match` / `attr_values_equal` lived here for the multi-arg +// overload tail-agreement check. v1 restricts overloads to a single +// arg (see `classify_overload_set`), so we don't compare tails. The +// helpers are kept as a record in git history; restore when adding +// multi-arg overloads. + +/// Collect every `type NAME = UNDERLYING` declaration in `document`, +/// qualified by enclosing `namespace eval` prefix (so a `type widget` +/// declared inside `namespace eval foo {}` registers as `foo::widget`, +/// matching how procs already qualify). Duplicate declarations emit +/// a warning and the later one wins — same shape as duplicate-proc +/// handling above. +pub fn build_type_decl_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let mut table = HashMap::new(); + collect_type_decls(&document.stmts, "", &mut table, diags); + table +} + +fn collect_type_decls<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + table: &mut HashMap, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::TypeDecl(td) => { + let Some(name) = td.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if table.insert(qualified.clone(), td).is_some() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "duplicate definition of type {qualified}; \ + later definition wins" + ), + span: td.name_span, + }); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + if name == "extern" { + continue; + } + let nested = qualify(prefix, name); + collect_type_decls(&ns.body, &nested, table, diags); + } + CommandKind::Proc(proc) => { + // Nested type decls inside proc bodies are unusual + // but not illegal — walk them so they register. + collect_type_decls(&proc.body, prefix, table, diags); + } + _ => {} + } + } +} + +/// Mirror of [`build_type_decl_table`] for `enum NAME = { ... }` +/// declarations. Duplicate enums warn and the later one wins — +/// same shape as type-decl handling. +pub fn build_enum_decl_table<'doc>( + document: &'doc Document, + diags: &mut Vec, +) -> HashMap { + let mut table = HashMap::new(); + collect_enum_decls(&document.stmts, "", &mut table, diags); + table +} + +fn collect_enum_decls<'doc>( + stmts: &'doc [Stmt], + prefix: &str, + table: &mut HashMap, + diags: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::EnumDecl(ed) => { + let Some(name) = ed.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if table.insert(qualified.clone(), ed).is_some() { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "duplicate definition of enum {qualified}; \ + later definition wins" + ), + span: ed.name_span, + }); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + if name == "extern" { + continue; + } + let nested = qualify(prefix, name); + collect_enum_decls(&ns.body, &nested, table, diags); + } + CommandKind::Proc(proc) => { + collect_enum_decls(&proc.body, prefix, table, diags); + } + _ => {} + } + } +} + +/// Per-enum sanity checks. v1: variants must have distinct names; +/// payload types are syntactically valid (already enforced by +/// `enum_parse`); a payload that references an unknown user type +/// is a soft warning for now (could be defined cross-batch). +fn validate_enum_decls( + enum_table: &HashMap, + _type_table: &HashMap, + diags: &mut Vec, +) { + for (qualified, ed) in enum_table { + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for v in &ed.variants { + if !seen.insert(v.name.as_str()) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "enum `{qualified}` declares variant `{}` more than \ + once. Each variant name must be unique within an \ + enum.", + v.name + ), + span: v.name_span, + }); + } + } + } +} + +/// Walk the document and reject [`TypeExpr::Qualified`] anywhere +/// other than as a proc's first-arg type annotation. Qualified +/// types (`E::V`) are only meaningful as overload-dispatch +/// indicators unless they resolve to a declared newtype — those +/// pass through as regular namespaced type references. +/// +/// `newtype_names` carries the qualified names of every declared +/// newtype in the document (built via +/// [`build_type_decl_table`]); it's the disambiguator between +/// enum-variant refs and namespaced newtype refs. +fn validate_qualified_positions( + document: &Document, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + fn walk_stmts( + stmts: &[Stmt], + newtype_names: &std::collections::HashSet, + diags: &mut Vec, + ) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { continue }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(sig) = proc.signature.as_ref() { + for (i, arg) in sig.args.iter().enumerate() { + if let Some(ty) = arg.type_annotation.as_ref() { + // The first arg may be Qualified; + // tail args may NOT (unless a + // known-newtype ref, handled by the + // reject fn itself). + let allow_qualified = i == 0; + reject_nested_qualified( + ty, + allow_qualified, + newtype_names, + diags, + ); + } + } + if let Some(ret) = sig.return_type.as_ref() { + reject_nested_qualified( + ret, + false, + newtype_names, + diags, + ); + } + } + walk_stmts(&proc.body, newtype_names, diags); + } + CommandKind::NamespaceEval(ns) => { + walk_stmts(&ns.body, newtype_names, diags); + } + CommandKind::TypeDecl(td) => { + if let Some(ty) = td.underlying.as_ref() { + reject_nested_qualified( + ty, + false, + newtype_names, + diags, + ); + } + } + CommandKind::EnumDecl(ed) => { + for v in &ed.variants { + if let Some(ty) = v.payload.as_ref() { + reject_nested_qualified( + ty, + false, + newtype_names, + diags, + ); + } + } + } + _ => {} + } + } + } + walk_stmts(&document.stmts, newtype_names, diags); +} + +fn reject_nested_qualified( + ty: &TypeExpr, + allow_top_qualified: bool, + newtype_names: &std::collections::HashSet, + diags: &mut Vec, +) { + match ty { + TypeExpr::Named { .. } => {} + TypeExpr::Generic { args, .. } => { + // Inside a generic, nested Qualified is never allowed — + // except for known-newtype references, which are just + // namespaced type names. + for a in args { + reject_nested_qualified(a, false, newtype_names, diags); + } + } + TypeExpr::Qualified { + namespace, + variant, + span, + .. + } => { + // A qualified name that resolves to a declared newtype + // is a regular namespaced type reference — legal + // wherever a Named type is legal, including return + // types and generic args. + let qualified = format!("{namespace}::{variant}"); + if newtype_names.contains(&qualified) { + return; + } + if !allow_top_qualified { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "qualified type `{namespace}::{variant}` is only \ + legal as the first-argument type annotation on an \ + overloaded handler proc. It can't appear as a \ + return type, generic argument, type-decl \ + underlying, or enum-variant payload." + ), + span: *span, + }); + } + } + } +} + +/// For each newtype declaration `T`, verify the user provided the +/// required `T::repr`, `T::from`, `T::to` procs with the correct +/// shapes: +/// +/// - `T::repr` takes one arg named `v` of type `T` (or untyped), +/// returns `string` (or untyped). +/// - `T::from` takes one arg named `v` of type `` (or +/// untyped), returns `T` (or untyped). +/// - `T::to` takes one arg named `v` of type `T` (or untyped), +/// returns `` (or untyped). +/// +/// Type annotations are *optional* on these procs — an untyped +/// arg or return slot is accepted as a "trust the user" form +/// (some procs ship pre-arg-types and were authored before the +/// shape check existed). The arg COUNT and NAME (`v`) are +/// always enforced; the type slots get a stricter check only +/// when the user opted in by annotating them. +fn validate_type_decl_triplets( + type_table: &HashMap, + sig_table: &HashMap, + diags: &mut Vec, +) { + use crate::ast::TypeExpr; + for (qualified_name, td) in type_table { + let underlying = td.underlying.as_ref(); + let slots: &[(&str, Option<&TypeExpr>, Option<&str>)] = &[ + // (slot, arg type expected, return type expected as + // type-name). We pass the type-name via Option<&str> + // and compare with TypeExpr::Named's name; that's + // sufficient for the v1 set (all involved types are + // either named primitives or named newtypes; no + // generics in repr/from/to signatures). + ( + "repr", + // arg should be T + Some(&named_lit(qualified_name)), + Some("string"), + ), + ( + "from", + // arg should be + underlying, + // return should be T + Some(qualified_name.as_str()), + ), + ( + "to", + // arg should be T + Some(&named_lit(qualified_name)), + // return should be + underlying.and_then(|u| match u { + TypeExpr::Named { name, .. } => Some(name.as_str()), + _ => None, + }), + ), + ]; + for (slot, expected_arg, expected_ret) in slots { + let want = format!("{qualified_name}::{slot}"); + let Some(sig) = sig_table.get(&want) else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype `{qualified_name}` is missing required \ + proc `{qualified_name}::{slot}` (see \ + docs/htcl-return-types.md)." + ), + span: td.name_span, + }); + continue; + }; + // Arg count + name. + if sig.args.len() != 1 || sig.args[0].name != "v" { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}` must \ + take exactly one argument named `v`" + ), + span: sig.span, + }); + continue; + } + // Arg type — only checked when the user annotated it. + if let (Some(actual), Some(expected)) = + (sig.args[0].type_annotation.as_ref(), expected_arg) + { + if !types_match(actual, expected) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}`: \ + arg `v` is declared `{}` but should be \ + `{}`", + render_type_inline(actual), + render_type_inline(expected) + ), + span: sig.args[0].name_span, + }); + } + } + // Return type — only checked when the user annotated it + // and we know what to compare against. + if let (Some(actual), Some(want_name)) = + (sig.return_type.as_ref(), expected_ret) + { + // Compare on the identifier's name. For Qualified + // (namespaced newtype refs like `dcmac::GtChProps`) + // we join the parts so the compare matches the + // qualified-name key `want_name` carries. + let actual_name_owned: String; + let actual_name = match actual { + TypeExpr::Named { name, .. } => name.as_str(), + TypeExpr::Generic { name, .. } => name.as_str(), + TypeExpr::Qualified { + namespace, variant, .. + } => { + actual_name_owned = format!("{namespace}::{variant}"); + actual_name_owned.as_str() + } + }; + if actual_name != *want_name { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "newtype proc `{qualified_name}::{slot}` \ + returns `{}` but should return `{want_name}`", + render_type_inline(actual) + ), + span: sig.span, + }); + } + } + } + } +} + +/// Build a one-shot `TypeExpr::Named` literal for comparison +/// purposes. The span is meaningless here — we only ever +/// inspect the name. +fn named_lit(name: &str) -> crate::ast::TypeExpr { + crate::ast::TypeExpr::Named { + name: name.to_string(), + span: Span::new(0, 0), + } +} + +/// Structural equality on type expressions, ignoring spans. +/// +/// `Qualified { ns, var }` is treated as equivalent to +/// `Named { name: "ns::var" }` — the two forms describe the same +/// identifier and callers that need to compare an ast-parsed +/// annotation against a synthetic Named expected type (e.g. +/// `named_lit(qualified_name)` inside newtype-triplet validation) +/// shouldn't see a spurious mismatch. +fn types_match(a: &crate::ast::TypeExpr, b: &crate::ast::TypeExpr) -> bool { + use crate::ast::TypeExpr; + match (a, b) { + ( + TypeExpr::Named { name: an, .. }, + TypeExpr::Named { name: bn, .. }, + ) => an == bn, + ( + TypeExpr::Generic { + name: an, args: aa, .. + }, + TypeExpr::Generic { + name: bn, args: ba, .. + }, + ) => { + an == bn + && aa.len() == ba.len() + && aa.iter().zip(ba.iter()).all(|(x, y)| types_match(x, y)) + } + // Cross-form equivalence for qualified newtype references + // (`dcmac::GtChProps` on one side, `Named("dcmac::GtChProps")` + // on the other). Commutative. + ( + TypeExpr::Qualified { + namespace, variant, .. + }, + TypeExpr::Named { name, .. }, + ) + | ( + TypeExpr::Named { name, .. }, + TypeExpr::Qualified { + namespace, variant, .. + }, + ) => *name == format!("{namespace}::{variant}"), + ( + TypeExpr::Qualified { + namespace: ans, + variant: av, + .. + }, + TypeExpr::Qualified { + namespace: bns, + variant: bv, + .. + }, + ) => ans == bns && av == bv, + _ => false, + } +} + +/// Render a type expression for inclusion in a diagnostic message. +/// Mirrors `vw-analyzer/src/htcl_backend.rs::render_type` — kept +/// in sync by convention since the analyzer can't depend on +/// validate.rs. +fn render_type_inline(ty: &crate::ast::TypeExpr) -> String { + use crate::ast::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = + args.iter().map(render_type_inline).collect(); + format!("{name}<{}>", inner.join(",")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => { + format!("{namespace}::{variant}") + } + } +} + +/// Tcl core builtins that legitimately take either `-flag` +/// arguments natively (`string match -nocase`, `regexp -line`, +/// `lsort -unique`) or take positional list arguments that +/// commonly start with `-` (e.g. `lappend cmd -ruledeck $x` where +/// `-ruledeck` is being appended as a literal token, not parsed +/// by `lappend`). Calls to anything in this list pass the +/// unknown-call check unconditionally. +/// +/// Keep this small but pragmatic: a missed builtin produces a +/// pestering error on calls that work fine; an over-included name +/// hides a real "you forgot to src @x" mistake. The set below is +/// the standard Tcl core surface most htcl bodies actually use. +fn is_known_tcl_builtin(name: &str) -> bool { + matches!( + name, + // Container ops whose positional args often look like flags. + "lappend" + | "lset" + | "linsert" + | "lreplace" + | "lrange" + | "lindex" + | "list" + | "llength" + | "dict" + | "array" + | "set" + | "unset" + | "incr" + | "append" + | "concat" + // String / regex / sort builtins that accept `-flag`s natively. + | "string" + | "regexp" + | "regsub" + | "lsort" + | "lsearch" + | "switch" + | "format" + | "scan" + | "binary" + // Control flow. Note these are parsed as `Generic` + // commands (no dedicated `CommandKind`), so + // `validate_command` sees `if`/`while`/`foreach`/... as + // regular calls and would otherwise send every one + // through the fuzzy sweep — `if` alone can hit six + // figures of call sites in a joined-source dump. + | "if" + | "elseif" + | "else" + | "then" + | "for" + | "foreach" + | "while" + | "do" + | "break" + | "continue" + | "proc" + | "source" + // Flow / introspection / interp. + | "after" + | "eval" + | "uplevel" + | "upvar" + | "apply" + | "info" + | "package" + | "catch" + | "try" + | "throw" + | "error" + | "return" + | "expr" + | "subst" + | "time" + | "trace" + | "unknown" + // More list / string builtins. + | "split" + | "join" + | "lreverse" + | "lassign" + | "lmap" + | "lrepeat" + | "dict_get" + // I/O & filesystem. + | "puts" + // `putr` is our compile-time repr-dispatching shim: + // `putr $x` gets rewritten in `crate::putr::rewrite` + // to `puts [T::repr -v $x]` when the argument's type + // is statically known, else to plain `puts $x`. The + // rewrite fires before any code reaches Tcl, so at + // eval time `putr` isn't a real command — the + // analyzer needs to recognize it as a builtin so + // undefined-proc checks don't flag the call sites. + | "putr" + | "gets" + | "read" + | "close" + | "open" + | "file" + | "exec" + | "fconfigure" + | "fileevent" + | "flush" + | "socket" + | "chan" + | "pwd" + | "cd" + | "glob" + | "pid" + | "env" + | "clock" + // Channels / Tk-style. + | "namespace" + | "variable" + | "global" + | "rename" + | "interp" + ) +} + +/// Compiler-emitted primitive-prelude procs: `::` for a +/// primitive type `T` and a conversion suffix. `emit_primitive_prelude` +/// (in `crate::repr`) ships a `namespace eval { proc repr … }` +/// block for each primitive at session start — they are NOT `src`d, so +/// a direct call like `list::repr -v {…}` or `dict::repr -v $d` has no +/// entry in the proc table and would otherwise trip the unknown-call +/// check. Same rationale as `putr` in [`is_known_tcl_builtin`]: the +/// analyzer has to know these exist. Keep the `(type, suffix)` sets in +/// lockstep with `emit_primitive_prelude`. +fn is_primitive_prelude_proc(name: &str) -> bool { + let Some((ns, leaf)) = name.rsplit_once("::") else { + return false; + }; + matches!(ns, "string" | "int" | "bool" | "unit" | "list" | "dict") + && matches!(leaf, "repr" | "from" | "to" | "to_raw" | "from_raw") +} + +/// Join a namespace prefix with a member name using Tcl's `::` +/// separator. The empty prefix yields the bare name (used at the +/// document root where there's no enclosing namespace). +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +fn validate_command( + cmd: &Command, + source: &str, + table: &HashMap, + env: &SigEnv, + var_table: &VarTypeTable, + diags: &mut Vec, +) { + let call_name = match &cmd.kind { + CommandKind::Generic => match cmd.words.first() { + Some(w) => match w.as_text() { + Some(t) => t, + None => return, + }, + None => return, + }, + // Don't validate inside declarations themselves — those + // aren't calls. (NamespaceEval is a declaration; its body's + // statements are validated by the recursion in + // `validate_stmts`.) + CommandKind::Proc(_) + | CommandKind::Set + | CommandKind::Src(_) + | CommandKind::NamespaceEval(_) + | CommandKind::TypeDecl(_) + | CommandKind::EnumDecl(_) => { + return; + } + }; + // `extern::name` is the user's opt-out: "this call resolves + // to a runtime Tcl proc, don't analyze its signature." Lowering + // strips the prefix and aliases the underlying proc into place. + if crate::lower::is_extern_call(call_name) { + return; + } + let Some(sig) = table.get(call_name) else { + // Unknown call. Two paths fire an error: + // + // 1. The call uses `-flag` keyword arguments. Almost + // always the user meant an htcl wrapper that isn't + // loaded — shipping it raw to the EDA backend either + // errors cryptically or misinterprets the args. + // + // 2. The unqualified name has a matching namespaced + // proc in scope (e.g., `get_bd_addr_spaces` when + // `vivado_cmd::get_bd_addr_spaces` exists). That's a + // missed namespace prefix on the same wrapper the + // user is calling elsewhere with the qualified name. + // Catching this even for positional-only calls is + // what makes the analyzer's behavior consistent — + // otherwise `assign_bd_address` errors (has `-flag` + // args) but `[get_bd_addr_spaces X]` inside its arg + // silently passes, which reads as an analyzer gap. + // + // A positional-only call to a bare Tcl builtin (`llength`, + // `dict`, etc.) still passes cleanly: `is_known_tcl_builtin` + // filters those, and there's no namespaced homonym. + // + // Short-circuit builtins FIRST — every `if`, `set`, `puts`, + // `list`, etc. hits this path and there can be tens of + // thousands of such call sites in a healthy source. The + // downstream `suffix_index` and fuzzy sweep are cheap per + // call (index is O(1), fuzzy allocs a bare-name Vec) but + // even cheap-per-call adds up at that scale. + if is_known_tcl_builtin(call_name) + || is_primitive_prelude_proc(call_name) + { + return; + } + let uses_keyword = cmd.words.iter().skip(1).any(|w| { + w.as_text() + .is_some_and(|t| t.starts_with('-') && t.len() > 1) + }); + let namespaced_match = if !call_name.contains("::") { + env.suffix_index + .get(call_name) + .and_then(|qs| qs.iter().min_by_key(|s| s.len())) + .map(|s| s.to_string()) + } else { + None + }; + // A Levenshtein-close hit against a proc the table already + // knows about is strong evidence the call is USER code with a + // typo. Compute lazily — only after we've confirmed the call + // isn't a builtin or prelude proc, so `bare_names` isn't + // allocated on every one of the thousands of legitimate + // builtin calls in a large workspace. + // + // Match against BOTH qualified names (`ip::generate_dcmac`) + // and their bare suffixes (`generate_dcmac`). Bare calls in + // the same namespace as the target are the common case — + // e.g. inside `namespace eval ip { ... }`, a call to + // `generate_dcmacc` fuzzy-matches the bare `generate_dcmac` + // at distance 1, but the qualified `ip::generate_dcmac` at + // distance 5 (past the length-scaled threshold). Rebuild + // the qualified name for the "did you mean" hint by + // finding the table entry whose suffix matches. + let need_fuzzy = !uses_keyword && namespaced_match.is_none(); + let (fuzzy_match, fuzzy_hint) = if need_fuzzy { + let suggestion = + suggest_name(call_name, env.qualified_names.iter().copied()) + .or_else(|| { + suggest_name( + call_name, + env.bare_names.iter().map(String::as_str), + ) + }); + let hint_name = suggestion.as_ref().map(|s| { + if s.contains("::") { + s.clone() + } else { + env.suffix_index + .get(s.as_str()) + .and_then(|qs| qs.iter().min_by_key(|k| k.len())) + .map(|q| q.to_string()) + .or_else(|| table.get(s.as_str()).map(|_| s.clone())) + .unwrap_or_else(|| s.clone()) + } + }); + (suggestion, hint_name) + } else { + // Non-fuzzy path: `-flag` args or an exact namespaced + // homonym trip the diagnostic on their own; the + // suggestion (when the -flag path fires) comes from + // the plain `suggest_name` sweep below. + (None, None) + }; + let should_flag = + uses_keyword || namespaced_match.is_some() || fuzzy_match.is_some(); + if should_flag { + // Prefer the exact namespaced match as the "did you + // mean" — it's a stronger signal than the fuzzy + // Levenshtein suggestion, which for the bare-name + // case would surface the same or a nearby name + // anyway. + let hint = if let Some(qualified) = &namespaced_match { + format!(" — did you mean `{qualified}`?") + } else if let Some(s) = fuzzy_hint { + format!(" — did you mean `{s}`?") + } else { + match suggest_name( + call_name, + env.qualified_names.iter().copied(), + ) { + Some(s) => format!(" — did you mean `{s}`?"), + None => String::new(), + } + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "undefined proc `{call_name}`{hint}; either \ + `src` a module that defines it or use \ + `extern::{call_name}` to call the underlying \ + Tcl proc directly" + ), + span: cmd.words[0].span, + }); + } + return; + }; + + // Parse keyword args from the command's words. The first word is + // the call name; the remaining words alternate -flag/value. + let mut idx = 1usize; + let mut seen: HashMap = HashMap::new(); + while idx < cmd.words.len() { + let word = &cmd.words[idx]; + let flag_text = match word.as_text() { + Some(t) if t.starts_with('-') => &t[1..], + Some(t) => { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!("expected keyword argument, found {t}"), + span: word.span, + }); + idx += 1; + continue; + } + None => { + diags.push(Diagnostic { + severity: Severity::Error, + message: "expected keyword argument".into(), + span: word.span, + }); + idx += 1; + continue; + } + }; + let flag_name = flag_text.to_string(); + let value_word = cmd.words.get(idx + 1); + + // O(1) flag lookup via the precomputed per-sig arg index. + // Fall back to `sig.find` when we don't have an index for + // this call — the index is keyed by qualified proc name, + // and a namespaced-shadow / entry-point mismatch would + // point us at a sig without a prebuilt index. + let arg_lookup: Option<&ProcArg> = env + .arg_indexes + .get(call_name) + .and_then(|idx| idx.get(&flag_name)) + .copied() + .or_else(|| sig.find(&flag_name)); + match arg_lookup { + None => { + let known: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + let hint = if known.is_empty() { + String::new() + } else { + format!(". Possible values are {}", known.join(", ")) + }; + diags.push(Diagnostic { + severity: Severity::Error, + message: format!("undefined argument -{flag_name}{hint}"), + span: word.span, + }); + } + Some(arg) => { + if let Some(prev) = seen.insert(flag_name.clone(), word.span) { + let _ = prev; + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!("duplicate argument -{flag_name}"), + span: word.span, + }); + } + if let Some(value) = value_word { + validate_value(call_name, arg, value, source, diags); + // Nominal type check for the value expression. + // Only fires when BOTH sides have known types — + // the caller wrote a `: TYPE` annotation on the + // arg (populated by proc_args parsing into + // `ProcArg.type_annotation`) AND the value word + // is one whose type `value_type` can infer + // (`[proc-call]` return type or `$var` binding). + // Literals and mixed compounds silently skip + // (gradual typing). + // + // Identity via `types_match` — no alias walking + // (see `VarTypeTable` docs). `Quad0Ch1Props ≠ + // Quad1Ch0Props ≠ Properties` even when both + // alias the same underlying; that's what + // catches the copy-paste-wrong-constructor bug + // this check exists for. + if let Some(declared) = &arg.type_annotation { + if let Some(actual) = + value_type(value, table, var_table) + { + if !types_match(declared, &actual) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "type mismatch for -{}: expected \ + `{}`, found `{}`", + flag_name, + render_type_inline(declared), + render_type_inline(&actual), + ), + span: value.span, + }); + } + } else if is_bool_type(declared) { + // `bool`-typed slot with a value the + // `value_type` pass can't infer. The + // ONLY textual values that produce a + // known `bool` are the bare `true` / + // `false` literals (see `value_type`); + // any other whole-word text literal at + // this slot is a mistyped bool + // (`-flag 1`, `-flag yes`, `-flag + // potato`). Reject with a message + // naming the offending literal so the + // caller can rewrite it. + // + // Skips vars/cmdsubst and compound + // words: those return None from + // value_type because we couldn't + // deduce the type, not because they're + // definitively wrong. + if let Some(lit) = value.as_text() { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "type mismatch for -{}: expected \ + `bool`, found literal `{}` (use \ + `true` or `false`)", + flag_name, lit, + ), + span: value.span, + }); + } + } + } + // Redundant-default warning: the caller wrote + // `-flag X` where X exactly equals the arg's + // `@default(X)`. Fires only for literal-text + // values on both sides — `$var` / `[cmd]` + // arg-side and non-literal defaults skip. Empty + // defaults (`@default("")`) are a common + // "sentinel for unset" idiom in the generated + // wrappers, so we don't flag a matching + // `-flag ""` — the user asked for the value + // explicitly and the wrapper's `__vw_kw_..._set` + // machinery would still observe kw_set=true. + warn_if_redundant_default(&flag_name, arg, value, diags); + } else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} is missing a value" + ), + span: word.span, + }); + } + } + } + // Step past the flag and its value. + idx += if value_word.is_some() { 2 } else { 1 }; + } + + // Build canonical `@one_of` groups. Each arg's `@one_of(...)` + // declares an alternatives set: the arg itself plus the named + // siblings. We collapse declarations from each direction (sib A + // says `@one_of(B)` and sib B says `@one_of(A)`) into one + // canonical group, then check that **exactly one** arg from each + // group is supplied at the call site. + // + // Args participating in a group are treated as optional for the + // missing-required check below — the group rule is the source of + // truth for "must supply something." + let one_of_groups = collect_one_of_groups(sig); + let in_one_of: std::collections::HashSet<&str> = one_of_groups + .iter() + .flat_map(|g| g.iter().map(String::as_str)) + .collect(); + + // Required-args check. An arg is required when it has no + // `@default` to fall back to — the user must supply a value. + // Args in an `@one_of` group are governed by the group rule + // instead, so skip them here. + for arg in &sig.args { + if seen.contains_key(&arg.name) { + continue; + } + if in_one_of.contains(arg.name.as_str()) { + continue; + } + // Optional if the arg has EITHER `@default(...)` (Rust-style + // "value if omitted") OR `@baseline(...)` (documented + // fresh-IP value; runtime still gates emission on + // `__vw_kw_..._set`, so omission is a legitimate call + // shape). Missing both → the caller MUST supply a value. + let is_required = arg.attribute("default").is_none() + && arg.attribute("baseline").is_none(); + if is_required { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "missing required argument -{name}", + name = arg.name, + ), + span: cmd.span, + }); + } + } + + // `@one_of` groups: exactly one alternative must be present. + for group in &one_of_groups { + let present: Vec<&str> = group + .iter() + .filter(|n| seen.contains_key(n.as_str())) + .map(String::as_str) + .collect(); + if present.len() == 1 { + continue; + } + let opts: Vec = group.iter().map(|n| format!("-{n}")).collect(); + let message = if present.is_empty() { + format!( + "missing required argument — exactly one of {} must be \ + supplied", + opts.join(", ") + ) + } else { + let got: Vec = + present.iter().map(|n| format!("-{n}")).collect(); + format!( + "exactly one of {} may be supplied, got {}", + opts.join(", "), + got.join(", ") + ) + }; + diags.push(Diagnostic { + severity: Severity::Error, + message, + span: cmd.span, + }); + } + + // Inter-arg deps for present args. + for (flag_name, flag_span) in &seen { + // Same O(1) shortcut as the parse-loop lookup — falls + // back to the linear scan when the index doesn't cover + // this sig. + let Some(arg) = env + .arg_indexes + .get(call_name) + .and_then(|idx| idx.get(flag_name.as_str())) + .copied() + .or_else(|| sig.find(flag_name)) + else { + continue; + }; + if let Some(req) = arg.attribute("requires") { + for value in &req.values { + let referenced = match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.as_str(), + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + }; + if !seen.contains_key(referenced) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} requires -{referenced} \ + to also be set" + ), + span: *flag_span, + }); + } + } + } + if let Some(conflicts) = arg.attribute("conflicts") { + for value in &conflicts.values { + let referenced = match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.as_str(), + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + }; + if seen.contains_key(referenced) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{flag_name} conflicts with \ + -{referenced}" + ), + span: *flag_span, + }); + } + } + } + if arg.attribute("deprecated").is_some() { + let msg = arg + .attribute("deprecated") + .and_then(|a| a.values.first()) + .map(|v| v.as_str().to_string()) + .unwrap_or_default(); + let m = if msg.is_empty() { + format!("argument -{flag_name} is deprecated") + } else { + format!("argument -{flag_name} is deprecated: {msg}") + }; + diags.push(Diagnostic { + severity: Severity::Warning, + message: m, + span: *flag_span, + }); + } + } +} + +/// Collect canonical `@one_of` alternatives groups for a signature. +/// +/// Each arg's `@one_of(sib1, sib2, ...)` declares that exactly one +/// of `{arg, sib1, sib2, ...}` must be supplied at the call site. +/// We treat the declaration as symmetric (both `dict @one_of(name)` +/// and `name @one_of(dict)` describe the same group), so a `BTreeSet` +/// of the participating names canonicalizes each group regardless of +/// which direction (or which redundant copies) the author wrote. +fn collect_one_of_groups( + sig: &ProcSignature, +) -> Vec> { + use std::collections::BTreeSet; + let mut seen: std::collections::HashSet> = + std::collections::HashSet::new(); + let mut out: Vec> = Vec::new(); + for arg in &sig.args { + let Some(attr) = arg.attribute("one_of") else { + continue; + }; + let mut group: BTreeSet = BTreeSet::new(); + group.insert(arg.name.clone()); + for value in &attr.values { + match value { + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => { + group.insert(value.clone()); + } + AttributeValue::Integer { .. } + | AttributeValue::Keyed { .. } => continue, + } + } + if group.len() >= 2 && seen.insert(group.clone()) { + out.push(group); + } + } + out +} + +fn validate_value( + call_name: &str, + arg: &ProcArg, + value_word: &Word, + _source: &str, + diags: &mut Vec, +) { + // For Phase 2 we only validate literal-text values. Word forms + // that include `$var` or `[cmd]` are runtime-dynamic; we let them + // through silently. Future work can teach the validator about + // values produced by known builtins. + let Some(literal) = literal_value(value_word) else { + return; + }; + + if let Some(enum_attr) = arg.attribute("enum") { + check_enum( + call_name, &arg.name, enum_attr, &literal, value_word, diags, + ); + } + if let Some(range_attr) = arg.attribute("range") { + check_range( + call_name, &arg.name, range_attr, &literal, value_word, diags, + ); + } +} + +/// Warn when a caller passes `-flag X` where X exactly equals the +/// arg's `@default(X)`. Fires on literal-text values on BOTH sides; +/// interpolated values (`$var`, `[cmd]`) are runtime-dynamic and +/// skip. Skips empty-string defaults — those are the "sentinel for +/// unset" idiom used across the auto-generated wrappers. +fn warn_if_redundant_default( + flag_name: &str, + arg: &ProcArg, + value_word: &Word, + diags: &mut Vec, +) { + let Some(default_attr) = arg.attribute("default") else { + return; + }; + // `@default` takes one positional value. Skip weird shapes + // (no value, multiple values, `key=value`) — they aren't a + // meaningful equality target. + let [only] = default_attr.values.as_slice() else { + return; + }; + let default_str: &str = match only { + AttributeValue::Integer { value, .. } => { + return warn_if_int_match(flag_name, *value, value_word, diags); + } + AttributeValue::String { value, .. } + | AttributeValue::Ident { value, .. } => value.as_str(), + AttributeValue::Keyed { .. } => return, + }; + // Sentinel-unset convention — see doc comment. + if default_str.is_empty() { + return; + } + let Some(literal) = literal_value(value_word) else { + return; + }; + if literal == default_str { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "redundant `-{flag_name} {literal}` — this is the arg's \ + default; omit the flag" + ), + span: value_word.span, + }); + } +} + +fn warn_if_int_match( + flag_name: &str, + default: i64, + value_word: &Word, + diags: &mut Vec, +) { + let Some(literal) = literal_value(value_word) else { + return; + }; + let Ok(actual) = literal.parse::() else { + return; + }; + if actual == default { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "redundant `-{flag_name} {literal}` — this is the arg's \ + default; omit the flag" + ), + span: value_word.span, + }); + } +} + +fn literal_value(word: &Word) -> Option { + let mut out = String::new(); + for part in &word.parts { + match part { + WordPart::Text { value, .. } => out.push_str(value), + WordPart::Escape { value, .. } => out.push(*value), + WordPart::VarRef { .. } | WordPart::CmdSubst { .. } => { + // Dynamic content — not a literal. + return None; + } + } + } + Some(out) +} + +fn check_enum( + _call_name: &str, + arg_name: &str, + enum_attr: &Attribute, + literal: &str, + value_word: &Word, + diags: &mut Vec, +) { + let allowed: Vec = enum_attr + .values + .iter() + .map(|v| match v { + AttributeValue::Integer { value, .. } => value.to_string(), + AttributeValue::Ident { value, .. } + | AttributeValue::String { value, .. } => value.clone(), + // `key=value` in @enum(…) doesn't make semantic sense + // (enum values are positional). Include the literal + // `key=value` string so a runtime match against the + // raw arg still works if someone actually did that. + AttributeValue::Keyed { .. } => v.to_tcl_literal(), + }) + .collect(); + if !allowed.iter().any(|a| a == literal) { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "value {literal} for -{arg_name} is not in @enum. Possible \ + values are {}", + allowed.join(", ") + ), + span: value_word.span, + }); + } +} + +fn check_range( + _call_name: &str, + arg_name: &str, + range_attr: &Attribute, + literal: &str, + value_word: &Word, + diags: &mut Vec, +) { + let (Some(min), Some(max)) = + (range_attr.values.first(), range_attr.values.get(1)) + else { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!( + "@range on -{arg_name} should have two numeric bounds" + ), + span: range_attr.span, + }); + return; + }; + let ( + AttributeValue::Integer { value: min, .. }, + AttributeValue::Integer { value: max, .. }, + ) = (min, max) + else { + diags.push(Diagnostic { + severity: Severity::Warning, + message: format!("@range on -{arg_name} has non-integer bounds"), + span: range_attr.span, + }); + return; + }; + let Ok(n) = literal.parse::() else { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "argument -{arg_name} expects an integer, found {literal}" + ), + span: value_word.span, + }); + return; + }; + if n < *min || n > *max { + diags.push(Diagnostic { + severity: Severity::Error, + message: format!( + "value {n} for -{arg_name} is out of @range({min}, {max})" + ), + span: value_word.span, + }); + } +} + +/// Standard compiler-style "did you mean X?" suggestion: pick the +/// in-scope name with the smallest edit distance from `target`, +/// within a length-scaled threshold. Returns `None` when no +/// candidate is close enough (so unknown calls that aren't +/// near-misses don't get nonsense suggestions tacked on). +fn suggest_name<'a, I>(target: &str, candidates: I) -> Option +where + I: IntoIterator, +{ + // rustc-style threshold: scales with name length so single-char + // typos count for short names, but a 12-char identifier + // tolerates a few keystroke errors. Floor at 1, ceiling at 3 — + // anything past 3 starts producing surprising suggestions. + // + // Proc names in the workspace are ASCII-only (identifier grammar + // rejects non-ASCII), so `.len()` is the character count. That + // matters because `.chars().count()` walks the whole string + // (O(N_bytes)) and this suggestion path runs for every unknown + // positional call — with a 5000-candidate table, replacing + // `chars().count()` with `.len()` inside the length-diff + // shortcut collapses the fuzzy sweep from minutes to + // milliseconds on the metroid workspace. + let target_len = target.len(); + let threshold = (target_len / 3).clamp(1, 3); + let mut best: Option<(usize, &str)> = None; + for cand in candidates { + // Length-difference lower-bound: levenshtein(a, b) ≥ ||a| - |b||. + // Skip candidates whose length already exceeds the threshold — + // no amount of substitution/insertion can bring the distance + // in range. + let cand_len = cand.len(); + let len_diff = target_len.abs_diff(cand_len); + if len_diff > threshold { + continue; + } + let d = levenshtein_capped(target, cand, threshold); + if d == 0 || d > threshold { + continue; + } + if best.map(|(b, _)| d < b).unwrap_or(true) { + best = Some((d, cand)); + } + } + best.map(|(_, s)| s.to_string()) +} + +/// Length-capped Levenshtein — returns `max+1` (or larger) as soon +/// as the running row minimum exceeds `max`, avoiding the full +/// O(m*n) sweep when the caller only cares whether the distance +/// is ≤ `max`. Every call from `suggest_name` cares about a +/// threshold of at most 3, so this ends most comparisons after a +/// handful of cells — critical when the candidate table has +/// thousands of entries. +fn levenshtein_capped(a: &str, b: &str, max: usize) -> usize { + // Proc-name identifiers are ASCII, so bytes == chars and we can + // skip the `chars().collect::>()` alloc pair that + // otherwise fires for every comparison in the fuzzy sweep. + let a = a.as_bytes(); + let b = b.as_bytes(); + let m = a.len(); + let n = b.len(); + if m == 0 { + return n; + } + if n == 0 { + return m; + } + let mut prev: Vec = (0..=n).collect(); + let mut cur = vec![0usize; n + 1]; + let sentinel = max + 1; + for i in 1..=m { + cur[0] = i; + let mut row_min = cur[0]; + for j in 1..=n { + let sub = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + sub); + if cur[j] < row_min { + row_min = cur[j]; + } + } + if row_min > max { + return sentinel; + } + std::mem::swap(&mut prev, &mut cur); + } + prev[n] +} + +/// Standard Levenshtein edit distance — number of single-character +/// insertions, deletions, or substitutions to turn `a` into `b`. +/// Two-row rolling table; O(n*m) time, O(n) space. +#[allow(dead_code)] +fn levenshtein(a: &str, b: &str) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let m = a.len(); + let n = b.len(); + if m == 0 { + return n; + } + if n == 0 { + return m; + } + let mut prev: Vec = (0..=n).collect(); + let mut cur = vec![0usize; n + 1]; + for i in 1..=m { + cur[0] = i; + for j in 1..=n { + let sub = if a[i - 1] == b[j - 1] { 0 } else { 1 }; + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + sub); + } + std::mem::swap(&mut prev, &mut cur); + } + prev[n] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse; + + fn diags(src: &str) -> Vec { + let parsed = parse(src); + // Parse errors shouldn't be present in these tests; assert + // so that test failures point at the right layer. + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + // Filter out unused-variable warnings AND undefined-variable + // errors. This module tests the arg / type / enum / + // qualified-position validators; fixtures typically declare + // test-only procs with unused args and reference free vars + // to keep the snippets short. The unused / undefined passes + // have their own tests in `unused::tests` / `undefined::tests`. + validate(&parsed.document, src) + .into_iter() + .filter(|d| { + let unused = d.severity == Severity::Warning + && (d.message.starts_with("unused proc arg ") + || d.message.starts_with("unused local ")); + let undefined = d.severity == Severity::Error + && d.message.starts_with("undefined variable "); + !(unused || undefined) + }) + .collect() + } + + fn proc_decl(body: &str, call: &str) -> String { + format!("proc axis_interface {{\n{body}\n}} {{ # body\n}}\n{call}\n") + } + + #[test] + fn happy_path_no_diagnostics() { + let src = proc_decl( + " @default(0) has_tkeep\n @default(8) tdata_num_bytes", + "axis_interface -has_tkeep 1 -tdata_num_bytes 16", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn unknown_arg() { + let src = + proc_decl(" @default(0) has_tkeep", "axis_interface -has_typo 1"); + let d = diags(&src); + assert_eq!(d.len(), 1); + assert!( + d[0].message.contains("undefined argument -has_typo"), + "{:?}", + d + ); + assert!(d[0].message.contains("Possible values are has_tkeep")); + } + + #[test] + fn missing_required() { + let src = proc_decl(" @required width", "axis_interface"); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("missing required"))); + } + + #[test] + fn enum_rejects_unlisted_value() { + let src = proc_decl( + " @enum(1, 2, 4, 8) tdata_num_bytes", + "axis_interface -tdata_num_bytes 3", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("@enum"))); + } + + #[test] + fn enum_accepts_listed_value() { + let src = proc_decl( + " @enum(1, 2, 4, 8) tdata_num_bytes", + "axis_interface -tdata_num_bytes 4", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn range_check() { + let src = + proc_decl(" @range(1, 16) width", "axis_interface -width 32"); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("out of @range"))); + } + + #[test] + fn requires_dependency() { + let src = proc_decl( + " @default(0) has_tuser\n @requires(has_tuser) tuser_width", + "axis_interface -tuser_width 8", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("requires")), "{:?}", d); + } + + #[test] + fn conflicts_dependency() { + let src = proc_decl( + " has_a\n @conflicts(has_a) has_b", + "axis_interface -has_a 1 -has_b 1", + ); + let d = diags(&src); + assert!(d.iter().any(|d| d.message.contains("conflicts"))); + } + + #[test] + fn one_of_requires_exactly_one_alternative() { + // Two args in an @one_of group — neither supplied → error. + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface", + ); + let d = diags(&src); + assert!( + d.iter().any(|m| m.message.contains("exactly one of -a, -b") + && m.message.contains("must be supplied")), + "{:?}", + d + ); + } + + #[test] + fn one_of_satisfied_by_either_alternative() { + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface -a 1", + ); + assert!(diags(&src).is_empty()); + } + + #[test] + fn one_of_rejects_both_alternatives() { + // Both supplied — should be reported once (group rule). + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") @one_of(a) b", + "axis_interface -a 1 -b 2", + ); + let d = diags(&src); + assert!( + d.iter().any(|m| m.message.contains("got -a, -b")), + "{:?}", + d + ); + } + + #[test] + fn one_of_arg_is_not_treated_as_required() { + // An @one_of arg without @default should NOT trigger the + // separate "missing required" error — the group rule + // supersedes individual required-ness. + let src = + proc_decl(" @one_of(b) a\n @one_of(a) b", "axis_interface -a 1"); + let d = diags(&src); + assert!( + d.iter() + .all(|m| !m.message.contains("missing required argument -a") + && !m.message.contains("missing required argument -b")), + "{:?}", + d + ); + } + + #[test] + fn one_of_declarations_are_symmetric() { + // Declaring `@one_of(b)` on `a` alone is enough; we don't need + // the reverse on `b`. + let src = proc_decl( + " @default(\"\") @one_of(b) a\n @default(\"\") b", + "axis_interface", + ); + let d = diags(&src); + let group_errors: Vec<_> = d + .iter() + .filter(|m| { + m.message.contains("exactly one of") + && m.message.contains("must be supplied") + }) + .collect(); + assert_eq!(group_errors.len(), 1, "{:?}", d); + } + + #[test] + fn namespace_eval_proc_validates_at_qualified_name() { + // A proc declared inside `namespace eval project { ... }` + // should be reachable from the validator at its qualified + // name (`project::set_target_language`), so `@enum` + // constraints on its args still catch bad values at call + // sites — exactly like a top-level proc declaration would. + let src = "\ +namespace eval project { + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { } +} +project::set_target_language -proj p -language Klingon +"; + let d = diags(src); + assert!( + d.iter().any(|m| m.message.contains("Klingon") + && m.message.contains("@enum")), + "{:?}", + d + ); + } + + #[test] + fn namespaced_proc_satisfied_by_valid_args() { + let src = "\ +namespace eval project { + proc set_target_language { + proj + @enum(VHDL, Verilog) language + } { } +} +project::set_target_language -proj p -language VHDL +"; + assert!(diags(src).is_empty()); + } + + #[test] + fn nested_namespace_eval_qualifies_recursively() { + let src = "\ +namespace eval outer { + namespace eval inner { + proc foo { @enum(a, b) x } { } + } +} +outer::inner::foo -x bogus +"; + let d = diags(src); + assert!(d.iter().any(|m| m.message.contains("bogus")), "{:?}", d); + } + + #[test] + fn unknown_call_gets_did_you_mean_suggestion() { + // The exact shape that caught the user's typo in metroid: + // a single-char edit-distance miss against a known proc + // should produce a `did you mean ...` suggestion. + let src = "\ +namespace eval port { + proc plumb_if_pin { + name + pin + } { } +} +port::plum_if_pin -name p -pin q +"; + let d = diags(src); + let err = d.iter().find(|m| m.severity == Severity::Error).unwrap(); + assert!( + err.message.contains("did you mean `port::plumb_if_pin`"), + "{}", + err.message + ); + } + + /// Bare positional call to a name that's one character off + /// from a defined proc — same shape as the metroid typo where + /// `generate_dcmacc` silently escaped the validator. The old + /// `uses_keyword || namespaced_match` gate short-circuited to + /// false for zero-arg positional calls; the fuzzy-match gate + /// now catches it. + #[test] + fn bare_positional_typo_of_known_proc_is_flagged() { + let src = "\ +namespace eval ip { + proc generate_dcmac {} unit { } + proc go {} unit { + generate_dcmacc + } +} +"; + let d = diags(src); + let err = d + .iter() + .find(|m| { + m.severity == Severity::Error + && m.message.contains("generate_dcmacc") + }) + .unwrap_or_else(|| panic!("no diagnostic on typo: {d:?}")); + assert!( + err.message.contains("did you mean `ip::generate_dcmac`"), + "{}", + err.message + ); + } + + #[test] + fn unrelated_unknown_call_has_no_suggestion() { + // A name with no near-miss should NOT get a fake suggestion + // tacked on — that's just misleading noise. + let src = "totally_made_up_thing -arg 1\n"; + let d = diags(src); + let err = d.iter().find(|m| m.severity == Severity::Error).unwrap(); + assert!(!err.message.contains("did you mean"), "{}", err.message); + } + + #[test] + fn unknown_keyword_call_is_an_error() { + // No proc declaration in scope, no `extern::` prefix — the + // call uses `-flag` shape so the validator demands the user + // be explicit about the dependency. + let src = "create_project -in_memory 1 -name foo\n"; + let d = diags(src); + assert!( + d.iter().any(|m| m.severity == Severity::Error + && m.message.contains("create_project") + && m.message.contains("extern::")), + "{:?}", + d + ); + } + + #[test] + fn extern_prefixed_call_skips_unknown_check() { + // `extern::` is the user's opt-out: they're calling a raw + // Tcl proc deliberately. No diagnostic even though the + // name isn't in the signature table. + let src = "extern::create_project -name foo\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn positional_unknown_call_is_allowed() { + // No `-flag` args → looks like a positional Tcl builtin + // call (puts, set, etc.). Pass through silently. + let src = "puts hello\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn positional_call_with_namespaced_match_is_flagged() { + // `get_thing X` positional-only would normally pass (raw + // Tcl assumption), but here a `foo::get_thing` is in + // scope — the unqualified form is almost certainly a + // missed namespace prefix, worth flagging with a + // "did you mean" that names the exact match. + let src = "\ +namespace eval foo { + proc get_thing {name} { return $name } +} +puts [get_thing X] +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|e| e.message.contains("undefined proc `get_thing`")) + .collect(); + assert_eq!( + errs.len(), + 1, + "expected one undefined-proc diag, got {d:?}" + ); + assert!( + errs[0].message.contains("foo::get_thing"), + "expected `foo::get_thing` suggestion, got: {}", + errs[0].message, + ); + } + + #[test] + fn positional_call_with_no_namespaced_match_still_passes() { + // No matching namespaced proc → keep the "raw Tcl + // builtin assumption" semantics for positional calls. + // A bare `some_native X` with nothing named `*::some_native` + // in scope stays silent. + let src = "puts [some_native X]\n"; + let d = diags(src); + assert!( + d.iter().all(|e| !e.message.contains("undefined proc")), + "unexpected diag: {d:?}", + ); + } + + #[test] + fn known_tcl_builtin_with_keyword_args_is_allowed() { + // `string match -nocase ...` is a legitimate Tcl-core + // pattern; the allowlist keeps it from triggering the + // unknown-call error. + let src = "string match -nocase pat str\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn primitive_prelude_reprs_are_not_undefined() { + // `list::repr` / `dict::repr` (and the other primitive + // conversion procs) ship via `emit_primitive_prelude`, not + // `src`, so their `-v` calls must NOT trip the unknown-call + // check. Regression guard for the `list::`/`dict::` errors. + let src = "\ +puts [list::repr -v {a b c}] +puts [dict::repr -v {foo 1 bar 2}] +puts [string::repr -v hi] +puts [int::from -v 3] +"; + let d = diags(src); + assert!( + d.iter().all(|e| !e.message.contains("undefined proc")), + "primitive prelude reprs should resolve, got: {d:?}", + ); + } + + #[test] + fn namespace_eval_extern_is_rejected() { + let src = "namespace eval extern { proc foo {} { } }\n"; + let d = diags(src); + assert!( + d.iter() + .any(|m| m.message.contains("reserved namespace name")), + "{:?}", + d + ); + } + + #[test] + fn duplicate_arg_warns() { + let src = proc_decl(" has_a", "axis_interface -has_a 1 -has_a 2"); + let d = diags(&src); + assert!( + d.iter().any(|d| d.message.contains("duplicate argument")), + "{:?}", + d + ); + } + + #[test] + fn dynamic_value_skips_enum_check() { + let src = + proc_decl(" @enum(1, 2, 4) width", "axis_interface -width $x"); + // $x is runtime; we don't statically know it's outside the + // enum, so no enum diagnostic. + let d = diags(&src); + assert!(d.iter().all(|d| !d.message.contains("@enum"))); + } + + #[test] + fn validates_call_inside_proc_body() { + // A bad flag on a call nested in another proc's body should be + // diagnosed, same as at the top level. + let src = "\ +proc if_tport {\n type\n name\n} { }\n\ +proc axis_if {\n kind\n} {\n if_tport -type t -namze m\n}\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("undefined argument -namze")), + "{:?}", + d + ); + } + + #[test] + fn validates_call_inside_command_substitution() { + // The user case: `set cell [create_cpm5 -foo bar]`. The + // validator must descend into `[…]` so the bad flag is caught + // the same way it is at the top level. + let src = "\ +proc create_cpm5 {\n @default(0) name\n} { }\n\ +set cell [create_cpm5 -foo bar]\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("undefined argument -foo")), + "{:?}", + d + ); + } + + #[test] + fn arg_with_no_default_is_implicitly_required() { + // `name` has neither `@default` nor `@required` — calling the + // proc without a value for it should still error. + let src = "\ +proc create_cpm5 {\n name\n} { }\n\ +create_cpm5\n"; + let d = diags(src); + assert!( + d.iter() + .any(|d| d.message.contains("missing required argument -name")), + "{:?}", + d + ); + } + + #[test] + fn implicit_required_satisfied_when_supplied() { + let src = "\ +proc create_cpm5 {\n name\n} { }\n\ +create_cpm5 -name x\n"; + assert!(diags(src).is_empty()); + } + + #[test] + fn extra_signatures_resolve_unknown_calls_from_prior_batches() { + // The REPL session case: a wrapper declared in a prior + // batch is in `extra`; the new batch's bare call to it must + // resolve (no `extern::` error) and its keyword args must + // validate against the prior signature. + let prior_src = "\ +namespace eval vivado { + proc create_project { + @default(\"\") name + @enum(0, 1) @default(0) in_memory + } { } +} +"; + let prior_parsed = parse(prior_src); + assert!(prior_parsed.errors.is_empty()); + let mut sink = Vec::new(); + let prior_table = + build_signature_table(&prior_parsed.document, &mut sink); + + // New batch: bare `vivado::create_project -name foo`. No + // declaration in scope here — only the prior batch's table + // saves it from the unknown-keyword-call error. + let new_src = "vivado::create_project -name foo\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_table, + ); + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "{:?}", + diags + ); + + // And the keyword-args still get validated — a bad enum + // value should still error even though the sig came in + // through `extra`. + let bad_src = "vivado::create_project -in_memory bogus\n"; + let bad_parsed = parse(bad_src); + let bad_diags = validate_with_signatures( + &bad_parsed.document, + bad_src, + &prior_table, + ); + assert!( + bad_diags + .iter() + .any(|d| d.message.contains("bogus") + && d.message.contains("@enum")), + "{:?}", + bad_diags + ); + } + + #[test] + fn doc_signatures_shadow_extra_without_warning() { + // Re-declaring a proc in the new batch should NOT raise the + // "duplicate definition" warning against the prior-batch + // signature — that's a normal `src @lib` reload case in the + // REPL and would be noisy. The new declaration takes + // precedence. + let prior_src = "proc foo { @default(0) x } { }\n"; + let prior_parsed = parse(prior_src); + let mut sink = Vec::new(); + let prior_table = + build_signature_table(&prior_parsed.document, &mut sink); + + let new_src = "proc foo { @default(1) y } { }\nfoo -y 2\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_table, + ); + assert!( + diags.iter().all(|d| !d.message.contains("duplicate")), + "{:?}", + diags + ); + // And the new sig is the one that resolved: `-y` is + // accepted, `-x` would have been the prior sig's arg. + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "{:?}", + diags + ); + } + + #[test] + fn unknown_positional_call_is_not_validated() { + // Bare positional call to an unknown name (could be a Tcl + // builtin) is silently accepted. Unknown calls with + // `-flag` args are the *only* unknown-call case that + // errors — see `unknown_keyword_call_is_an_error`. + let src = "axis_interface tkeep_yes 1\n"; + assert!(diags(src).is_empty()); + } + + // --- type-decl triplet enforcement (step 1b) ---------------- + + /// Build a valid type+triplet block — bd_cell with all three + /// procs present so the validator should accept it. + fn full_triplet_src() -> &'static str { + "type bd_cell = string\n\ + proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n" + } + + #[test] + fn type_decl_with_full_triplet_passes() { + let src = full_triplet_src(); + let d = diags(src); + assert!( + d.iter().all(|d| !d.message.contains("missing required")), + "unexpected diagnostics: {:?}", + d + ); + } + + #[test] + fn type_decl_missing_repr_emits_diagnostic() { + let src = "type bd_cell = string\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("missing required")) + .expect("expected diagnostic"); + assert!(hit.message.contains("bd_cell::repr"), "{:?}", hit); + assert_eq!(hit.severity, Severity::Error); + } + + #[test] + fn type_decl_missing_all_three_lists_each() { + let src = "type widget = string\n"; + let d = diags(src); + // Now each missing slot emits its own diagnostic, so we + // assert each one shows up separately. + let missing: Vec<&str> = d + .iter() + .filter(|d| d.message.contains("missing required proc")) + .map(|d| d.message.as_str()) + .collect(); + assert!(missing.iter().any(|m| m.contains("widget::repr"))); + assert!(missing.iter().any(|m| m.contains("widget::from"))); + assert!(missing.iter().any(|m| m.contains("widget::to"))); + } + + #[test] + fn type_decl_wrong_arg_type_emits_diagnostic() { + // Annotate the v arg with the wrong type and expect a + // shape-mismatch diagnostic. + let src = "type widget = string\n\ + proc widget::repr {v: int} string { return $v }\n\ + proc widget::from {v: string} widget { return $v }\n\ + proc widget::to {v: widget} string { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("widget::repr")) + .expect("expected mismatch diagnostic"); + assert!( + hit.message.contains("`int`") || hit.message.contains("int"), + "{:?}", + hit + ); + assert!(hit.message.contains("widget"), "{:?}", hit); + } + + #[test] + fn type_decl_wrong_return_type_emits_diagnostic() { + let src = "type widget = string\n\ + proc widget::repr {v: widget} int { return 0 }\n\ + proc widget::from {v: string} widget { return $v }\n\ + proc widget::to {v: widget} string { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("returns")) + .expect("expected return-type mismatch diagnostic"); + assert!(hit.message.contains("widget::repr"), "{:?}", hit); + assert!(hit.message.contains("string"), "{:?}", hit); + } + + #[test] + fn type_decl_unannotated_triplet_still_passes() { + // Existence-only check stays a fallback when the user + // hasn't annotated the procs yet. + let src = "type widget = string\n\ + proc widget::repr {v} { return $v }\n\ + proc widget::from {v} { return $v }\n\ + proc widget::to {v} { return $v }\n"; + let d = diags(src); + assert!( + d.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + d + ); + } + + #[test] + fn type_decl_in_namespace_qualifies() { + let src = "namespace eval x {\n\ + type widget = string\n\ + }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| d.message.contains("missing required")) + .expect("expected diagnostic"); + // The namespace-qualified name should appear in the message. + assert!(hit.message.contains("x::widget"), "{:?}", hit); + assert!(hit.message.contains("x::widget::repr")); + } + + #[test] + fn prior_batch_procs_satisfy_current_batch_type_decl() { + // Batch 1: just declares the procs. + let prior_src = "proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let prior = parse(prior_src); + let mut prior_diags = Vec::new(); + let prior_sigs = + build_signature_table(&prior.document, &mut prior_diags); + // Batch 2: declares the type — should NOT complain because + // the procs live in the prior batch's signature table. + let new_src = "type bd_cell = string\n"; + let new_parsed = parse(new_src); + let diags = validate_with_signatures( + &new_parsed.document, + new_src, + &prior_sigs, + ); + assert!( + diags + .iter() + .all(|d| !d.message.contains("missing required")), + "got: {:?}", + diags + ); + } + + #[test] + fn prior_batch_type_decl_does_not_re_trigger_in_current_batch() { + // Batch 1: declares the type, no procs yet (would error + // in isolation). + let prior_src = "type bd_cell = string\n"; + let prior = parse(prior_src); + let mut prior_diags = Vec::new(); + let prior_types = + build_type_decl_table(&prior.document, &mut prior_diags); + // Batch 2: adds the procs. The type is in `extra_types`, + // and the procs are in batch 2's signature table. Putting + // them together via validate_with_extras should pass. + let new_src = "proc bd_cell::repr {v} { return $v }\n\ + proc bd_cell::from {v} { return $v }\n\ + proc bd_cell::to {v} { return $v }\n"; + let new_parsed = parse(new_src); + let empty_sigs: HashMap = HashMap::new(); + let d = validate_with_extras( + &new_parsed.document, + new_src, + &empty_sigs, + &prior_types, + ); + assert!( + d.iter().all(|d| !d.message.contains("missing required")), + "got: {:?}", + d + ); + } + + // --- enum + overload classifier (step 3) ---------------------- + + #[test] + fn enum_decl_with_unique_variants_passes() { + let src = "enum Direction = {\n North\n South\n East\n West\n}\n"; + let d = diags(src); + assert!(d.is_empty(), "got: {:?}", d); + } + + #[test] + fn enum_decl_with_duplicate_variants_errors() { + let src = "enum Bad = {\n A: int\n B: string\n A: bool\n}\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| { + d.severity == Severity::Error + && d.message.contains("variant `A`") + }) + .expect("expected duplicate-variant diagnostic"); + assert!(hit.message.contains("more than once"), "{:?}", hit); + } + + #[test] + fn overload_set_with_exhaustive_arms_classifies() { + let src = "\ +enum Property = {\n Scalar: string\n Nested: int\n}\n\ +proc handle {v: Property::Scalar} { return $v }\n\ +proc handle {v: Property::Nested} { return $v }\n"; + let parsed = parse(src); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let mut diags = Vec::new(); + let (sig_table, overloads) = build_signature_table_with_overloads( + &parsed.document, + &std::collections::HashSet::new(), + &mut diags, + ); + assert!( + diags.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + diags + ); + let info = overloads.get("handle").expect("overload info for `handle`"); + assert_eq!(info.enum_name, "Property"); + assert_eq!(info.variants.len(), 2); + let names: Vec<&str> = info + .variants + .iter() + .map(|v| v.variant_name.as_str()) + .collect(); + assert!(names.contains(&"Scalar")); + assert!(names.contains(&"Nested")); + // Public-name entry exists in the sig table. + assert!(sig_table.contains_key("handle")); + // Specializations also register under mangled names so + // analyzer drill-down works. + assert!(sig_table.contains_key("__handle__Scalar")); + assert!(sig_table.contains_key("__handle__Nested")); + } + + #[test] + fn ad_hoc_overload_emits_hard_error() { + let src = "\ +proc foo {v: int} { return $v }\n\ +proc foo {v: string} { return $v }\n"; + let d = diags(src); + let hit = d + .iter() + .find(|d| { + d.severity == Severity::Error + && d.message.contains("ad-hoc overloading") + }) + .expect("expected ad-hoc-overloading diagnostic"); + assert!(hit.message.contains("foo"), "{:?}", hit); + } + + #[test] + fn overload_with_mismatched_enums_errors() { + let src = "\ +enum A = {\n X\n Y\n}\n\ +enum B = {\n P\n Q\n}\n\ +proc foo {v: A::X} { }\n\ +proc foo {v: B::P} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("mixes enums")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_duplicate_variant_errors() { + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {v: E::A} { }\n\ +proc foo {v: E::A} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("two") + && d.message.contains("E::A")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_extra_args_errors() { + // v1 restricts overloaded procs to exactly one arg (the + // dispatched variant). Extra tail args trip the arity + // check. + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {\n v: E::A\n x\n} { }\n\ +proc foo {\n v: E::B\n y\n} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("exactly ONE arg")), + "got: {:?}", + d + ); + } + + #[test] + fn overload_with_mismatched_return_type_errors() { + let src = "\ +enum E = {\n A: int\n B: int\n}\n\ +proc foo {v: E::A} int { return 0 }\n\ +proc foo {v: E::B} string { return \"\" }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("return type")), + "got: {:?}", + d + ); + } + + #[test] + fn reserved_prefix_user_proc_errors() { + let src = "proc __foo {v} { return $v }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("reserved") + && d.message.contains("__")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_in_return_position_errors() { + let src = "\ +enum E = {\n A\n}\n\ +proc bad {} E::A { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified") + && d.message.contains("only legal")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_in_tail_arg_errors() { + let src = "\ +enum E = {\n A\n B\n}\n\ +proc bad {\n v: E::A\n x: E::B\n} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified")), + "got: {:?}", + d + ); + } + + #[test] + fn qualified_type_inside_generic_errors() { + let src = "\ +enum E = {\n A\n}\n\ +proc bad {x: list} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified")), + "got: {:?}", + d + ); + } + + /// A qualified name that resolves to a declared newtype + /// (`dcmac::GtChProps`) is legal wherever a Named type is + /// legal — including return-type slots on the newtype's own + /// `from`/`empty` helpers. + #[test] + fn namespaced_newtype_return_type_allowed() { + let src = "\ +namespace eval dcmac {}\n\ +namespace eval dcmac::T {}\n\ +type dcmac::T = string\n\ +proc dcmac::T::repr {v: dcmac::T} string { return $v }\n\ +proc dcmac::T::from {v: string} dcmac::T { return $v }\n\ +proc dcmac::T::to {v: dcmac::T} string { return $v }\n"; + let d = diags(src); + let errs: Vec<_> = + d.iter().filter(|d| d.severity == Severity::Error).collect(); + assert!(errs.is_empty(), "unexpected errors: {errs:?}"); + } + + /// A namespaced newtype used as a tail-arg type annotation + /// (non-first-arg position, previously ruled out for + /// Qualified) passes when the name resolves to a real newtype. + #[test] + fn namespaced_newtype_tail_arg_allowed() { + // `use` returns via `dcmac::T::to` so the return-type + // check sees a `string`-typed value matching the + // declared `string` return. A raw `return $slot` (leaking + // the newtype out as its underlying) is now correctly + // flagged as a type mismatch — the analyzer wants + // callers to cross the newtype boundary through the + // explicit `T::to` conversion. + let src = "\ +namespace eval dcmac {}\n\ +namespace eval dcmac::T {}\n\ +type dcmac::T = string\n\ +proc dcmac::T::repr {v: dcmac::T} string { return $v }\n\ +proc dcmac::T::from {v: string} dcmac::T { return $v }\n\ +proc dcmac::T::to {v: dcmac::T} string { return $v }\n\ +proc use {name slot: dcmac::T} string { return [dcmac::T::to -v $slot] }\n"; + let d = diags(src); + let errs: Vec<_> = + d.iter().filter(|d| d.severity == Severity::Error).collect(); + assert!(errs.is_empty(), "unexpected errors: {errs:?}"); + } + + /// Unknown qualified names (no matching `type` decl) still + /// get rejected — the disambiguator only clears names that + /// resolve to a real newtype. + #[test] + fn unknown_qualified_still_rejected() { + let src = "proc bad {name} Unknown::Thing { return $name }\n"; + let d = diags(src); + assert!( + d.iter().any(|d| d.severity == Severity::Error + && d.message.contains("qualified") + && d.message.contains("only legal")), + "got: {:?}", + d + ); + } + + #[test] + fn recursive_enum_passes() { + // Inner generic with whitespace needs brace-wrapping at + // the word level — that's the existing type-decl rule. + let src = "\ +enum Property = {\n Scalar: string\n Nested: Properties\n}\n\ +type Properties = {dict}\n\ +proc Properties::repr {v} { return $v }\n\ +proc Properties::from {v} { return $v }\n\ +proc Properties::to {v} { return $v }\n"; + let d = diags(src); + // No errors — Property/Properties cycle is fine (Tcl + // resolves at call time) and the triplet exists for the + // type-decl side. + assert!( + d.iter().all(|d| d.severity != Severity::Error), + "got: {:?}", + d + ); + } + + // ------------------------------------------------------------------ + // Undefined `src @` module check. + // ------------------------------------------------------------------ + + fn src_diags(src: &str, known_deps: &[&str]) -> Vec { + let parsed = crate::parser::parse(src); + assert!( + parsed.errors.is_empty(), + "unexpected parse errors: {:?}", + parsed.errors + ); + let deps: std::collections::HashSet = + known_deps.iter().map(|s| s.to_string()).collect(); + validate_with_all_extras_and_vars( + &parsed.document, + src, + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + &std::collections::HashSet::new(), + &deps, + ) + .into_iter() + .filter(|d| d.message.starts_with("unknown src module")) + .collect() + } + + #[test] + fn src_at_named_unresolved_flagged() { + let src = "src @gtwiz-versal\n"; + let d = src_diags(src, &["vivado-cmd", "cpm5"]); + assert_eq!(d.len(), 1); + assert_eq!(d[0].severity, Severity::Error); + assert!(d[0].message.contains("gtwiz-versal"), "{}", d[0].message); + assert!( + d[0].message.contains("[dependencies.gtwiz-versal]"), + "{}", + d[0].message + ); + // Span should cover the `@gtwiz-versal` text. + let bytes = + &src.as_bytes()[d[0].span.start as usize..d[0].span.end as usize]; + assert_eq!(std::str::from_utf8(bytes).unwrap(), "@gtwiz-versal"); + } + + #[test] + fn src_at_named_resolved_clean() { + let src = "src @vivado-cmd\n"; + let d = src_diags(src, &["vivado-cmd"]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + #[test] + fn src_at_named_multi_flagged() { + let src = "src @foo\nsrc @bar\nsrc @cpm5\n"; + let d = src_diags(src, &["cpm5"]); + assert_eq!(d.len(), 2); + // Distinct spans. + assert_ne!(d[0].span.start, d[1].span.start); + } + + #[test] + fn src_bare_path_not_flagged() { + // Relative path form — never triggers the `@` check + // even when known_deps is empty. Filesystem existence gets + // validated downstream by the loader; the analyzer's + // job here is only the dep-name lookup. + let src = "src ./ports.htcl\n"; + let d = src_diags(src, &[]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + #[test] + fn src_subpath_reported_with_hint() { + // `@foo/sub` — the subpath appears in the diagnostic + // message so the user sees the exact directive that + // failed. Also verifies subpath doesn't confuse the + // classifier. + let src = "src @gtwiz-versal/module\n"; + let d = src_diags(src, &["cpm5"]); + assert_eq!(d.len(), 1); + assert!( + d[0].message.contains("@gtwiz-versal/module"), + "{}", + d[0].message + ); + } + + #[test] + fn empty_deps_no_check() { + // The check must no-op when the caller doesn't know about + // any deps — matches the behavior for unit tests / non- + // workspace-aware callers who invoke `validate` directly. + let src = "src @gtwiz-versal\nsrc @other\n"; + let d = src_diags(src, &[]); + assert!(d.is_empty(), "unexpected diags: {d:?}"); + } + + // ------ call-site type check --------------------------------- + // + // Three fixtures exercise the arg-type check: + // + // 1. Matching newtypes → no diagnostic (happy path). + // 2. Mismatched newtypes (the copy-paste-wrong-constructor + // bug in ip/gtm.htcl) → one diagnostic naming both types. + // 3. Unknown-type value (literal string) → silent skip + // (gradual typing). Sanity-checks that the check doesn't + // over-fire on values we can't infer. + + /// Set up a minimal document with two newtypes and one taker + /// proc that accepts a `-slot` of each. Callers append their + /// own top-level call. + fn typed_slots_fixture(tail: &str) -> String { + format!( + "type ns::TypeA = Properties\n\ + type ns::TypeB = Properties\n\ + proc make_a {{}} ns::TypeA {{ return {{}} }}\n\ + proc make_b {{}} ns::TypeB {{ return {{}} }}\n\ + proc take {{ @default(\"\") slot: ns::TypeA }} {{ }}\n\ + {tail}", + ) + } + + #[test] + fn type_check_matching_types_no_diagnostic() { + // `-slot [make_a]` — expected ns::TypeA, actual ns::TypeA. + let src = typed_slots_fixture("take -slot [make_a]\n"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diags: {d:?}", + ); + } + + #[test] + fn type_check_mismatched_types_errors() { + // `-slot [make_b]` — expected ns::TypeA, actual ns::TypeB. + // This is the class of bug the check exists to catch. + let src = typed_slots_fixture("take -slot [make_b]\n"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 type diag, got {d:?}"); + let msg = &type_errs[0].message; + assert!( + msg.contains("ns::TypeA") && msg.contains("ns::TypeB"), + "message should name both types: {msg}", + ); + assert!(msg.contains("-slot"), "message should name the arg: {msg}",); + } + + #[test] + fn type_check_unknown_value_is_silent() { + // `-slot hello` — value is a plain literal, type unknown. + // The check must NOT fire (gradual typing). + let src = typed_slots_fixture("take -slot hello\n"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on untyped literal: {d:?}", + ); + } + + #[test] + fn type_check_var_binding_flows_through_set() { + // `set x [make_b]; take -slot $x` — actual type flows + // through the `set` binding into the `$x` reference. + let src = typed_slots_fixture("set x [make_b]\ntake -slot $x\n"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!( + type_errs.len(), + 1, + "expected 1 type diag through `set`, got {d:?}", + ); + } + + // ------ bool literal check ---------------------------------- + // + // Sanity check the four legs of the bool-typed slot machinery: + // `true` and `false` land clean, `1` and arbitrary garbage + // both error with a message that names the offending literal + // and the arg. + + fn bool_slot_fixture(value: &str) -> String { + format!( + "proc take {{ @default(false) flag: bool }} {{ }}\n\ + take -flag {value}\n", + ) + } + + #[test] + fn bool_literal_true_no_diagnostic() { + let src = bool_slot_fixture("true"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on `true`: {d:?}", + ); + } + + #[test] + fn bool_literal_false_no_diagnostic() { + let src = bool_slot_fixture("false"); + let d = diags(&src); + assert!( + d.iter().all(|x| !x.message.contains("type mismatch")), + "unexpected type diag on `false`: {d:?}", + ); + } + + #[test] + fn bool_literal_integer_1_errors() { + // The specific class of bug this pass exists to catch — + // `-enable_reg_interface 1` accepted silently today. + let src = bool_slot_fixture("1"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 diag, got {d:?}"); + let msg = &type_errs[0].message; + assert!(msg.contains("bool"), "message names type: {msg}"); + assert!(msg.contains("1"), "message names literal: {msg}"); + assert!(msg.contains("-flag"), "message names arg: {msg}"); + } + + #[test] + fn bool_literal_arbitrary_string_errors() { + // Guards against `potato` / `yes` / `on` sliding through + // as "Tcl also accepts this as truthy" — HTCL's bool + // surface is exactly `true` / `false`. + let src = bool_slot_fixture("potato"); + let d = diags(&src); + let type_errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("type mismatch")) + .collect(); + assert_eq!(type_errs.len(), 1, "expected 1 diag, got {d:?}"); + assert!( + type_errs[0].message.contains("potato"), + "message names offending literal: {}", + type_errs[0].message, + ); + } + + // ------ return-type check ------------------------------------ + // + // Annotated procs must produce a value whose type matches the + // declared return type on every `return X` in the body — + // including returns buried inside `if`/`while`/etc bodies. + + #[test] + fn return_type_matching_annotation_no_diag() { + let src = "\ +type ns::TypeA = Properties +proc make_a {} ns::TypeA { return {} } +proc use_a {} ns::TypeA { + set x [make_a] + return $x +} +"; + let d = diags(src); + assert!( + d.iter() + .all(|x| !x.message.contains("return type mismatch")), + "unexpected diags: {d:?}", + ); + } + + #[test] + fn return_type_mismatch_errors() { + // Body returns `ns::TypeA` but annotation says `ns::TypeB`. + // This is the shape the user hit with `configure_gtm` + // wrongly annotated `cpm5::Config`. + let src = "\ +type ns::TypeA = Properties +type ns::TypeB = Properties +proc make_a {} ns::TypeA { return {} } +proc mismatched {} ns::TypeB { + set x [make_a] + return $x +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("return type mismatch")) + .collect(); + assert_eq!(errs.len(), 1, "expected 1 diag, got {d:?}"); + let msg = &errs[0].message; + assert!( + msg.contains("ns::TypeA") && msg.contains("ns::TypeB"), + "message names both types: {msg}", + ); + } + + #[test] + fn return_type_check_descends_into_if_body() { + // Wrong-typed return buried inside an `if` body — the + // walker parses the braced body and finds the return. + let src = "\ +type ns::TypeA = Properties +type ns::TypeB = Properties +proc make_a {} ns::TypeA { return {} } +proc branchy {} ns::TypeB { + if 1 { + set x [make_a] + return $x + } + return {} +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("return type mismatch")) + .collect(); + assert!(!errs.is_empty(), "expected mismatch inside if, got {d:?}"); + } + + #[test] + fn baseline_arg_is_optional_and_does_not_warn_when_matched() { + // `@baseline(X)` behaves like `@default(X)` for optionality + // (the arg is NOT required), but explicitly passing X does + // NOT fire the redundant-default warning. This is the + // preset-shifted-baseline case: omitting the flag doesn't + // pin the value to X, so an explicit `-flag X` is + // meaningful, not redundant. + let src = "\ +proc use_it { @baseline(\"156.25\") freq } unit { puts $freq } +use_it -freq \"156.25\" +use_it +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("redundant")), + "baseline arg should not trigger redundant-default warn: {d:?}", + ); + assert!( + d.iter() + .all(|x| !x.message.contains("missing required argument")), + "baseline arg should be optional (like @default): {d:?}", + ); + } + + #[test] + fn return_of_bare_identifier_matching_var_errors() { + // The metroid `configure_cpm5` bug: `set cpm5_cfg [...]` + // followed by `return cpm5_cfg` (missing `$`). Without the + // sigil this returns the literal string "cpm5_cfg" and + // silently subverts the annotation. The `set` must have a + // typed RHS so `cpm5_cfg` lands in the var table — an + // untyped literal (`set x 1`) wouldn't (see + // `value_type_with_procs`); the actual code uses a + // `cpm5::configure` call whose return type IS known. + let src = "\ +type cpm5::Config = Properties +proc make_config {} cpm5::Config { return {} } +proc configure_cpm5 {} cpm5::Config { + set cpm5_cfg [make_config] + return cpm5_cfg +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("bare identifier")) + .collect(); + assert_eq!(errs.len(), 1, "expected missing-sigil diag, got {d:?}"); + assert!( + errs[0].message.contains("did you mean `return $cpm5_cfg`"), + "message names the intended var: {}", + errs[0].message, + ); + } + + #[test] + fn return_of_bare_true_false_ok() { + // `return true` / `return false` are the canonical bool + // literals — the bareword check must NOT fire on them even + // if a var of the same name were in scope. + let src = "\ +proc is_ok {} bool { return true } +proc is_bad {} bool { return false } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("bare identifier")), + "unexpected bare-identifier diag on bool literal: {d:?}", + ); + } + + #[test] + fn return_of_int_literal_ok() { + // `return 42` at `int` — bare integer literal, not a + // missing-sigil case. + let src = "\ +proc answer {} int { return 42 } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("bare identifier")), + "unexpected bare-identifier diag on int literal: {d:?}", + ); + } + + #[test] + fn return_bareword_at_qualified_type_errors() { + // Bareword `return foo` at a qualified return type — even + // when `foo` isn't a var in scope, a qualified newtype + // can't come from a bare text literal. + let src = "\ +type cpm5::Config = Properties +proc bad {} cpm5::Config { return not_a_var_anywhere } +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("return type mismatch")) + .collect(); + assert_eq!(errs.len(), 1, "expected mismatch diag, got {d:?}"); + assert!( + errs[0].message.contains("cpm5::Config") + && errs[0].message.contains("not_a_var_anywhere"), + "message names type and literal: {}", + errs[0].message, + ); + } + + #[test] + fn redundant_default_string_arg_warns() { + // `-freq "33.333"` when the arg's `@default("33.333")` — + // the flag is a no-op. Fires with a "redundant" warning. + let src = "\ +proc use_clock { @default(\"33.333\") freq } unit { puts $freq } +use_clock -freq \"33.333\" +"; + let d = diags(src); + let warns: Vec<_> = d + .iter() + .filter(|x| x.message.contains("redundant")) + .collect(); + assert_eq!( + warns.len(), + 1, + "expected 1 redundant-default warn, got {d:?}" + ); + assert_eq!(warns[0].severity, Severity::Warning); + } + + #[test] + fn redundant_default_bare_matches_quoted_default() { + // Same test, but the caller writes the value bare + // (`-freq 33.333`) while the default is quoted + // (`@default("33.333")`). literal_value normalizes both. + let src = "\ +proc use_clock { @default(\"33.333\") freq } unit { puts $freq } +use_clock -freq 33.333 +"; + assert_eq!( + diags(src) + .iter() + .filter(|x| x.message.contains("redundant")) + .count(), + 1, + ); + } + + #[test] + fn redundant_default_int_arg_warns() { + // `-count 0` when default is `@default(0)` — integer + // path; both sides normalize through `i64::parse`. + let src = "\ +proc count_up { @default(0) count } unit { puts $count } +count_up -count 0 +"; + assert_eq!( + diags(src) + .iter() + .filter(|x| x.message.contains("redundant")) + .count(), + 1, + ); + } + + #[test] + fn non_default_value_does_not_warn() { + // `-freq 100` when default is `33.333` — different value, + // no warning. + let src = "\ +proc use_clock { @default(\"33.333\") freq } unit { puts $freq } +use_clock -freq 100 +"; + assert!(!diags(src).iter().any(|x| x.message.contains("redundant")),); + } + + #[test] + fn empty_default_is_sentinel_no_warn() { + // `@default("")` is the "unset sentinel" idiom in the + // generated wrappers. Explicitly passing `""` still flips + // the `__vw_kw_..._set` guard, so it isn't strictly + // redundant — no warning. + let src = "\ +proc configure { @default(\"\") config } unit { puts $config } +configure -config \"\" +"; + assert!(!diags(src).iter().any(|x| x.message.contains("redundant")),); + } + + #[test] + fn interpolated_value_does_not_warn() { + // Runtime-dynamic values (`$var`, `[cmd]`) can't be + // compared to a literal default at analysis time — skip + // silently. + let src = "\ +proc use_clock { @default(\"33.333\") freq } unit { puts $freq } +set x 33.333 +use_clock -freq $x +"; + assert!(!diags(src).iter().any(|x| x.message.contains("redundant")),); + } + + #[test] + fn bare_return_in_annotated_proc_errors() { + let src = "\ +type ns::TypeA = Properties +proc bad {} ns::TypeA { + return +} +"; + let d = diags(src); + let errs: Vec<_> = d + .iter() + .filter(|x| x.message.contains("bare `return`")) + .collect(); + assert_eq!(errs.len(), 1, "expected bare-return diag, got {d:?}"); + } + + #[test] + fn bare_return_in_unit_proc_is_ok() { + // `unit` return type = "no meaningful value"; bare + // `return` is idiomatic for side-effecting procs that + // early-out on a condition. + let src = "\ +proc side_effect {x} unit { + if $x { return } + return +} +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("bare `return`")), + "unexpected bare-return diag: {d:?}", + ); + } + + #[test] + fn unannotated_proc_return_with_value_errors() { + // No return annotation + `return X` → an error. The proc's + // shape declares "side effects only," and a value-carrying + // return contradicts that. + let src = "\ +proc anything {} { return 42 } +"; + let d = diags(src); + assert!( + d.iter() + .any(|x| x.message.contains("no declared return type")), + "expected diag, got: {d:?}", + ); + } + + #[test] + fn unannotated_proc_bare_return_ok() { + // Bare `return` is fine in a side-effect proc — common + // early-exit pattern. + let src = "\ +proc anything {} { puts hi; return } +"; + let d = diags(src); + assert!( + d.iter() + .all(|x| !x.message.contains("no declared return type")), + "unexpected diag, got: {d:?}", + ); + } + + #[test] + fn unannotated_proc_return_value_inside_if_errors() { + // A value-return buried in a control-flow branch still fires. + let src = "\ +proc anything { x: int } { if {$x > 0} { return 42 } } +"; + let d = diags(src); + assert!( + d.iter() + .any(|x| x.message.contains("no declared return type")), + "expected diag, got: {d:?}", + ); + } + + // ─── must-return / fallthrough analysis ───────────────────── + + fn has_fallthrough_diag(d: &[Diagnostic]) -> bool { + d.iter().any(|x| x.message.contains("may fall through")) + } + + #[test] + fn empty_body_annotated_proc_errors() { + // The user's `configure_txr1` case: annotated with a + // real type but the body is empty. Must-return should + // flag it. + let src = "\ +type MyType = string +proc configure_txr1 {} MyType { } +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn single_puts_body_annotated_proc_errors() { + // Body has a side-effecting `puts` and no return; the + // last statement's result isn't a MyType. + let src = "\ +type MyType = string +proc foo {} MyType { + puts hello +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn if_no_else_annotated_proc_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + if 1 { + return {} + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn while_body_return_annotated_proc_errors() { + // Even with a `return` inside the loop body, the loop + // may not execute — must-return still fires. + let src = "\ +type MyType = string +proc bad {} MyType { + while 1 { + return {} + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn if_else_both_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + if 1 { + return {} + } else { + return {} + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn if_elseif_else_all_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + if 1 { + return {} + } elseif 2 { + return {} + } else { + return {} + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn implicit_last_expression_typed_proc_call_no_error() { + // Trailing typed proc-call is an implicit return in Tcl. + let src = "\ +type MyType = string +proc make_it {} MyType { return {} } +proc user {} MyType { + make_it +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn implicit_last_expression_extern_call_no_error() { + // Trailing extern call is the user's opt-out for raw + // Tcl. Trust it as a valid implicit return. + let src = "\ +type MyType = string +proc user {} MyType { + extern::some_raw_tcl_proc -foo bar +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn switch_with_default_all_return_no_error() { + let src = "\ +type MyType = string +proc ok {} MyType { + switch $x { + a { return {} } + default { return {} } + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn switch_no_default_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + switch $x { + a { return {} } + b { return {} } + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn switch_default_falls_through_errors() { + let src = "\ +type MyType = string +proc bad {} MyType { + switch $x { + a { return {} } + default { puts hi } + } +} +"; + let d = diags(src); + assert!(has_fallthrough_diag(&d), "diags: {d:?}"); + } + + #[test] + fn try_body_and_handler_terminate_no_error() { + // Matches the vw-ip generator's wrap pattern: + // `try { return X } on error { error "…" }`. + let src = "\ +type MyType = string +proc gen {} MyType { + try { + return {} + } on error {msg} { + error \"foo.$msg\" + } +} +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn unit_annotated_empty_body_no_error() { + // `unit` return type doesn't require a value. + let src = "\ +proc side_effect {} unit { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn newtype_triplet_empty_body_no_error() { + // T::from and friends are exempted before the must-return + // check fires. + let src = "\ +type ns::T = string +proc ns::T::from {v: string} ns::T { return $v } +proc ns::T::empty {} ns::T { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + #[test] + fn enum_overload_arm_empty_body_no_error() { + // First-arg-Qualified shape (overload arm) is exempted + // before the must-return check fires. + let src = "\ +enum E = { + A: string + B: string +} +proc handle {v: E::A} string { } +"; + let d = diags(src); + assert!(!has_fallthrough_diag(&d), "unexpected diags: {d:?}"); + } + + // ─── @test attribute semantic checks ─────────────────────── + + #[test] + fn test_attribute_dedicated_eda_ok() { + let src = "@test(dedicated-eda)\nproc t {} { }\n"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("dedicated-eda")), + "unexpected diag: {d:?}", + ); + } + + #[test] + fn test_attribute_wrong_positional_value_warns() { + let src = "@test(bogus)\nproc t {} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected `dedicated-eda`-related warning: {d:?}", + ); + } + + #[test] + fn test_attribute_on_proc_with_args_warns() { + // Zero-arg only for MVP. + let src = "@test\nproc t {x: int} { }\n"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("zero arguments")), + "expected zero-arg warning: {d:?}", + ); + } + + #[test] + fn test_attribute_on_nested_proc_warns() { + // Only top-level @test procs are runnable. + let src = "\ +proc outer {} { + @test + proc inner {} { puts hi } +} +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("nested proc")), + "expected nested-proc warning: {d:?}", + ); + } + + // ─── @test(target=…) keyed-item checks ───────────────────── + + #[test] + fn test_attribute_with_dedicated_eda_and_target_ok() { + let src = "\ +@test(dedicated-eda target=\"xcvm3358-vsvh1747-2M-e-S\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("`@test") + && !x.message.contains("target=")), + "unexpected @test diag: {d:?}", + ); + } + + #[test] + fn test_target_without_dedicated_eda_warns() { + // Shared bucket can't honor per-test parts. + let src = "\ +@test(target=\"xcvm3358-vsvh1747-2M-e-S\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected dedicated-eda requirement warning: {d:?}", + ); + } + + #[test] + fn test_attribute_unknown_key_warns() { + let src = "\ +@test(dedicated-eda family=versal) +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("unrecognized key")), + "expected unrecognized-key warning: {d:?}", + ); + } + + #[test] + fn test_attribute_variant_with_dedicated_eda_ok() { + let src = "\ +@test(dedicated-eda variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().all(|x| !x.message.contains("`@test")), + "unexpected @test diag: {d:?}", + ); + } + + #[test] + fn test_variant_without_dedicated_eda_warns() { + let src = "\ +@test(variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("dedicated-eda")), + "expected dedicated-eda requirement warning: {d:?}", + ); + } + + #[test] + fn test_target_and_variant_together_warns() { + // Mutually exclusive within `@test` — variants own their + // parts, so specifying both is a config bug. + let src = "\ +@test(dedicated-eda target=\"xcv...\" variant=\"vpk120\") +proc t {} { } +"; + let d = diags(src); + assert!( + d.iter().any(|x| x.severity == Severity::Warning + && x.message.contains("pick one of")), + "expected pick-one-of warning: {d:?}", + ); + } +} diff --git a/vw-ip/Cargo.toml b/vw-ip/Cargo.toml new file mode 100644 index 0000000..9846a73 --- /dev/null +++ b/vw-ip/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vw-ip" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +ipxact.workspace = true +vw-htcl = { path = "../vw-htcl" } +vw-quote = { path = "../vw-quote" } +thiserror.workspace = true +serde.workspace = true +quick-xml = { version = "0.37", features = ["serialize"] } +regex.workspace = true +toml.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-ip/src/cips_dict.rs b/vw-ip/src/cips_dict.rs new file mode 100644 index 0000000..b132e23 --- /dev/null +++ b/vw-ip/src/cips_dict.rs @@ -0,0 +1,1011 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Schema loader for Xilinx's `structured_tcldict` IP-XACT parameters. +//! +//! Some IP-XACT parameters in CIPS-family components are declared with +//! `structured_tcldict`: +//! the IP-XACT value is just an opaque space-separated `KEY VAL …` +//! dict string. The real schema for those inner fields lives in +//! out-of-band data files Vivado ships: +//! +//! - `versal/flows/automation/cipsToPsWiz_Porting/csv_files/` +//! - `param_mapping_direct.csv` — `(KEY, {DEFAULT}, …)` per row. +//! - `param_mapping_presets.csv` — preset-bundle layout for +//! `mode`-style selector fields like `CLOCK_MODE`, `BOOT_MODE`. +//! - `versal/cips_hip//guidata/ParamInfo.xml` — per-field +//! `` text, used as a doc comment. +//! - `versal/cips_hip//global/global_preset*.xml` and +//! `versal/cips_hip//presets/**/*.xml` — `` entries used to widen `@enum(…)` lists, same format +//! already parsed by [`crate::presets`]. +//! +//! We deliberately ignore the deprecated +//! `flows/automation/deprecated/cips_pswiz_key_and_value.csv` — its +//! content is a subset of the two supported CSVs above. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::overrides::OverridesFile; +use crate::paired_list::{parse_paired_list, PairedValue}; + +#[derive(Debug, Clone, Default)] +pub struct DictSchema { + /// Scalar fields at this level — each becomes a typed + /// `@default(…) name` proc arg in the emitted constructor. + pub fields: Vec, + /// Sub-schemas for nested paired-list slots at this level. Keyed + /// by the same upper-snake XML name the field would carry; the + /// emitter picks a nested-namespace proc name from that key. + /// `LR0_SETTINGS` → sub-schema whose `fields` are `RX_HD_EN`, + /// `TX_HD_EN`, etc., emitted as + /// `gtwiz_versal::intf::gt_settings::lr0_settings`. + pub sub_schemas: BTreeMap, +} + +#[derive(Debug, Clone, Default)] +pub struct DictField { + /// IP-XACT-style upper-snake name, e.g. `PCIE_APERTURES_DUAL_ENABLE`. + pub name: String, + /// Default value as recorded in the supporting data files. May be + /// empty when no default is known (rare); the generator treats it + /// the same as any other defaultless arg. + pub default: String, + /// Display-name / one-line description from `ParamInfo.xml`, when + /// present. + pub description: Option, + /// `@enum(…)` choices we were able to recover from preset files + /// or from `param_mapping_presets.csv`. Empty when no enum data + /// was found. + pub enum_values: BTreeSet, + /// When true, the generator emits `@baseline()` instead + /// of `@default()` for this field. Set via + /// `overrides.toml` (`baseline = true`) for scalar knobs whose + /// value can be shifted out from under a caller by a sibling + /// `-preset` / `-board_interface` — the runtime wrapper is + /// unchanged (still `if kw_set { dict set }`), but the annotation + /// documents the value as a baseline, and the redundant-default + /// lint stops flagging call-site values that match. See + /// [`crate::overrides::FieldOverride::baseline`] for the full + /// rationale. + pub baseline: bool, +} + +impl DictSchema { + /// Build a nested `DictSchema` from a paired-list default value. + /// + /// This is the entry point for typed-constructor emission on + /// Properties-shaped params that AREN'T sourced from Xilinx's + /// `structured_tcldict` CSVs — gtwiz-versal's + /// `INTF*_TXRX_OPTIONAL_PORTS`, `INTF*_CHANNEL_MAP`, etc. The + /// paired-list parser turns the XML default into a tree of + /// `PairedValue::Scalar` (leaves) and `PairedValue::Nested` + /// (sub-slots); this walks that tree, applying `overrides` at + /// each level to attach `@enum(...)` restrictions and default + /// refinements where the XML is silent. + /// + /// `shape_path` is the `::`-separated ident chain the caller + /// will use to look up overrides — e.g. `"intf::gt_settings"` + /// for the top-level `gt_settings` slot on the `intf` family. + /// Sub-schemas recurse with the current field name appended + /// (`"intf::gt_settings::lr0_settings"` for the LR0 slot). + pub fn from_paired_default( + default: &str, + shape_path: &str, + overrides: &OverridesFile, + ) -> Self { + let pairs = parse_paired_list(default); + Self::from_pairs(&pairs, shape_path, overrides) + } + + /// Walk `self`'s fields and re-apply `overrides` at the given + /// `shape_path`. Used when a schema is COPIED from an anchor + /// param to a different slot — the anchor's fields were built + /// with the anchor's shape path, but at the copy site the same + /// fields are surfaced under a different shape path that may + /// have its own overrides. Idempotent: a field with no matching + /// override in either place stays unchanged. + pub fn reapply_overrides( + &mut self, + shape_path: &str, + overrides: &OverridesFile, + ) { + // Shape-wide baseline switch: when set, EVERY field in the + // shape is treated as baseline regardless of per-field + // config. Cheap to check once per shape. + let shape_baseline = overrides.shape_baseline(shape_path); + for f in &mut self.fields { + if shape_baseline { + f.baseline = true; + } + let field_lookup = lower(&f.name); + let Some(fo) = overrides.field(shape_path, &field_lookup) else { + continue; + }; + if let Some(default) = &fo.default { + f.default = default.clone(); + } + if let Some(enum_values) = &fo.enum_values { + f.enum_values = enum_values.iter().cloned().collect(); + } + if fo.baseline { + f.baseline = true; + } + } + for (sub_name, sub) in &mut self.sub_schemas { + let sub_path = format!("{shape_path}::{}", lower(sub_name)); + sub.reapply_overrides(&sub_path, overrides); + } + } + + fn from_pairs( + pairs: &[(String, PairedValue)], + shape_path: &str, + overrides: &OverridesFile, + ) -> Self { + // Shape-wide baseline switch — same semantic as + // `reapply_overrides`: when true, every field the pairs + // loop constructs gets `baseline = true` without needing a + // per-field opt-in. + let shape_baseline = overrides.shape_baseline(shape_path); + let mut fields = Vec::new(); + let mut sub_schemas = BTreeMap::new(); + for (raw_name, value) in pairs { + match value { + PairedValue::Nested(inner) => { + // Nested slot — recurse. Sub-shape path extends + // the current path with the lowercase form of + // this key, matching the convention the emitter + // uses when writing the sub-proc's namespace + // segment. + let sub_path = if shape_path.is_empty() { + lower(raw_name) + } else { + format!("{shape_path}::{}", lower(raw_name)) + }; + let sub = Self::from_pairs(inner, &sub_path, overrides); + sub_schemas.insert(raw_name.clone(), sub); + } + PairedValue::Scalar(default) => { + // Merge overrides on top of the XML default. + // Field key inside the override file is the + // lowercase form (matching the emitted arg name); + // the underlying DictField preserves the raw + // upper-snake name for downstream `set_property` + // key composition. + let field_lookup = lower(raw_name); + let field_override = + overrides.field(shape_path, &field_lookup); + let (final_default, enum_values, baseline) = + match field_override { + Some(fo) => { + let d = fo + .default + .clone() + .unwrap_or_else(|| default.clone()); + let e = fo + .enum_values + .clone() + .map(|v| { + v.into_iter().collect::>() + }) + .unwrap_or_default(); + (d, e, fo.baseline) + } + None => (default.clone(), BTreeSet::new(), false), + }; + fields.push(DictField { + name: raw_name.clone(), + default: final_default, + description: None, + enum_values, + baseline: baseline || shape_baseline, + }); + } + } + } + Self { + fields, + sub_schemas, + } + } +} + +/// Lowercase form of a field/slot name used for override lookup and +/// (downstream) as the emitted arg / namespace segment. Uppercase +/// letters map to lowercase; underscores and digits pass through +/// verbatim. Matches `generate::lowercase_ident`'s behavior on +/// upper-snake input (no digit-suffix handling needed here — Xilinx +/// keys don't start with digits). +fn lower(s: &str) -> String { + s.to_ascii_lowercase() +} + +/// Returns the schema for each `structured_tcldict` parameter we can +/// find data for. Keys are the IP-XACT parameter names +/// (`PS_PMC_CONFIG`, …); the matching `_INTERNAL` variants point at +/// the same schema. +/// +/// Empty when the Xilinx `data/` ancestor can't be located. +pub fn load_schemas(component_path: &Path) -> HashMap { + let mut out = HashMap::new(); + let Some(data_root) = find_data_root(component_path) else { + return out; + }; + if let Some(schema) = load_ps_pmc_schema(&data_root) { + out.insert("PS_PMC_CONFIG".to_string(), schema.clone()); + out.insert("PS_PMC_CONFIG_INTERNAL".to_string(), schema); + } + out +} + +/// Inputs scoped to a CIPS `PS_PMC_CONFIG`. +fn load_ps_pmc_schema(data_root: &Path) -> Option { + let pspmc = data_root.join("versal/cips_hip/pspmc"); + let csv_dir = + data_root.join("versal/flows/automation/cipsToPsWiz_Porting/csv_files"); + if !pspmc.is_dir() || !csv_dir.is_dir() { + return None; + } + + let mut fields: HashMap = HashMap::new(); + parse_direct_csv(&csv_dir.join("param_mapping_direct.csv"), &mut fields); + parse_presets_csv(&csv_dir.join("param_mapping_presets.csv"), &mut fields); + + // Drop keys that belong to a different `structured_tcldict`. + fields.retain(|name, _| { + !name.starts_with("CPM_") + && !name.starts_with("XRAM_") + && !is_cips_toplevel(name) + }); + if fields.is_empty() { + return None; + } + + layer_param_info(&pspmc.join("guidata/ParamInfo.xml"), &mut fields); + layer_presets(&pspmc, &mut fields); + + let mut sorted: Vec = fields.into_values().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + Some(DictSchema { + fields: sorted, + // PS_PMC_CONFIG is a flat CSV-driven schema — no nested + // sub-slots. The from_paired_default path is what populates + // sub_schemas for XML-derived schemas. + sub_schemas: BTreeMap::new(), + }) +} + +/// Names of `` entries the CIPS IP-XACT exposes at +/// the top level of the component — the containers Vivado writes as +/// `CONFIG.` in the outer `set_property -dict`. We must NOT +/// re-emit them as inner dict fields of `ps_pmc_config` (the schema +/// this filter runs against), or the emitted wrapper would produce +/// `CONFIG.PS_PMC_CONFIG.PS_PMC_CONFIG` and similar recursion. +/// +/// Only container / component-identity names belong here. Ordinary +/// scalar knobs like `PMC_REF_CLK_FREQMHZ`, `PMC_ALT_REF_CLK_FREQMHZ`, +/// `GT_REFCLK_MHZ`, `AURORA_LINE_RATE_GPBS`, +/// `BOOT_SECONDARY_PCIE_ENABLE` are ALSO settable at the top level in +/// the Vivado GUI, but Xilinx's `param_mapping_direct.csv` groups them +/// under `PS_PMC_CONFIG` too — nesting them in the dict works fine +/// because Vivado accepts both forms, and it's the only path that +/// preserves them (the CIPS `component.xml` doesn't list them +/// individually, so a "recover from IP-XACT" filter would drop them +/// entirely — the bug this comment now guards against). +fn is_cips_toplevel(name: &str) -> bool { + matches!( + name, + "Component_Name" + | "PS_PMC_CONFIG" + | "PS_PMC_CONFIG_INTERNAL" + | "PS_PMC_CONFIG_APPLIED" + | "CPM_CONFIG" + | "CPM_CONFIG_INTERNAL" + | "XRAM_CONFIG" + | "XRAM_CONFIG_INTERNAL" + ) +} + +/// `param_mapping_direct.csv` layout: `,{CIPS_DEFAULT},,{PSWIZ_DEFAULT}`. +/// The `{…}` value cells routinely contain commas (Tcl list syntax), +/// so we tokenize comma-separated columns at brace depth 0 rather than +/// splitting on every comma. Rows whose value has unbalanced braces +/// (the Xilinx CSV does ship a handful of those — line-wrapped or +/// truncated by the vendor) keep the field name but no default. +fn parse_direct_csv(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let text = text.strip_prefix('\u{feff}').unwrap_or(&text); + for line in text.lines() { + let cols = split_brace_aware(line); + let (Some(key), Some(raw_default)) = ( + cols.first().map(|s| s.trim()), + cols.get(1).map(|s| s.trim()), + ) else { + continue; + }; + if key.is_empty() || !is_safe_key(key) { + continue; + } + let stripped = unwrap_one_brace(raw_default); + let default = if braces_balanced(stripped) { + stripped.to_string() + } else { + String::new() + }; + fields.entry(key.to_string()).or_insert_with(|| DictField { + name: key.to_string(), + default, + description: None, + enum_values: BTreeSet::new(), + ..Default::default() + }); + } +} + +/// Split a CSV row on commas at brace depth 0. Treats `{` and `}` as +/// Tcl-style grouping characters so that `KEY,{a,b,c},…` splits into +/// three columns rather than five. +fn split_brace_aware(line: &str) -> Vec<&str> { + let mut out = Vec::new(); + let bytes = line.as_bytes(); + let mut start = 0usize; + let mut depth: i32 = 0; + for (i, b) in bytes.iter().enumerate() { + match b { + b'{' => depth += 1, + b'}' => depth -= 1, + b',' if depth == 0 => { + out.push(&line[start..i]); + start = i + 1; + } + _ => {} + } + } + out.push(&line[start..]); + out +} + +fn braces_balanced(s: &str) -> bool { + let mut depth: i32 = 0; + for b in s.bytes() { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth < 0 { + return false; + } + } + _ => {} + } + } + depth == 0 +} + +/// `param_mapping_presets.csv` layout: a single header row lists +/// preset-selector field names (e.g. `BOOT_MODE,CLOCK_MODE,…`); each +/// data row has the selector name in column 0 and one valid value in +/// column 1. The CSV repeats the header row between sections — we +/// detect those repeats and skip them so the header names don't get +/// mistakenly recorded as values of each other. +fn parse_presets_csv(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let text = text.strip_prefix('\u{feff}').unwrap_or(&text); + let mut lines = text.lines().filter(|l| !l.trim().is_empty()); + let Some(header_line) = lines.next() else { + return; + }; + let headers: BTreeSet = header_line + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty() && is_safe_key(s)) + .map(str::to_string) + .collect(); + + let mut by_key: HashMap> = HashMap::new(); + for line in lines { + let cols: Vec<&str> = line.split(',').map(str::trim).collect(); + let Some(first) = cols.first().filter(|c| !c.is_empty()) else { + continue; + }; + // Skip repeated header rows: every non-empty cell is itself a + // declared header name. + let is_header_row = + cols.iter().all(|c| c.is_empty() || headers.contains(*c)); + if is_header_row { + continue; + } + if !headers.contains(*first) { + continue; + } + let Some(val) = cols.get(1).filter(|v| !v.is_empty()) else { + continue; + }; + by_key + .entry((*first).to_string()) + .or_default() + .insert(unwrap_one_brace(val).to_string()); + } + + for name in headers { + let mut enums = by_key.remove(&name).unwrap_or_default(); + // Vivado UI convention: preset-selector fields always offer + // `Custom` as the "configure each inner field manually" choice + // even when the CSV doesn't enumerate it. It's also the most + // useful default — picking a preset bundle locks the inner + // fields, picking `Custom` lets the user override them. + enums.insert("Custom".to_string()); + let default = "Custom".to_string(); + let f = fields.entry(name.clone()).or_insert_with(|| DictField { + name: name.clone(), + default: default.clone(), + description: None, + enum_values: BTreeSet::new(), + ..Default::default() + }); + for v in enums { + f.enum_values.insert(v); + } + } +} + +/// Strip one layer of Tcl-style braces from a value if present: +/// `{0}` → `0`, `{{ENABLE 0}}` → `{ENABLE 0}`. Leaves unbalanced or +/// unbraced inputs alone. +fn unwrap_one_brace(s: &str) -> &str { + let s = s.trim(); + if s.len() >= 2 && s.starts_with('{') && s.ends_with('}') { + // Verify the outer braces actually pair with each other (i.e. + // depth reaches 0 only at the final `}`). + let mut depth: i32 = 0; + for (i, b) in s.bytes().enumerate() { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 && i != s.len() - 1 { + return s; // not a single outer pair + } + } + _ => {} + } + } + return &s[1..s.len() - 1]; + } + s +} + +/// Conservative IP-XACT-style identifier check: starts with a letter +/// or `_`, then alphanumerics / `_`. Anything else is data we don't +/// understand and should ignore (rather than mistake for a field). +fn is_safe_key(s: &str) -> bool { + let mut chars = s.chars(); + let Some(c0) = chars.next() else { + return false; + }; + if !(c0.is_ascii_alphabetic() || c0 == '_') { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// Layer `X` text from a `ParamInfo.xml` +/// onto matching field descriptions. Uses a tiny line-oriented scan +/// — the schema is too irregular to demand a full XML parser. +fn layer_param_info(path: &Path, fields: &mut HashMap) { + let Ok(text) = fs::read_to_string(path) else { + return; + }; + let mut current: Option = None; + for line in text.lines() { + if let Some(start) = line.find("") { + let after = &line[start + "".len()..]; + if let Some(end) = after.find("") { + let text = after[..end].trim(); + if !text.is_empty() { + if let Some(f) = fields.get_mut(name) { + f.description = Some(text.to_string()); + } + } + } + } + if line.contains("") { + current = None; + } + } + } +} + +/// Layer enum values from preset XML files onto matching fields. +/// We only consume `` entries — those are +/// genuine selector-style enumerations (BOOT_MODE, SMON_ALARMS, …) +/// where Vivado's UI offers a fixed dropdown. The XMLs also contain +/// `` entries, but those are concrete +/// values *applied* by a parent preset; they're not an exhaustive +/// list of valid values. For a numeric field like +/// `PMC_CRP_PL0_REF_CTRL_FREQMHZ` the user can supply any frequency +/// the clock generator can synthesize (e.g. `250`, `195`), so +/// treating `` values as an `@enum` would wrongly reject those. +fn layer_presets(pspmc_dir: &Path, fields: &mut HashMap) { + let mut paths = Vec::new(); + paths.push(pspmc_dir.join("global/global_preset.xml")); + paths.push(pspmc_dir.join("global/global_presetForNonPS.xml")); + walk_for_xml(&pspmc_dir.join("presets"), &mut paths); + + for p in paths { + let Ok(text) = fs::read_to_string(&p) else { + continue; + }; + for line in text.lines() { + if let Some((param, val)) = + extract_two_attrs(line, "` works). +fn extract_two_attrs<'a>( + line: &'a str, + tag: &str, + attr_a: &str, + attr_b: &str, +) -> Option<(&'a str, &'a str)> { + let tag_idx = line.find(tag)?; + let after_tag = &line[tag_idx + tag.len()..]; + let (a, rest) = scan_attr(after_tag, attr_a)?; + let (b, _) = scan_attr(rest, attr_b)?; + Some((a, b)) +} + +fn scan_attr<'a>(s: &'a str, name: &str) -> Option<(&'a str, &'a str)> { + let needle_eq = format!("{name}=\""); + let idx = s.find(&needle_eq)?; + let after = &s[idx + needle_eq.len()..]; + let end = after.find('"')?; + Some((&after[..end], &after[end + 1..])) +} + +fn walk_for_xml(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let path = e.path(); + if path.is_dir() { + walk_for_xml(&path, out); + } else if path.extension().and_then(|s| s.to_str()) == Some("xml") { + out.push(path); + } + } +} + +/// Walk up `start` looking for an ancestor literally named `data`. +fn find_data_root(start: &Path) -> Option { + for ancestor in start.ancestors() { + if ancestor.file_name().and_then(|s| s.to_str()) == Some("data") { + return Some(ancestor.to_path_buf()); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_brace_aware_respects_tcl_groups() { + assert_eq!(split_brace_aware("a,b,c"), vec!["a", "b", "c"]); + // Commas inside `{…}` stay attached to that cell. + assert_eq!( + split_brace_aware("KEY,{a,b,c},NEXT"), + vec!["KEY", "{a,b,c}", "NEXT"] + ); + // Nested braces. + assert_eq!( + split_brace_aware("K,{{x,y} {z,w}},end"), + vec!["K", "{{x,y} {z,w}}", "end"] + ); + } + + #[test] + fn parse_direct_csv_drops_unbalanced_brace_defaults() { + use std::io::Write; + let f = tempfile::NamedTempFile::new().unwrap(); + // First row is well-formed; second has an unterminated brace + // group (matches a real bug in Xilinx's vendor CSV). + writeln!(&f, "GOOD_KEY,{{0}},GOOD_KEY,{{0}}").unwrap(); + writeln!(&f, "BAD_KEY,{{a,b,c").unwrap(); + let mut m: HashMap = HashMap::new(); + parse_direct_csv(f.path(), &mut m); + assert_eq!(m["GOOD_KEY"].default, "0"); + // BAD_KEY is recorded as a field but with no default. + assert!(m.contains_key("BAD_KEY")); + assert_eq!(m["BAD_KEY"].default, ""); + } + + #[test] + fn unwrap_one_brace_strips_outer_pair_only() { + assert_eq!(unwrap_one_brace("{0}"), "0"); + assert_eq!(unwrap_one_brace("{{ENABLE 0}}"), "{ENABLE 0}"); + assert_eq!(unwrap_one_brace("Custom"), "Custom"); + // Unbalanced — leave alone. + assert_eq!(unwrap_one_brace("{a"), "{a"); + // Two adjacent groups — not a single outer pair. + assert_eq!(unwrap_one_brace("{a}{b}"), "{a}{b}"); + } + + #[test] + fn safe_key_rejects_anything_with_braces_or_special_chars() { + assert!(is_safe_key("PS_USE_PMCPL_CLK0")); + assert!(is_safe_key("_misc")); + assert!(!is_safe_key("{0}")); + assert!(!is_safe_key("0_LEADS_WITH_DIGIT")); + assert!(!is_safe_key("")); + assert!(!is_safe_key("has space")); + } + + #[test] + fn discovery_returns_empty_outside_xilinx_layout() { + let dir = tempfile::tempdir().unwrap(); + let loose = dir.path().join("component.xml"); + std::fs::write(&loose, "").unwrap(); + assert!(load_schemas(&loose).is_empty()); + } + + /// Build a tempdir mimicking the Xilinx layout closely enough to + /// exercise the loader end-to-end without a Vivado install. + #[test] + fn loads_minimum_viable_schema_from_synthetic_layout() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + + let pspmc = data.join("versal/cips_hip/pspmc"); + let csvs = + data.join("versal/flows/automation/cipsToPsWiz_Porting/csv_files"); + let global = pspmc.join("global"); + let guidata = pspmc.join("guidata"); + let presets = pspmc.join("presets"); + std::fs::create_dir_all(&csvs).unwrap(); + std::fs::create_dir_all(&global).unwrap(); + std::fs::create_dir_all(&guidata).unwrap(); + std::fs::create_dir_all(&presets).unwrap(); + + std::fs::write( + csvs.join("param_mapping_direct.csv"), + "\ +PCIE_APERTURES_DUAL_ENABLE,{0},PCIE_APERTURES_DUAL_ENABLE,{0} +PS_PCIE_RESET,{{ENABLE 0}},PS_PCIE_RESET,{ENABLE 0 IO PS_MIO_18:19} +SMON_ALARMS,{Set_Alarms_On},SMON_ALARMS,{Set_Alarms_On} +CPM_PCIE0_MODES,{None},CPM_PCIE0_MODES,{None} +", + ) + .unwrap(); + std::fs::write( + csvs.join("param_mapping_presets.csv"), + "\ +BOOT_MODE,CLOCK_MODE +BOOT_MODE,JTAG Boot +BOOT_MODE,Master Mode +CLOCK_MODE,Custom +CLOCK_MODE,REF CLK 33.33 MHz +", + ) + .unwrap(); + std::fs::write( + guidata.join("ParamInfo.xml"), + r#" + + + What do you want to do with Alarms? + + +"#, + ) + .unwrap(); + std::fs::write( + presets.join("sysmon.xml"), + r#" + + + +"#, + ) + .unwrap(); + // Empty global presets so the loader still finds the file. + std::fs::write(global.join("global_preset.xml"), "").unwrap(); + + let component = data.join("ip/xilinx/versal_cips_v3_4/component.xml"); + std::fs::create_dir_all(component.parent().unwrap()).unwrap(); + std::fs::write(&component, "").unwrap(); + + let schemas = load_schemas(&component); + assert!( + schemas.contains_key("PS_PMC_CONFIG"), + "schemas: {schemas:?}" + ); + assert!(schemas.contains_key("PS_PMC_CONFIG_INTERNAL")); + let s = &schemas["PS_PMC_CONFIG"]; + let by_name: HashMap<&str, &DictField> = + s.fields.iter().map(|f| (f.name.as_str(), f)).collect(); + // From direct.csv (with CPM_ filtered out): + assert!(by_name.contains_key("PCIE_APERTURES_DUAL_ENABLE")); + assert_eq!(by_name["PCIE_APERTURES_DUAL_ENABLE"].default, "0"); + assert_eq!( + by_name["PS_PCIE_RESET"].default, "{ENABLE 0}", + "should strip one brace layer" + ); + // CPM_ keys are filtered out. + assert!(!by_name.contains_key("CPM_PCIE0_MODES")); + // From ParamInfo: description present for SMON_ALARMS. + assert_eq!( + by_name["SMON_ALARMS"].description.as_deref(), + Some("What do you want to do with Alarms?") + ); + // From presets: enum widened. + assert!(by_name["SMON_ALARMS"].enum_values.contains("Set_Alarms_On")); + assert!(by_name["SMON_ALARMS"] + .enum_values + .contains("Set_Alarms_Off")); + // From presets.csv: CLOCK_MODE present with "Custom" as default. + assert!(by_name.contains_key("CLOCK_MODE")); + assert_eq!(by_name["CLOCK_MODE"].default, "Custom"); + assert!(by_name["CLOCK_MODE"].enum_values.contains("Custom")); + // BOOT_MODE's CSV row only lists "JTAG Boot" and "Master Mode" — + // we should still inject `Custom` automatically (Vivado convention). + assert_eq!(by_name["BOOT_MODE"].default, "Custom"); + assert!(by_name["BOOT_MODE"].enum_values.contains("Custom")); + assert!(by_name["BOOT_MODE"].enum_values.contains("JTAG Boot")); + } + + // ------------------------------------------------------------------ + // DictSchema::from_paired_default — XML-driven schema extraction. + // ------------------------------------------------------------------ + + #[test] + fn from_paired_default_flat_schema() { + // The `intf::channel_map` shape — flat pairs, no nesting. + let src = "INTF0_RX0 QUAD0_RX0 INTF0_TX0 QUAD0_TX0"; + let s = DictSchema::from_paired_default( + src, + "intf::channel_map", + &OverridesFile::default(), + ); + assert_eq!(s.fields.len(), 2); + assert!(s.sub_schemas.is_empty()); + assert_eq!(s.fields[0].name, "INTF0_RX0"); + assert_eq!(s.fields[0].default, "QUAD0_RX0"); + assert!(s.fields[0].enum_values.is_empty()); + } + + #[test] + fn from_paired_default_nested_lr_schema() { + // `INTF_LR_SETTINGS` shape — the nested-slot case. + let src = "LR0_SETTINGS {RX_HD_EN 0 TX_HD_EN 0} LR1_SETTINGS { }"; + let s = DictSchema::from_paired_default( + src, + "intf::txrx_optional_ports", + &OverridesFile::default(), + ); + // No scalar fields at this level — every entry is a nested + // slot. Both LR0_SETTINGS and LR1_SETTINGS become sub-schemas. + assert!(s.fields.is_empty()); + assert_eq!(s.sub_schemas.len(), 2); + let lr0 = s.sub_schemas.get("LR0_SETTINGS").expect("LR0 present"); + assert_eq!(lr0.fields.len(), 2); + assert_eq!(lr0.fields[0].name, "RX_HD_EN"); + assert_eq!(lr0.fields[0].default, "0"); + let lr1 = s.sub_schemas.get("LR1_SETTINGS").expect("LR1 present"); + assert!(lr1.fields.is_empty()); + assert!(lr1.sub_schemas.is_empty()); + } + + #[test] + fn from_paired_default_two_levels_matches_txrx_shape() { + // `INTF0_TXRX_OPTIONAL_PORTS` shape distilled — flat outer + // pairs terminating in a nested INTF_LR_SETTINGS whose + // values are themselves paired dicts. + let src = "GT_TYPE GTY GT_DIRECTION DUPLEX \ + INTF_LR_SETTINGS {LR0_SETTINGS {RX_HD_EN 0}}"; + let s = DictSchema::from_paired_default( + src, + "intf::txrx_optional_ports", + &OverridesFile::default(), + ); + // Two flat scalar fields, one nested sub-schema. + assert_eq!(s.fields.len(), 2); + assert_eq!(s.fields[0].name, "GT_TYPE"); + assert_eq!(s.fields[1].name, "GT_DIRECTION"); + assert_eq!(s.sub_schemas.len(), 1); + let intf_lr = s + .sub_schemas + .get("INTF_LR_SETTINGS") + .expect("INTF_LR_SETTINGS present"); + assert_eq!(intf_lr.sub_schemas.len(), 1); + let lr0 = intf_lr + .sub_schemas + .get("LR0_SETTINGS") + .expect("LR0 sub-schema"); + assert_eq!(lr0.fields.len(), 1); + assert_eq!(lr0.fields[0].name, "RX_HD_EN"); + } + + #[test] + fn overrides_apply_to_matching_shape_path() { + // Field-level enum refinement on a specific shape path. + // XML default is silent on RX_PAM_SEL's bounds; the + // override attaches `@enum(NRZ, PAM4)`. + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = std::collections::HashMap::new(); + fields.insert( + "rx_pam_sel".to_string(), + FieldOverride { + enum_values: Some(vec!["NRZ".into(), "PAM4".into()]), + default: None, + baseline: false, + }, + ); + ov.shapes.insert( + "intf::gt_settings::lr0_settings".into(), + ShapeOverrides { + fields, + baseline: false, + }, + ); + // Note the shape path passed at the root is the outer shape; + // the LR0_SETTINGS sub-schema descends to + // `intf::gt_settings::lr0_settings`. + let src = "LR0_SETTINGS {RX_PAM_SEL NRZ RX_HD_EN 0}"; + let s = DictSchema::from_paired_default(src, "intf::gt_settings", &ov); + let lr0 = s + .sub_schemas + .get("LR0_SETTINGS") + .expect("LR0_SETTINGS sub-schema"); + let rx_pam = lr0 + .fields + .iter() + .find(|f| f.name == "RX_PAM_SEL") + .expect("RX_PAM_SEL field"); + assert_eq!( + rx_pam.enum_values, + ["NRZ".to_string(), "PAM4".to_string()] + .into_iter() + .collect() + ); + // Non-overridden field keeps the XML default and empty enum. + let rx_hd = lr0 + .fields + .iter() + .find(|f| f.name == "RX_HD_EN") + .expect("RX_HD_EN field"); + assert!(rx_hd.enum_values.is_empty()); + assert_eq!(rx_hd.default, "0"); + } + + #[test] + fn override_baseline_flag_flows_into_dict_field() { + // `overrides.toml` marks a field as baseline — the flag + // has to end up on the extracted `DictField` so the + // emitter can pick the `@baseline` vs `@default` attribute. + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = std::collections::HashMap::new(); + fields.insert( + "rx_refclk_frequency".to_string(), + FieldOverride { + enum_values: None, + default: None, + baseline: true, + }, + ); + ov.shapes.insert( + "intf::gt_settings::lr0_settings".into(), + ShapeOverrides { + fields, + baseline: false, + }, + ); + let src = "LR0_SETTINGS {RX_REFCLK_FREQUENCY 156.25 RX_HD_EN 0}"; + let s = DictSchema::from_paired_default(src, "intf::gt_settings", &ov); + let lr0 = s.sub_schemas.get("LR0_SETTINGS").unwrap(); + let rx_freq = lr0 + .fields + .iter() + .find(|f| f.name == "RX_REFCLK_FREQUENCY") + .expect("RX_REFCLK_FREQUENCY field"); + assert!(rx_freq.baseline, "baseline flag should be set"); + assert_eq!(rx_freq.default, "156.25"); + let rx_hd = lr0 + .fields + .iter() + .find(|f| f.name == "RX_HD_EN") + .expect("RX_HD_EN field"); + assert!(!rx_hd.baseline, "non-flagged field stays baseline=false"); + } + + #[test] + fn shape_wide_baseline_marks_every_field() { + // Shape-level `baseline = true` applies to EVERY scalar + // field the schema extractor produces, regardless of per- + // field configuration. Per-field overrides still layer on + // top; they just can't OPT OUT of the shape-wide switch. + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = HashMap::new(); + fields.insert( + "rx_pam_sel".to_string(), + FieldOverride { + enum_values: Some(vec!["NRZ".into(), "PAM4".into()]), + default: None, + baseline: false, + }, + ); + ov.shapes.insert( + "intf::lr0_settings".into(), + ShapeOverrides { + fields, + baseline: true, + }, + ); + let src = "RX_PAM_SEL NRZ RX_HD_EN 0 RX_REFCLK_FREQUENCY 156.25"; + let s = DictSchema::from_paired_default(src, "intf::lr0_settings", &ov); + // Every field — even the one WITHOUT a per-field override + // (RX_HD_EN, RX_REFCLK_FREQUENCY) and the one WITH a + // per-field enum but no per-field baseline (RX_PAM_SEL) — + // must land as baseline. That's the whole point. + assert!(s.fields.iter().all(|f| f.baseline), "{:?}", s.fields); + } + + #[test] + fn override_default_replaces_xml_default() { + use crate::overrides::{FieldOverride, OverridesFile, ShapeOverrides}; + let mut ov = OverridesFile::default(); + let mut fields = std::collections::HashMap::new(); + fields.insert( + "rx_line_rate".to_string(), + FieldOverride { + enum_values: None, + default: Some("25.78125".into()), + baseline: false, + }, + ); + ov.shapes.insert( + "intf::lr0_settings".into(), + ShapeOverrides { + fields, + baseline: false, + }, + ); + let src = "RX_LINE_RATE 10.3125 RX_HD_EN 0"; + let s = DictSchema::from_paired_default(src, "intf::lr0_settings", &ov); + let rx_line = s + .fields + .iter() + .find(|f| f.name == "RX_LINE_RATE") + .expect("RX_LINE_RATE field"); + assert_eq!(rx_line.default, "25.78125"); + } +} diff --git a/vw-ip/src/family.rs b/vw-ip/src/family.rs new file mode 100644 index 0000000..846c3ff --- /dev/null +++ b/vw-ip/src/family.rs @@ -0,0 +1,424 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Detect **indexed families** among sibling tree nodes. +//! +//! An indexed family is a set of sibling nodes whose labels differ +//! only by a trailing digit run — e.g. `MAC_PORT0`, `MAC_PORT1`, …, +//! `MAC_PORT5` — AND whose per-node parameter shapes are identical +//! once the digit is stripped. When present, the family's N sibling +//! nodes collapse into one constructor proc plus N kwargs on the +//! parent, and the parent's body emits ONE atomic +//! `set_property -dict` across everything. +//! +//! One guardrail keeps the detection safe: +//! +//! - **Strict direct-param shape match.** After stripping the +//! `_` prefix from each direct param, the resulting +//! name-set must be identical across members, and each name's +//! `default_value` / `choice_ref` / `is_user_configurable` triple +//! must agree. Any divergence falls back to per-N. +//! +//! Detection walks the whole tree — the family need not sit at +//! root level, because intermediate grouping nodes (e.g. DCMAC's +//! `MAC` node aggregating `MAC_PORT0..5`) don't emit their own proc. +//! Detected families ALWAYS land as kwargs on the top-level +//! `::create` proc; the sibling sub-procs that today set the +//! direct params of each family member (`::mac_port0..5`) +//! disappear in favor of the composed constructor. +//! +//! Sub-nodes UNDER family members (e.g. `MAC_PORT0_RX`, +//! `MAC_PORT0_TX` — each their own tree node with their own direct +//! params) continue to emit as per-N sub-procs. They set properties +//! deeper in the CONFIG namespace than the family constructor +//! covers, so leaving them per-N is safe — the atomicity bug only +//! affects the family's direct-param slice. +//! +//! Callers can opt out further via [`DetectOptions::excluded_stems`] +//! when a specific stem needs to stay per-N. + +use std::collections::{BTreeMap, HashMap}; + +use ipxact::Parameter; + +use crate::tree::{strip_prefix, Node}; + +/// Options for [`detect_families`]. Every knob defaults to "detect +/// everything the guardrails allow"; the caller can subtract from +/// there via `excluded_stems` when a specific family causes trouble. +#[derive(Clone, Debug, Default)] +pub struct DetectOptions { + /// Family stems (e.g. `"MAC_PORT"`) to leave per-N even when + /// the guardrails would otherwise collapse them. Populated by + /// the CLI's `--no-collapse=STEM,STEM,…` flag. Empty by default. + pub excluded_stems: Vec, +} + +/// A detected indexed family — enough information for the emitter +/// to produce a constructor and weave the kwargs into the parent +/// proc. +#[derive(Clone, Debug)] +pub struct IndexedFamily<'a> { + /// The common label prefix with trailing digits stripped + /// (`"MAC_PORT"`, `"GT_CH"`, `"FEC_SLICE"`, …). Used to name + /// the emitted `::` constructor and the + /// `::` newtype. + pub stem: String, + /// The concrete digit indices present in the source, in + /// ascending order (e.g. `[0, 1, 2, 3, 4, 5]` for DCMAC's + /// MAC_PORT family). Members can be non-contiguous. + pub indices: Vec, + /// Direct-params from the FIRST member. The emitter treats + /// these as canonical shape; enum values are still unioned + /// across all members (see [`Self::members_direct`]) so per- + /// member enum-choice differences don't leak into the + /// constructor's arg constraints. + pub shape: Vec<&'a Parameter>, + /// All members' direct-param lists, in index order (parallel + /// to [`Self::indices`]). Emission uses this to union enum + /// values across members — some IPs give the same logical + /// field different `choiceRef` sets per index (DCMAC's + /// MAC_PORT0 has 27 CONFIG_C0 choices while MAC_PORT1 has 12), + /// and the collapsed constructor needs to accept the superset + /// so callers can populate any port they want. + pub members_direct: Vec>, + /// Parent node's label (empty for root children). Used to + /// place the family's kwargs on the correct parent proc. + pub parent_label: String, + /// Original label of the shape-carrier member (e.g. + /// `"MAC_PORT0"`). Used to strip param-name prefixes at + /// emission time via [`strip_prefix`]. + pub shape_member_label: String, + /// Parallel to [`Self::indices`] — each member's own label + /// (e.g. `["MAC_PORT0", "MAC_PORT1", …, "MAC_PORT5"]`). + /// Emission uses this to strip the correct prefix from each + /// member's params when unioning enum values. + pub member_labels: Vec, +} + +/// Walk `root`'s subtree, collecting every indexed family whose +/// members' direct params shape-match AND whose enclosing parent +/// chain contains no already-indexed node. Sub-nodes UNDER family +/// members are ignored by this pass — they keep emitting per-N. +/// +/// The "no indexed ancestor" rule is what the user's "follow suit +/// for nested" instruction means in practice: DCMAC's +/// `MAC_PORT0..5` collapse (parent chain is root → `MAC`, +/// neither indexed) but CPM5's `PF0_BAR0..N` don't (parent chain +/// runs through `PCIE0` / `PF0`, both digit-indexed themselves). +pub fn detect_families<'a>( + root: &Node<'a>, + opts: &DetectOptions, +) -> Vec> { + let mut out = Vec::new(); + detect_at(root, opts, /* indexed_ancestor */ false, &mut out); + out +} + +fn detect_at<'a>( + node: &Node<'a>, + opts: &DetectOptions, + indexed_ancestor: bool, + out: &mut Vec>, +) { + // Group this node's children by stem. + let mut by_stem: BTreeMap)>> = BTreeMap::new(); + for child in &node.children { + let Some((stem, idx)) = split_trailing_digits(&child.label) else { + continue; + }; + by_stem.entry(stem).or_default().push((idx, child)); + } + // Try to collapse each stem-group. Members that don't pass the + // guardrails fall through untouched — the emitter keeps + // producing per-N procs for them. + for (stem, mut members) in by_stem { + if members.len() < 2 { + continue; + } + if opts.excluded_stems.iter().any(|s| s == &stem) { + continue; + } + if indexed_ancestor { + // Nested-under-indexed context. Follow-suit rule: + // keep emitting per-N. + continue; + } + // Shape match on direct params: same name-set across + // members + same (default, is_user_configurable) per + // field. `choice_ref` intentionally NOT compared — + // per-member enum-choice differences are semantically + // OK for the family constructor as long as we union + // them at emit time (see IndexedFamily::members_direct). + members.sort_by_key(|(idx, _)| *idx); + if !shapes_match(&members) { + continue; + } + let (_, first) = members[0]; + // Skip empty-shape families — nothing to hoist into the + // constructor. Their direct-params list is empty because + // all their params live in sub-nodes; the per-N sub-procs + // handle those and there's no atomicity benefit to + // emitting a stub constructor. + if first.direct.is_empty() { + continue; + } + out.push(IndexedFamily { + stem: stem.clone(), + indices: members.iter().map(|(idx, _)| *idx).collect(), + shape: first.direct.clone(), + members_direct: members + .iter() + .map(|(_, n)| n.direct.clone()) + .collect(), + parent_label: node.label.clone(), + shape_member_label: first.label.clone(), + member_labels: members + .iter() + .map(|(_, n)| n.label.clone()) + .collect(), + }); + } + // Recurse — a family may live under an intermediate grouping + // node (DCMAC's `MAC` node aggregates `MAC_PORT0..5`). + for child in &node.children { + // Once we cross into an indexed node, everything below + // inherits "nested" status and stays per-N. + let child_indexed = + indexed_ancestor || split_trailing_digits(&child.label).is_some(); + detect_at(child, opts, child_indexed, out); + } +} + +/// If `label` ends in a run of ASCII digits, return `(stem, index)`. +/// Empty stem or non-digit-suffix labels return `None` — those aren't +/// indexed family members. +fn split_trailing_digits(label: &str) -> Option<(String, u32)> { + let bytes = label.as_bytes(); + let mut cut = bytes.len(); + while cut > 0 && bytes[cut - 1].is_ascii_digit() { + cut -= 1; + } + if cut == bytes.len() || cut == 0 { + // No trailing digits OR digits are the whole label + // (e.g. `"0"` alone — not a family we can name after a stem). + return None; + } + let stem = label[..cut].to_string(); + let idx: u32 = label[cut..].parse().ok()?; + Some((stem, idx)) +} + +/// Compare all members' direct-param sets against member[0]'s. +/// True when every member has the same set of index-stripped param +/// names AND each name's triple matches. +fn shapes_match(members: &[(u32, &Node<'_>)]) -> bool { + let mut per_member: Vec>> = + Vec::with_capacity(members.len()); + for (_, n) in members { + let mut map = HashMap::new(); + for p in &n.direct { + let short = strip_prefix(&p.name, &n.label).to_string(); + map.insert( + short, + ShapeSlot { + default: p.value.default_value(), + user_config: p.value.is_user_configurable(), + }, + ); + } + per_member.push(map); + } + let first = &per_member[0]; + per_member.iter().skip(1).all(|m| { + m.len() == first.len() && first.iter().all(|(k, v)| m.get(k) == Some(v)) + }) +} + +/// The subset of a `Parameter`'s state that a family constructor +/// treats as the source of truth. Comparing these tuples across +/// members is our shape-equality check. +/// +/// Deliberately NOT included: +/// - `description` — human-authored prose that may reasonably vary +/// per member ("port 0 config" vs "port 1 config") without +/// changing the semantic shape. +/// - `choice_ref` — some IPs give the same logical field +/// different enum-choice sets per index (DCMAC's MAC_PORT0 vs +/// MAC_PORT1..5 CONFIG_C0). The atomicity-fix goal requires +/// allowing these to collapse; per-member choice_refs are +/// unioned at emit time via [`IndexedFamily::members_direct`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ShapeSlot<'a> { + default: &'a str, + user_config: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::{build_tree, TreeOptions}; + + fn p(name: &str, default: &str, choice_ref: Option<&str>) -> Parameter { + Parameter { + name: name.into(), + value: ipxact::ParamValue { + text: default.into(), + choice_ref: choice_ref.map(Into::into), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + } + } + + #[test] + fn strips_trailing_digits() { + assert_eq!( + split_trailing_digits("MAC_PORT0"), + Some(("MAC_PORT".into(), 0)) + ); + assert_eq!( + split_trailing_digits("MAC_PORT12"), + Some(("MAC_PORT".into(), 12)) + ); + assert_eq!(split_trailing_digits("MAC_PORT"), None); + assert_eq!(split_trailing_digits("0"), None); + assert_eq!(split_trailing_digits(""), None); + } + + /// Six sibling nodes with matching shape collapse to one family. + /// Uses 3-segment param names so the recursive tree lands the + /// per-port params directly on `MAC_PORT` (no sub-children), + /// which is the shape DCMAC's real params produce. + #[test] + fn dcmac_like_leaf_family_collapses() { + let mut params = Vec::new(); + for n in 0..6 { + params.push(p(&format!("MAC_PORT{n}_CONFIG"), "200GAUI-4", None)); + params.push(p(&format!("MAC_PORT{n}_ENABLE"), "0", None)); + params.push(p(&format!("MAC_PORT{n}_MODE"), "static", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert_eq!(families.len(), 1, "{families:#?}"); + let fam = &families[0]; + assert_eq!(fam.stem, "MAC_PORT"); + assert_eq!(fam.indices, vec![0, 1, 2, 3, 4, 5]); + assert_eq!(fam.shape.len(), 3); + } + + /// Nested siblings (each member has its own children) do NOT + /// collapse — the leaf-only rule keeps them per-N. + #[test] + fn nested_family_does_not_collapse() { + let mut params = Vec::new(); + // Two BARs, each with enough sub-params under `_BRIDGE` and + // `_QDMA` to trigger sub-nodes. + for n in 0..2 { + for kind in ["BRIDGE", "QDMA"] { + for i in 0..5 { + params.push(p( + &format!("PF0_BAR{n}_{kind}_FIELD{i}"), + "0", + None, + )); + } + } + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!( + families.iter().all(|f| f.stem != "PF0_BAR"), + "{families:#?}" + ); + } + + /// Sibling shapes that disagree (one has an extra field) fall + /// back to per-N. + #[test] + fn shape_mismatch_falls_back_to_per_n() { + // port 0 has 3 fields; port 1 has only 2 (extra `_C` missing). + let params = [ + p("MAC_PORT0_A", "0", None), + p("MAC_PORT0_B", "0", None), + p("MAC_PORT0_C", "0", None), + p("MAC_PORT1_A", "0", None), + p("MAC_PORT1_B", "0", None), + ]; + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// Sibling shapes with different defaults for the same field + /// also fall back to per-N — the collapsed constructor would + /// mis-report a shared default. + #[test] + fn different_defaults_prevent_collapse() { + let params = [ + p("MAC_PORT0_CONFIG", "200GAUI-4", None), + p("MAC_PORT1_CONFIG", "400GAUI-8", None), + ]; + let opts = TreeOptions { min_split_size: 1 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// Different `choice_ref` values across members SHOULD still + /// collapse — the constructor unions enum values at emit time. + /// (See ShapeSlot's doc comment for the rationale.) + #[test] + fn different_choice_refs_do_collapse() { + let mut params = Vec::new(); + for n in 0..3 { + params.push(p( + &format!("MAC_PORT{n}_CONFIG"), + "100CAUI-4", + Some(&format!("port{n}_choices")), + )); + params.push(p(&format!("MAC_PORT{n}_ENABLE"), "0", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let families = detect_families(&tree, &DetectOptions::default()); + assert_eq!(families.len(), 1, "{families:#?}"); + } + + /// Single-shape IPs (no sibling groups at all) produce zero + /// families. + #[test] + fn single_shape_ip_produces_no_families() { + let params = [ + p("ONE", "0", None), + p("TWO", "0", None), + p("THREE", "0", None), + ]; + let tree = build_tree(params.iter(), &TreeOptions::default()); + let families = detect_families(&tree, &DetectOptions::default()); + assert!(families.is_empty(), "{families:#?}"); + } + + /// `excluded_stems` skips detection for named stems even when + /// the guardrails would collapse them. + #[test] + fn excluded_stem_stays_per_n() { + let mut params = Vec::new(); + for n in 0..3 { + params.push(p(&format!("MAC_PORT{n}_A"), "0", None)); + params.push(p(&format!("MAC_PORT{n}_B"), "0", None)); + } + let opts = TreeOptions { min_split_size: 2 }; + let tree = build_tree(params.iter(), &opts); + let opts = DetectOptions { + excluded_stems: vec!["MAC_PORT".into()], + }; + let families = detect_families(&tree, &opts); + assert!(families.is_empty(), "{families:#?}"); + } +} diff --git a/vw-ip/src/generate.rs b/vw-ip/src/generate.rs new file mode 100644 index 0000000..1143ec8 --- /dev/null +++ b/vw-ip/src/generate.rs @@ -0,0 +1,3680 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Emit an htcl wrapper proc for an IP-XACT component. +//! +//! Two shapes, picked by `split_threshold`. Every emitted proc lives +//! in the IP's own namespace (`::…`) so name collisions across +//! IPs are structural and callers reach helpers by tab-completing +//! `::`. +//! +//! - **Single-proc** (small IPs like CIPS with 19 params): one +//! `::create` proc whose structured args mirror the IP's +//! parameters. Each arg gets `@default()` from the IP-XACT +//! default; `@enum(...)` when the parameter has a `choiceRef`. The +//! body emits `set_property -dict [list ...]` mapping each arg back +//! to its `CONFIG.` key. +//! +//! - **Split** (large IPs like CPM5 with ~8700 params): a top +//! `::create` proc that just creates the bd_cell and returns +//! its handle, plus one `::` sub-proc per parameter +//! prefix group. Each sub-proc takes the cell handle as its first +//! arg, then its own group's parameters. Small groups +//! (< `min_group_size`) collapse into a `_misc` sub-proc so we end +//! up with a manageable handful rather than dozens of singletons. +//! +//! Call-site composition: +//! ```tcl +//! set cps [cpm5::create cps] +//! cpm5::pcie1 $cps -max_link_speed "32.0_GT/s" -modes PCIE +//! ``` + +use std::fmt::Write; + +use ipxact::{Component, Parameter}; +use vw_htcl::emit::{Command, Doc, Item, Word}; + +use crate::family::{detect_families, DetectOptions, IndexedFamily}; +use crate::tree::{build_tree, strip_prefix, Node, TreeOptions}; + +#[derive(Clone, Debug)] +pub struct GenerateOptions { + /// Emit `## ` doc comments for parameters that have a + /// description in IP-XACT. + pub include_descriptions: bool, + /// Skip auto-resolve parameters; emit only user-configurable ones. + pub user_configurable_only: bool, + /// When the parameter count exceeds this, the generator emits a + /// hierarchy of procs instead of one. Tuned so CIPS (19) stays + /// single and CPM5 (8673) splits. + pub split_threshold: usize, + /// Don't split a subgroup into its own child proc unless it has at + /// least this many parameters. Smaller subgroups stay as direct + /// args of the parent so we don't get a long tail of singleton + /// procs. + pub min_split_size: usize, + /// Indexed-family stems to leave per-N (skip the composed- + /// constructor collapse). Passed through to + /// [`crate::family::DetectOptions::excluded_stems`]. Empty by + /// default; populated from the CLI's `--no-collapse=STEM,STEM,…` + /// flag when a specific stem needs to opt out. + pub no_collapse: Vec, + /// Per-IP TOML overrides file. Attaches `@enum(…)` refinements + /// and per-field default overrides to XML-derived DictSchemas + /// (see [`crate::overrides::OverridesFile`]). Empty when no + /// override file is present — the generator falls back to + /// XML-only defaults. + pub overrides: crate::overrides::OverridesFile, +} + +impl Default for GenerateOptions { + fn default() -> Self { + Self { + include_descriptions: true, + user_configurable_only: true, + split_threshold: 100, + min_split_size: 8, + no_collapse: Vec::new(), + overrides: crate::overrides::OverridesFile::default(), + } + } +} + +/// Multi-file generation output. +/// +/// Big IPs (gtwiz-versal at 160k lines, cpm5 at 40k) blow past +/// tree-sitter's default incremental-parse budget when squeezed +/// into one file. Splitting per split-node keeps every file under +/// ~20k lines — still large, but within reach of downstream tools. +/// +/// `main` is the primary `module.htcl` content, which sources every +/// entry in `subfiles` via `src ./` lines. Each subfile +/// contains one split-node's dict-schema newtypes + its constructor +/// proc, emitted with fully-qualified proc names so the file +/// stands alone (no namespace-block wrapping needed). +#[derive(Debug, Clone, Default)] +pub struct MultiFileOutput { + pub main: String, + /// `(basename, content)` pairs — basename is the file name + /// (no leading `./`, no directory prefix); the CLI writes each + /// to `/`. + pub subfiles: Vec<(String, String)>, +} + +impl MultiFileOutput { + /// Merge every subfile into `main` and return the concatenated + /// text. Preserves the pre-split single-file shape so unit tests + /// and callers that don't care about file layout can keep + /// treating the output as one string. + pub fn into_single(mut self) -> String { + for (_, sub) in &self.subfiles { + self.main.push('\n'); + self.main.push_str(sub); + } + self.main + } +} + +/// Generate the htcl wrapper text for `component`. +/// +/// `presets` carries supplementary parameter-value information loaded +/// from out-of-band sources (e.g. Vivado's `cpm_preset*.xml`); pass an +/// empty map when there are none. Values from `presets` are merged +/// with the IP-XACT `` entries when emitting `@enum(...)`. +pub fn generate( + component: &Component, + presets: &crate::presets::PresetMap, + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) -> MultiFileOutput { + let parameters: Vec<&Parameter> = component + .component_parameters() + .filter(|p| { + !opts.user_configurable_only || p.value.is_user_configurable() + }) + .collect(); + let mut out = if parameters.len() > opts.split_threshold { + generate_split(component, presets, ¶meters, opts, dict_schemas) + } else { + generate_single(component, presets, ¶meters, opts, dict_schemas) + }; + // Only the CSV-driven top-level schemas are surfaced here. + // XML-derived schemas for split-node params get emitted inline + // inside `emit_split_node_constructor`, and top-level XML + // schemas (unclaimed by split nodes) get emitted inside + // `generate_split` / `generate_single` themselves — see the + // dedicated merge steps there. + if !dict_schemas.is_empty() { + append_dict_sub_procs(&mut out, component, dict_schemas, opts); + } + // Peel off large split-node blocks into sibling files so + // tree-sitter (and any other line-oriented consumer) doesn't + // choke on the aggregate. See [`split_into_files`] for the + // peel heuristic and file-shape contract. + split_into_files(out) +} + +/// Emit one compositional-value constructor per IP-XACT +/// `structured_tcldict` parameter. Each schema gets a newtype +/// prelude (`namespace eval`, `type = Properties`, +/// `::repr`/`::from`/`::to`/`::empty`) plus a pure `::` +/// constructor that returns the typed value. The top proc's +/// matching kwarg (e.g. `-ps_pmc_config`) then takes the newtype +/// and unwraps it into the atomic `set_property -dict` — no +/// separate mutator call. +fn append_dict_sub_procs( + out: &mut String, + component: &Component, + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) { + let ip_name = sanitize_ident(&component.name); + let mut keys: Vec<&String> = dict_schemas.keys().collect(); + keys.sort(); + for param_name in keys { + let schema = &dict_schemas[param_name]; + if schema.fields.is_empty() { + continue; + } + writeln!(out).unwrap(); + // Top-level dict-schema procs (PS_PMC_CONFIG, etc.) live + // directly under `::` — pass an empty namespace prefix. + // Sub-schemas emitted while recursing pick up their own + // prefix chain from `emit_dict_sub_schemas`. + emit_dict_props_prelude(out, &ip_name, &[], param_name); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, &ip_name, &[], param_name, schema, opts); + } +} + +/// Emit the newtype declaration + `::from`/`::to`/`::repr`/`::empty` +/// helper procs for one dict-schema param. Mirror of +/// [`emit_family_prelude`] — same shape, different naming source. +/// +/// `namespace_prefix` composes intermediate namespace segments +/// between the IP name and the param name — for a nested +/// sub-constructor emitted under `::intf::gt_settings::`, pass +/// `["intf", "gt_settings"]`. Empty slice reproduces the original +/// flat `::` shape used by PS_PMC_CONFIG. +fn emit_dict_props_prelude( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, +) { + let qualified = dict_props_name(ip_name, namespace_prefix, param_name); + let ctor_lower = param_name.to_ascii_lowercase(); + let scope_display = if namespace_prefix.is_empty() { + format!("[{ip_name}::create]") + } else { + format!("[{ip_name}::{}]", namespace_prefix.join("::")) + }; + writeln!( + out, + "## Typed configuration value for {scope_display}'s \ + `-{ctor_lower}` slot. Construct with [{}::{ctor_lower}].", + proc_scope(ip_name, namespace_prefix) + ) + .unwrap(); + // Every ancestor namespace segment needs `namespace eval` so + // downstream `proc ::a::b::c` declarations resolve. Emit the + // whole chain from `` down to the newtype's own namespace. + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + for i in 0..namespace_prefix.len() { + let chain = std::iter::once(ip_name) + .chain(namespace_prefix[..=i].iter().copied()) + .collect::>() + .join("::"); + writeln!(out, "namespace eval {chain} {{}}").unwrap(); + } + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Compose the parent-namespace form for a nested sub-proc, without +/// the leaf. `("gtwiz_versal", ["intf", "gt_settings"])` → +/// `"gtwiz_versal::intf::gt_settings"`. Used by doc comments to +/// point at the containing scope. +fn proc_scope(ip_name: &str, namespace_prefix: &[&str]) -> String { + if namespace_prefix.is_empty() { + ip_name.to_string() + } else { + format!("{ip_name}::{}", namespace_prefix.join("::")) + } +} + +/// Emit the value-constructor `::` for one +/// dict-schema. Pure — no cell, no `-bd`, no `set_property`. Builds +/// a `Properties`-shaped dict from the supplied field kwargs and +/// wraps it in the newtype. The atomic materialization happens +/// later in `::create`'s body, where the matching +/// `-` kwarg gets unwrapped through `::to` + +/// `Properties::to_raw` and merged into the single +/// `set_property -dict` call. +fn emit_dict_sub_proc( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, + schema: &crate::DictSchema, + opts: &GenerateOptions, +) { + // Recurse into sub-schemas FIRST so nested newtypes are declared + // before the outer proc references them. The outer proc's slot + // args carry types like `Intf0GtSettingsLr0Settings` — those + // types must exist in the analyzer's view by the time the outer + // proc's signature is checked, so the emission order (deepest + // first, then parent) matches lexical declaration order. + let sub_ctors = emit_dict_sub_schemas( + out, + ip_name, + namespace_prefix, + param_name, + schema, + opts, + ); + + let ctor_local = param_name.to_ascii_lowercase(); + let ctor_scope = proc_scope(ip_name, namespace_prefix); + let ctor_name = format!("{ctor_scope}::{ctor_local}"); + let ret_ty = dict_props_name(ip_name, namespace_prefix, param_name); + + let mut doc = Doc::new(); + let scope_display = if namespace_prefix.is_empty() { + format!("[{ip_name}::create]") + } else { + format!("[{}]", proc_scope(ip_name, namespace_prefix)) + }; + doc.push(Item::DocComment(format!( + "Configuration value for {scope_display}'s \ + `-{ctor_local}` slot (`CONFIG.{param_name}`). Composes into \ + the top proc so every provided field lands in ONE atomic \ + `set_property -dict` call.", + ))); + if !schema.fields.is_empty() || !sub_ctors.is_empty() { + doc.push(Item::Blank); + } + for f in &schema.fields { + emit_dict_field_arg(&mut doc, f, opts); + } + // Typed slots for nested sub-schemas — the outer proc takes + // one arg per LRn slot (or equivalent), and the runtime merges + // each slot's `::to` unwrap into the top-level Properties + // dict under the slot's XML key. + for (raw_name, sub_ret_ty) in &sub_ctors { + let arg = lowercase_ident(raw_name); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{arg}: {sub_ret_ty}")), + ], + body: None, + })); + } + + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for f in &schema.fields { + let arg = lowercase_ident(&f.name); + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} \ + {{ dict set _vw_d {} ${arg} }}", + f.name + ) + .unwrap(); + } + // Sub-slot merges: unwrap the typed newtype to its raw paired + // dict via `::to`, and stash it under the slot's original + // XML key. The sub-dict IS the raw paired-list form Vivado + // accepts for tcldict-typed compound properties — a + // `Properties::to_raw` wrap here would try to flatten as a + // TAGGED tree (`Property::Scalar` / `Property::Nested`), but + // the wrapper's `_vw_d` holds bare-string leaves (see the + // scalar `dict set _vw_d KEY $arg` above), not tagged + // `Property::Scalar(x)`. Match the sibling emission at + // `emit_split_node_constructor` — same shape, same reason. + for (raw_name, sub_ret_ty) in &sub_ctors { + let arg = lowercase_ident(raw_name); + // Wrap the dict-set in a `catch` that prepends the HTCL + // arg name on error. Historically fired on the + // `Properties::to_raw` mistake above; retained + // defensively — if a caller reaches this proc with a + // malformed sub-value some future code path relies on, + // the message still points at the offending slot by its + // HTCL surface name (`lr0_settings.…`). + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + dict set _vw_d {raw_name} \ + [{sub_ret_ty}::to -v ${arg}]\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n \ + }}", + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + emit_proc(out, &ctor_name, &doc, Some(&ret_ty), &body); +} + +/// Recursively emit the sub-constructor procs for `schema`'s nested +/// slots. Each sub-schema gets its own prelude + proc, declared in +/// a namespace one level deeper than the outer schema +/// (`::::::`). Returns +/// `(raw_slot_name, sub_ret_ty)` pairs so the outer proc's emitter +/// can declare typed slot args referencing these newtypes. +fn emit_dict_sub_schemas( + out: &mut String, + ip_name: &str, + namespace_prefix: &[&str], + outer_param: &str, + schema: &crate::DictSchema, + opts: &GenerateOptions, +) -> Vec<(String, String)> { + let mut acc = Vec::new(); + if schema.sub_schemas.is_empty() { + return acc; + } + // Extend the namespace prefix with the outer param's lowercase + // form — every sub-slot lives one level under the outer proc's + // scope: `::::`. + let outer_lower = outer_param.to_ascii_lowercase(); + let mut deeper: Vec<&str> = namespace_prefix.to_vec(); + deeper.push(&outer_lower); + for (raw_name, sub_schema) in &schema.sub_schemas { + // Empty sub-schemas (no fields + no further sub-slots) + // arise when an anchor's inner slot is a placeholder — e.g. + // TXRX_OPTIONAL_PORTS's `INTF_LR_SETTINGS` has LR0_SETTINGS + // populated but LR1_SETTINGS..LR15_SETTINGS empty. Emitting + // a proc for those would produce a body-less arg list + // (`proc … { ## doc only }`) which the htcl parser rejects + // as "doc comment with no following argument". Skip them — + // the outer proc's typed slot becomes bare (no sub-slot + // arg) and the pair simply isn't set. + if sub_schema.fields.is_empty() && sub_schema.sub_schemas.is_empty() { + continue; + } + writeln!(out).unwrap(); + emit_dict_props_prelude(out, ip_name, &deeper, raw_name); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, ip_name, &deeper, raw_name, sub_schema, opts); + let sub_ret_ty = dict_props_name(ip_name, &deeper, raw_name); + acc.push((raw_name.clone(), sub_ret_ty)); + } + acc +} + +fn emit_dict_field_arg( + doc: &mut Doc, + f: &crate::DictField, + opts: &GenerateOptions, +) { + if opts.include_descriptions { + if let Some(desc) = f.description.as_deref().filter(|s| !s.is_empty()) { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + let mut words = Vec::new(); + if !f.enum_values.is_empty() { + let formatted: Vec = f + .enum_values + .iter() + .map(|v| format_attribute_value(v)) + .collect(); + words.push(Word::Raw(format!("@enum({})", formatted.join(", ")))); + } + // Dict fields are always optional: Vivado treats an unset + // inner key as "use the IP's implicit default", so make every + // arg defaultable. When the Xilinx CSV didn't yield a default — + // either the row was missing one or the value had unbalanced + // braces and was rejected — fall back to an empty string so the + // user can omit the arg and let Vivado decide. + // + // Attribute name switches to `@baseline()` when the + // field's `overrides.toml` entry sets `baseline = true` — for + // scalar knobs whose value can be shifted by a sibling + // `-preset` / `-board_interface`. Runtime shape is unchanged + // (the wrapper still uses `if kw_set { dict set }`); the + // annotation just tells the analyzer and the reader that + // omission doesn't reset to ``, and the redundant-default + // lint stops flagging call-site values that match. + let attr = if f.baseline { "baseline" } else { "default" }; + words.push(Word::Raw(format!( + "@{attr}({})", + format_attribute_value(&f.default) + ))); + let lowered = lowercase_ident(&f.name); + words.push(Word::Bare(lowered)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +// --------------------------------------------------------------------------- +// Single-proc shape. +// --------------------------------------------------------------------------- + +fn generate_single( + component: &Component, + presets: &crate::presets::PresetMap, + parameters: &[&Parameter], + opts: &GenerateOptions, + dict_schemas: &std::collections::HashMap, +) -> String { + let vlnv = component.vlnv(); + let ip_name = sanitize_ident(&component.name); + + let mut out = String::new(); + emit_file_header(&mut out, component, &vlnv); + writeln!( + out, + "## ({} configurable parameter{})", + parameters.len(), + if parameters.len() == 1 { "" } else { "s" } + ) + .unwrap(); + + // `configure`'s arg-doc — the whole documented kwarg pile. + // Same shape as the old inline-`create` version, minus the + // `-name` / `-bd` slots which are now `create`-only. + let mut configure_doc = Doc::new(); + for p in parameters { + emit_arg_decl( + &mut configure_doc, + component, + presets, + p, + opts, + "", + &ip_name, + dict_schemas, + &[], + ); + } + + // `create`'s arg-doc — the narrow surface. Just the + // instantiation-mode args + a single typed `-config`. + let mut create_doc = Doc::new(); + create_doc.push(Item::DocComment( + "Project-level IP module name, or (when `-bd 1`) the \ + instance name in the block design." + .into(), + )); + create_doc.push(Item::Command(Command::call( + "name", + std::iter::empty::(), + ))); + push_bd_switch_arg(&mut create_doc); + create_doc.push(Item::DocComment(format!( + "Typed configuration value. Construct with [{ip_name}::configure]. \ + Defaults to an empty config (all parameters take their IP defaults)." + ))); + create_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("config: {ip_name}::Config")), + ], + body: None, + })); + + // Config prelude ships at file TOP LEVEL — namespace-eval + // double-prefix bug means qualified proc names like + // `::Config::from` inside `namespace eval {…}` end + // up as `::::Config::from`. See emit_family_prelude + // for the same trick. + writeln!(out).unwrap(); + emit_config_prelude(&mut out, &ip_name); + writeln!(out).unwrap(); + + let dict_schema_newtypes = + build_dict_schema_newtypes(&ip_name, dict_schemas); + let configure_body = build_single_configure_body( + component, + parameters, + &ip_name, + &dict_schema_newtypes, + ); + let configure_body = + wrap_body_qualified_prefix(&configure_body, &ip_name, "configure"); + let create_body = build_single_create_body(&vlnv, &ip_name); + + // Emit both procs inside `namespace eval { … }` so + // `configure` and `create` register as `::configure` and + // `::create`. Same wrap idiom as vivado-cmd's log.htcl. + let mut procs = String::new(); + emit_proc( + &mut procs, + "configure", + &configure_doc, + Some(&config_name(&ip_name)), + &configure_body, + ); + writeln!(procs).unwrap(); + emit_proc( + &mut procs, + "create", + &create_doc, + Some("string"), + &create_body, + ); + write_namespace_block(&mut out, &ip_name, &procs); + out +} + +/// Wrap `body` in `namespace eval { … }`, indenting each +/// non-empty line by two spaces. The wrapper lets the enclosed +/// procs use bare names (`create`, `mac_port`, …) while remaining +/// externally addressable as `::` — exactly the shape +/// vivado-cmd's hand-written `log.htcl` / `ip.htcl` use. +fn write_namespace_block(out: &mut String, ip_name: &str, body: &str) { + writeln!(out, "namespace eval {ip_name} {{").unwrap(); + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Emit the `-bd` switch as a proc-arg declaration. +/// +/// `-bd false` (default) → `create_ip` (project-level IP module); +/// `-bd true` → `create_bd_cell` (block-design cell). Project IP +/// is the default because it's the shape Vivado's own +/// `write_ip_tcl`-generated scripts use, and most external tools +/// (simulators, downstream regeneration flows) expect wrappers +/// that create discoverable IP source objects. Wrappers going +/// into a block design still work — the caller passes +/// `-bd true`. Both paths return a plain `string` handle at +/// runtime (a bd_cell path or a project-IP module name); the +/// wrapper's declared return type is `string` so both flows +/// type-check under strict nominal identity, and callers that +/// need `bd_cell` semantics can promote via `bd_cell::from` at +/// their own use site. +fn push_bd_switch_arg(doc: &mut Doc) { + doc.push(Item::Blank); + doc.push(Item::DocComment( + "Create the IP as a project-level module (`false`, \ + default) via `create_ip`, or as a block-design cell \ + (`true`) via `create_bd_cell`. Returns the underlying \ + handle as a string in either case — a `/…` block-\ + design cell path when `true`, a raw module name when \ + `false`. Downstream `set_property -dict …` works on \ + both shapes." + .into(), + )); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(false)".into()), + Word::Bare("bd: bool".into()), + ], + body: None, + })); +} + +/// Build `::create`'s body (single shape). Just cell-creation, +/// config unwrap, `set_property` finalize, and return. All the +/// dict-assembly happens in `::configure` — see +/// [`build_single_configure_body`]. +/// +/// The `-bd` switch chooses between `create_bd_cell` (default +/// bd=1) for block-design usage and `create_ip` (bd=0) for +/// project-IP usage. Returns `$cell` or `$name` accordingly per +/// the two-mode contract [`emit_config_finalize`] and its callers +/// depend on. +fn build_single_create_body(vlnv: &str, ip_name: &str) -> String { + let mut out = String::new(); + // Guard: `-config` defaults to `""` because the analyzer's + // `@default(...)` grammar rejects bracket-expressions like + // `[::empty]`. Coerce to a real empty Config value at the + // top of the body so downstream `::to` unwrap works. + // Family / split-node kwargs on the old shape used the same + // placeholder pattern (see the old generate_split arg-decl + // block for the family precedent). + writeln!(out, "if {{$config eq \"\"}} {{").unwrap(); + writeln!(out, " set config [{ip_name}::Config::empty]").unwrap(); + writeln!(out, "}}").unwrap(); + // Two-mode create: `-bd true` produces a bd_cell path, `-bd + // false` (default) a project-IP module name. Both wind up in + // `$handle` as a plain string; the wrapper's declared return + // type is `string` so the strict-nominal-identity return-type + // check passes in either branch. Callers that need a + // `bd_cell` newtype at their use site promote with + // `bd_cell::from` (which validates the `/…` shape); callers + // that don't just carry the string through. + writeln!(out, "if {{$bd}} {{").unwrap(); + writeln!( + out, + " set handle [vivado_cmd::create_bd_cell -type ip -vlnv {vlnv} -name $name]" + ) + .unwrap(); + writeln!(out, "}} else {{").unwrap(); + writeln!( + out, + " set handle [vivado_cmd::create_ip -vlnv {vlnv} -module_name $name]" + ) + .unwrap(); + // `create_ip` returns an XCI transcript, not the module + // name — but downstream `set_property` and callers want to + // reference the module by name. Overwrite `handle` with + // `$name` so both branches yield the "thing you address + // set_property against." + writeln!(out, " set handle $name").unwrap(); + writeln!(out, "}}").unwrap(); + emit_config_finalize(&mut out, ip_name); + writeln!(out, "return $handle").unwrap(); + out +} + +/// Wrap a proc's assembled body with a `try/on error` that +/// prepends `..` to any error message escaping +/// from the body — so a nested Properties tree failure at +/// `::configure`'s outermost scope surfaces to the user as +/// +/// ```text +/// gtwiz_versal.configure.intf0.gt_settings.lr0_settings.tx_refclk_frequency.unknown variant: 156.25 +/// ``` +/// +/// The per-arg dict-set catches inside intermediate constructor +/// procs (see `emit_dict_sub_proc` and the sibling emissions in +/// `write_dict_assembly` / `write_dict_assembly_with_splits`) +/// grow the arg-name path; this wrapper adds the top-level +/// `.` prefix so the message reads as a fully +/// qualified dotted path from IP name to leaf field. +/// +/// `try` (Tcl 8.6+) propagates the body's `return` code cleanly +/// through the default `on ok` behavior — the error handler only +/// fires when the body errors, not when it returns normally. +fn wrap_body_qualified_prefix( + body: &str, + ip_name: &str, + proc_local: &str, +) -> String { + format!( + "try {{\n\ + {body}\ + }} on error {{__vw_msg}} {{\n \ + error \"{ip_name}.{proc_local}.$__vw_msg\"\n\ + }}\n" + ) +} + +/// Build `::configure`'s body (single shape). Pure dict +/// assembly + wrap in `::Config`. Zero side effects. +fn build_single_configure_body( + component: &Component, + parameters: &[&Parameter], + ip_name: &str, + dict_schema_newtypes: &std::collections::HashMap, +) -> String { + let mut out = String::new(); + write_dict_assembly( + &mut out, + component, + parameters, + "", + &[], + dict_schema_newtypes, + ); + let config_ty = config_name(ip_name); + // Lift the assembled `_vw_d` (flat `CONFIG. value` pairs + // with bare-string values) into a NESTED, tagged Properties + // tree — CONFIG at the top wraps `Property::Nested` containing + // every `` sub-key as `Property::Scalar`. Matches the + // shape `props::get` returns from a live BD cell, so consumers + // can use `dict get [::Config::to -v $cfg] CONFIG` + + // `Property::as_nested -v ...` to extract sub-trees. + writeln!( + out, + "return [{config_ty}::from -v [Properties::from_dotted_pairs -v $_vw_d]]" + ) + .unwrap(); + out +} + +// --------------------------------------------------------------------------- +// Split shape: top proc + one sub-proc per prefix group. +// --------------------------------------------------------------------------- + +fn generate_split( + component: &Component, + presets: &crate::presets::PresetMap, + parameters: &[&Parameter], + opts: &GenerateOptions, + dict_schemas: &std::collections::HashMap, +) -> String { + let vlnv = component.vlnv(); + let ip_name = sanitize_ident(&component.name); + let top_proc = format!("{ip_name}::create"); + + let tree = build_tree( + parameters.iter().copied(), + &TreeOptions { + min_split_size: opts.min_split_size, + }, + ); + + // Merge XML-derived DictSchemas for the top-level Properties- + // shaped params (`tree.direct` only, so we don't clobber + // schemas the split-node emitter builds for its own params). + // These end up in `dict_schemas` for both the top-proc + // arg-decl (`emit_arg_decl` picks up the typed newtype) and + // the tail `append_dict_sub_procs` call in `generate()` (which + // emits the newtype prelude + constructor proc for each). + let mut dict_schemas: std::collections::HashMap = + dict_schemas.clone(); + for p in &tree.direct { + if dict_schemas.contains_key(&p.name) { + continue; + } + // Anchor lookup: some Vivado top-level params carry a scalar + // sentinel default (`0`) while a sibling `` + // of the same name holds the actual paired-list schema. Use + // the model-param default as the anchor when the top-level + // default doesn't parse as paired. Enables typed-constructor + // emission for INTF_PARENT_PIN_LIST and similar — where the + // slot names live in the internal HDL-generic view rather + // than on the user-facing property. + let anchor_default = model_param_anchor_default(component, &p.name); + let use_anchor = !is_properties_shaped_param(p) + && anchor_default.as_deref().is_some_and(|d| { + !crate::paired_list::parse_paired_list(d).is_empty() + }); + if !is_properties_shaped_param(p) && !use_anchor { + continue; + } + let default = if use_anchor { + anchor_default.as_deref().unwrap() + } else { + p.value.default_value() + }; + let shape_path = lowercase_ident(&p.name); + let mut schema = crate::DictSchema::from_paired_default( + default, + &shape_path, + &opts.overrides, + ); + // Extrapolate `QUAD_` keys across every quad the IP + // ships (5 for gtwiz-versal) so the constructor exposes + // ALL slots, not just quad0's — the model-param default + // only enumerates one quad's worth as a template. + // Also attaches auto-derived pin-path enums when the key + // shape matches `QUAD_`. See + // `extrapolate_quad_schema` for the details. + if use_anchor { + extrapolate_quad_schema(&mut schema, component, &shape_path, opts); + } + if schema.fields.is_empty() && schema.sub_schemas.is_empty() { + continue; + } + dict_schemas.insert(p.name.clone(), schema); + } + // Take a reference to what all downstream code expects. + let dict_schemas = &dict_schemas; + + let families = detect_families( + &tree, + &DetectOptions { + excluded_stems: opts.no_collapse.clone(), + }, + ); + // Set of node labels whose per-N sub-proc we should NOT emit — + // they collapsed into a family constructor. Their sub-nodes + // (`MAC_PORT0_RX`, etc.) still emit; only the direct-param + // per-N node vanishes. + let collapsed_labels: std::collections::HashSet = families + .iter() + .flat_map(|f| { + f.indices.iter().map(move |i| stem_index_label(&f.stem, *i)) + }) + .collect(); + + // Collect every node that will emit a proc — the root for the + // top-level `::create` and every non-root node that has at + // least one direct parameter to configure AND hasn't been + // collapsed into a family. + let all_nodes = tree.collect(); + let emit_nodes: Vec<&Node> = all_nodes + .iter() + .copied() + .filter(|n| n.label.is_empty() || !n.direct.is_empty()) + .filter(|n| !collapsed_labels.contains(&n.label)) + .collect(); + + // Family-side lookups the top-proc emitter uses. + let family_merges: Vec> = families + .iter() + .map(|f| FamilyMerge { + stem: f.stem.clone(), + stem_lower: lowercase_ident(&f.stem), + indices: f.indices.clone(), + newtype_qualified: stem_props_name(&ip_name, &f.stem), + marker: std::marker::PhantomData, + }) + .collect(); + + let dict_schema_newtypes = + build_dict_schema_newtypes(&ip_name, dict_schemas); + + let mut out = String::new(); + emit_file_header(&mut out, component, &vlnv); + // Emit newtype declarations + constructor procs for XML-derived + // top-level dict schemas. These live flat under `::` (like + // PS_PMC_CONFIG) — the CSV-driven ones the caller supplied get + // emitted by `append_dict_sub_procs` at the tail of `generate`. + // Emitting them here keeps the top-proc's typed arg references + // (`hnic_pipe_parameters: ::HnicPipeParameters`) resolvable + // during validation. + for p in &tree.direct { + let Some(schema) = dict_schemas.get(&p.name) else { + continue; + }; + // Skip CSV-driven schemas (`append_dict_sub_procs` emits + // those at the tail). We differentiate two rails: + // * XML-derived schemas — populated above from `tree.direct` + // via `is_properties_shaped_param` OR via the + // model-param anchor path. Both are keyed under + // `p.name`; caller's original `dict_schemas` didn't + // hold them yet. We check by whether the schema has + // content — CSV and XML both do, so that's ambiguous. + // * Anchor-derived: `p` isn't structurally Properties + // but a sibling model param is. Emit iff we DID pick + // up an anchor for it. + // Simplest gate: emit whenever `dict_schemas` has an + // entry that wasn't in the original caller-supplied map. + // We express that indirectly via the two-track OR: + // Properties-shaped (usual path) OR model-param anchor + // present (INTF_PARENT_PIN_LIST-shaped path). + let has_anchor = model_param_anchor_default(component, &p.name) + .as_deref() + .is_some_and(|d| { + !crate::paired_list::parse_paired_list(d).is_empty() + }); + if !is_properties_shaped_param(p) && !has_anchor { + continue; + } + writeln!(&mut out).unwrap(); + emit_dict_props_prelude(&mut out, &ip_name, &[], &p.name); + writeln!(&mut out).unwrap(); + emit_dict_sub_proc(&mut out, &ip_name, &[], &p.name, schema, opts); + } + writeln!( + out, + "## {} configurable parameter{} across {} proc{}.", + parameters.len(), + if parameters.len() == 1 { "" } else { "s" }, + emit_nodes.len(), + if emit_nodes.len() == 1 { "" } else { "s" } + ) + .unwrap(); + writeln!(out, "##").unwrap(); + writeln!(out, "## Usage:").unwrap(); + writeln!(out, "## set cell [{top_proc} ]").unwrap(); + writeln!( + out, + "## $cell - ... ;# tab-complete by prefix" + ) + .unwrap(); + + // `configure`'s arg-doc — the whole documented kwarg pile + // (direct + family + split-node). Zero cell handles, zero + // `-name`/`-bd`. The old inline-`create` doc had all of these + // mixed with `-name`/`-bd`; the split lifts them out into + // `configure` so `::create`'s surface stays tiny. + let mut configure_doc = Doc::new(); + for p in &tree.direct { + emit_arg_decl( + &mut configure_doc, + component, + presets, + p, + opts, + "", + &ip_name, + dict_schemas, + &[], + ); + } + // Family kwargs: one `-` per family member, + // typed as the newtype, with a doc comment referencing the + // constructor via `[…]` for semantic-goto. + // + // `@default("")` is a placeholder — the analyzer's + // `@default(...)` grammar rejects bracket-expressions like + // `[::empty]`, so we can't declare the semantically-correct + // default in the annotation. The body's `__vw_kw__set` + // guard skips the merge loop when the caller didn't pass the + // slot, so `$` is never dereferenced with the placeholder + // value — the empty string never reaches the newtype machinery. + if !families.is_empty() { + if !tree.direct.is_empty() { + configure_doc.push(Item::Blank); + } + for f in &families { + let ctor = format!("{ip_name}::{}", lowercase_ident(&f.stem)); + let ty = stem_props_name(&ip_name, &f.stem); + for i in &f.indices { + let arg = format!("{}{i}", lowercase_ident(&f.stem)); + configure_doc.push(Item::DocComment(format!( + "Configuration for {} slot {i}. Construct with [{ctor}].", + f.stem + ))); + configure_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{arg}: {ty}")), + ], + body: None, + })); + } + } + } + // Family newtype preludes ship at the FILE TOP LEVEL — inside + // `namespace eval ` blocks, the analyzer double-prefixes + // qualified proc names (a proc `::T::from` inside `namespace + // eval ` becomes `::::T::from`, so external + // references never find it). Keeping them at top level with + // fully-qualified names avoids that and remains legal now that + // the validator accepts qualified newtype names (see Slice 4's + // changes to `validate::reject_nested_qualified`). An empty + // `namespace eval :: {}` prelude keeps Tcl happy. + if !families.is_empty() { + for f in &families { + writeln!(out).unwrap(); + emit_family_prelude(&mut out, &ip_name, f); + } + writeln!(out).unwrap(); + } + + // One value-constructor per non-root node that has direct + // parameters. Each is a pure function: takes typed field + // kwargs, returns a `::` newtype value that the + // top proc composes into its atomic `set_property -dict` call. + // + // `emit_nodes` already excludes labels collapsed into a + // family — those emit through `emit_family_constructor` above. + let split_nodes: Vec<&Node<'_>> = emit_nodes + .iter() + .filter(|n| !n.label.is_empty()) + .copied() + .collect(); + + // Split-shape newtype preludes are emitted INSIDE each + // split-node's own subfile (see `emit_split_node_constructor`) + // rather than up here at module.htcl top-level. That way + // opening `intf7.htcl` standalone in the LSP still finds the + // `gtwiz_versal::Intf7Props` declaration the file needs. + // module.htcl still sees them because it `src`s each subfile + // via the `## ==== split-file: ... ==== ##` peel-off pass. + + // Split-node kwargs on configure: one per split-shape + // constructor, typed as the node's newtype so configure + // composes ALL configuration into one Config value. + // Semantic-ref doc comment points at the constructor for + // goto/hover. + if !split_nodes.is_empty() { + if !tree.direct.is_empty() || !families.is_empty() { + configure_doc.push(Item::Blank); + } + for n in &split_nodes { + let ctor_suffix = sanitize_ident(&n.label.to_ascii_lowercase()); + let ctor = format!("{ip_name}::{ctor_suffix}"); + let ty = split_props_name(&ip_name, &n.label); + configure_doc.push(Item::DocComment(format!( + "Configuration for the {} sub-tree. Construct with \ + [{ctor}].", + n.label + ))); + configure_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("{ctor_suffix}: {ty}")), + ], + body: None, + })); + } + } + + // Config prelude at file top level (namespace-eval double- + // prefix workaround). Emit BEFORE the namespace-eval block + // opens so `::Config` is available when the enclosed + // `::configure` returns it and `::create` accepts it. + writeln!(out).unwrap(); + emit_config_prelude(&mut out, &ip_name); + writeln!(out).unwrap(); + + // `configure` body: pure dict assembly + wrap. No cell handle, + // no `set_property`, no `-bd` branch. + let mut configure_body = String::new(); + write_dict_assembly_with_splits( + &mut configure_body, + component, + &tree.direct, + &family_merges, + &dict_schema_newtypes, + &split_nodes, + &ip_name, + ); + let config_ty = config_name(&ip_name); + // Lift flat `CONFIG.` keys into a nested tagged Properties + // tree (see `build_single_configure_body` for the rationale). + writeln!( + configure_body, + "return [{config_ty}::from -v [Properties::from_dotted_pairs -v $_vw_d]]" + ) + .unwrap(); + + // `create`'s arg-doc — narrow surface: -name, -bd, -config. + let mut create_doc = Doc::new(); + create_doc.push(Item::DocComment( + "Project-level IP module name, or (when `-bd 1`) the \ + instance name in the block design." + .into(), + )); + create_doc.push(Item::Command(Command::call( + "name", + std::iter::empty::(), + ))); + push_bd_switch_arg(&mut create_doc); + create_doc.push(Item::DocComment(format!( + "Typed configuration value. Construct with [{ip_name}::configure]. \ + Defaults to an empty config (all parameters take their IP defaults)." + ))); + create_doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words: vec![ + Word::Raw("@default(\"\")".into()), + Word::Bare(format!("config: {config_ty}")), + ], + body: None, + })); + + let create_body = build_single_create_body(&vlnv, &ip_name); + + // Assemble the `namespace eval { … }` body in + // families → splits → configure → create order. + let mut procs = String::new(); + for (i, f) in families.iter().enumerate() { + if i > 0 { + writeln!(procs).unwrap(); + } + emit_family_constructor( + &mut procs, &ip_name, component, presets, opts, f, + ); + } + if !families.is_empty() { + writeln!(procs).unwrap(); + } + for (i, n) in split_nodes.iter().enumerate() { + if !families.is_empty() || i > 0 { + writeln!(procs).unwrap(); + } + // The `out` buffer collects dict-schema newtypes and + // constructor procs for this node's Properties args. They + // live at fully-qualified names like + // `gtwiz_versal::intf0::ChannelMap::from`, which Tcl only + // resolves correctly when NOT inside a `namespace eval + // { … }` block. So `emit_split_node_constructor` writes them + // to the outer `out` buffer (which is emitted at the top + // level of the file, before `write_namespace_block` wraps + // `procs`). + emit_split_node_constructor( + &mut procs, &mut out, &ip_name, component, presets, opts, n, + ); + } + if !families.is_empty() || !split_nodes.is_empty() { + writeln!(procs).unwrap(); + } + let configure_body = + wrap_body_qualified_prefix(&configure_body, &ip_name, "configure"); + emit_proc( + &mut procs, + "configure", + &configure_doc, + Some(&config_ty), + &configure_body, + ); + writeln!(procs).unwrap(); + emit_proc( + &mut procs, + "create", + &create_doc, + Some("string"), + &create_body, + ); + + write_namespace_block(&mut out, &ip_name, &procs); + out +} + +/// Emit the newtype prelude for a split-shape node. Same shape as +/// [`emit_family_prelude`] / [`emit_dict_props_prelude`] — one +/// `namespace eval {}` + `type = Properties` + the four +/// helper procs. Consumed by [`emit_split_node_constructor`] and +/// the top-proc merge loop in +/// [`write_set_property_dict_with_splits`]. +/// Emit the top-level `::Config = Properties` newtype at file +/// top level (outside `namespace eval {…}` to sidestep the +/// analyzer's double-prefix bug on qualified proc names inside +/// namespace-eval blocks). Structural mirror of +/// [`emit_split_props_prelude`] / [`emit_family_prelude`] / +/// [`emit_dict_props_prelude`] — same four helpers (`empty`, +/// `from`, `to`, `repr`) with identity implementations. The name +/// is always `::Config` — one per generated wrapper — so the +/// callsite pattern `set cfg [::configure -foo x]` returns a +/// value the analyzer knows about and `::create` can accept +/// via its typed `-config ::Config` param. +fn emit_config_prelude(out: &mut String, ip_name: &str) { + let qualified = config_name(ip_name); + writeln!( + out, + "## Typed configuration value for [{ip_name}::create]. \ + Construct with [{ip_name}::configure].", + ) + .unwrap(); + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + // Values are a properly nested tagged Properties tree by the + // time they reach Config (see `build_single_configure_body` — + // configure's return path lifts through + // `Properties::from_dotted_pairs`). Delegate to + // `Properties::repr` for uniform tagged-Property rendering — + // same `KEY Scalar(VALUE)` / `KEY Nested(…)` shape the REPL's + // syntax highlighter colours specially. + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Qualified name of the top-level Config newtype for `ip_name`. +fn config_name(ip_name: &str) -> String { + format!("{ip_name}::Config") +} + +fn emit_split_props_prelude(out: &mut String, ip_name: &str, label: &str) { + let qualified = split_props_name(ip_name, label); + let ctor_lower = label.to_ascii_lowercase(); + writeln!( + out, + "## Typed configuration value for [{ip_name}::create]'s \ + `-{ctor_lower}` slot. Construct with [{ip_name}::{ctor_lower}].", + ) + .unwrap(); + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {qualified} {{}}").unwrap(); + writeln!(out, "type {qualified} = Properties").unwrap(); + writeln!( + out, + "proc {qualified}::repr {{ v: {qualified} }} string \ + {{ return [Properties::repr -v [{qualified}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::from {{ v: Properties }} {qualified} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::to {{ v: {qualified} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {qualified}::empty {{}} {qualified} \ + {{ return [{qualified}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Emit the value-constructor for a split-shape node. Bare proc +/// name (``) — becomes `::` via the +/// enclosing `namespace eval { … }`. Pure — no cell handle, +/// no `-bd`, no `set_property`. Body builds a `Properties`-shaped +/// dict from the supplied field kwargs (index-stripped keys) and +/// wraps in the newtype. +fn emit_split_node_constructor( + out: &mut String, + // Buffer for output that must live OUTSIDE the ip-scoped + // `namespace eval { … }` block that wraps `out`. Dict-schema + // sub-procs (which use fully-qualified names like + // `gtwiz_versal::intf0::ChannelMap::from`) go here — placing them + // inside the `namespace eval` block causes Tcl to double the ip + // prefix (`gtwiz_versal::gtwiz_versal::intf0::…`) and the + // procs become undiscoverable. + outer_out: &mut String, + ip_name: &str, + component: &Component, + presets: &crate::presets::PresetMap, + opts: &GenerateOptions, + n: &Node<'_>, +) { + let ctor_local = sanitize_ident(&n.label.to_ascii_lowercase()); + let ret_ty = split_props_name(ip_name, &n.label); + + let mut doc = Doc::new(); + doc.push(Item::DocComment(format!( + "Configuration value for [{ip_name}::create]'s \ + `-{ctor_local}` slot ({} sub-tree). Composes into the top \ + proc so every provided field lands in ONE atomic \ + `set_property -dict` call.", + n.label + ))); + if !n.direct.is_empty() { + doc.push(Item::Blank); + } + // Build the XML-driven dict-schema map for this split-node's + // Properties-shaped params (channel_map, gt_settings, etc. on an + // `intf` node). Each schema gets emitted as a nested-namespace + // typed sub-proc under `::::`, and the arg-decl + // references the typed newtype instead of raw `Properties`. + // See [`build_split_dict_schemas`] for the extraction logic. + let node_local = sanitize_ident(&n.label.to_ascii_lowercase()); + let local_dict_schemas = build_split_dict_schemas(n, &node_local, opts); + for p in &n.direct { + emit_arg_decl( + &mut doc, + component, + presets, + p, + opts, + &n.label, + ip_name, + &local_dict_schemas, + &[node_local.as_str()], + ); + } + + // Split-file marker: everything between OPEN and CLOSE ends up + // in a sibling `.htcl` file (see `split_into_files`) with the + // main module.htcl gaining a `src ./` line at this + // spot. Basename derives from the split-node's local ident so + // gtwiz-versal's 8 intfN nodes land in `intf0.htcl`…`intf7.htcl`. + // For small IPs the marker still emits but every subfile stays + // tiny (a handful of lines) which the tree-sitter parser handles + // instantly. + writeln!(outer_out, "## ==== split-file: {node_local}.htcl ==== ##") + .unwrap(); + // Every subfile needs its own `src @vivado-cmd` so it can be + // analyzed (and hovered / go-to-def'd) standalone in the LSP + // without the analyzer opening it via module.htcl. The dict- + // schema prelude references `Properties::repr`, `Properties::empty`, + // and similar procs that live in the vivado-cmd library; without + // the src line the analyzer flags every reference as undefined. + writeln!(outer_out, "src @vivado-cmd").unwrap(); + writeln!(outer_out).unwrap(); + + // Split-shape newtype prelude for THIS node — moved from + // module.htcl into the subfile so opening the subfile + // standalone in the LSP finds the type declaration the file's + // own return-type annotations reference. + emit_split_props_prelude(outer_out, ip_name, &n.label); + + // Emit the per-node typed sub-procs first so their newtypes + // are visible before this proc's signature references them. + // Each sub-proc lives at `::::`; + // its newtype at `::::`. + // Placed on the outer buffer so the fully-qualified proc names + // (`gtwiz_versal::intf0::…`) don't get an accidental extra + // `gtwiz_versal::` prefix from Tcl's namespace resolution. + emit_split_dict_sub_procs( + outer_out, + ip_name, + &node_local, + &n.label, + &n.direct, + &local_dict_schemas, + opts, + ); + + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for p in &n.direct { + let field_key = strip_prefix(&p.name, &n.label); + let arg = lowercase_ident(field_key); + // Dict-schema-backed slot: unwrap the typed newtype to its + // raw paired dict via `::to`. Other Properties-shaped + // slots (none in gtwiz-versal after this change, but the + // path still handles legacy IPs whose defaults slip past + // the schema extractor) arrive as raw paired dicts; store + // `$arg` verbatim without `Properties::to_raw` (which would + // try to strip Scalar/Nested tags that aren't present). + let value_expr = if local_dict_schemas.contains_key(field_key) { + let ret_ty = + dict_props_name(ip_name, &[node_local.as_str()], field_key); + format!("[{ret_ty}::to -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Split-node scalar bool field — pure or pseudo-bool + // (kind picks the wire form). Same `if`-shape as the + // top-proc and composed-family paths so `$arg` stays + // a whole-word VarRef the usage walker sees. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the dict-set in a catch prepending the HTCL arg + // name — see the sibling emission in + // `emit_dict_sub_proc` for the diagnostic-path + // rationale. + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + dict set _vw_d {field_key} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n \ + }}" + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + // Emit the split-node's own constructor proc into the outer + // buffer (same file as its dict-schema sub-procs) with a fully + // qualified name — so it lives at `::` without + // needing to sit inside a `namespace eval {…}` block. That + // lets the whole split-node emission end up in ONE sibling file + // wrapped by split markers, instead of being torn between the + // main-file namespace-block and the outer-scope schema block. + let ctor_qualified = format!("{ip_name}::{ctor_local}"); + emit_proc(outer_out, &ctor_qualified, &doc, Some(&ret_ty), &body); + + // Split-file close marker — see the matching OPEN above and + // `split_into_files` for the peel logic. + writeln!( + outer_out, + "## ==== end split-file: {node_local}.htcl ==== ##" + ) + .unwrap(); + + // Consume `out` (the namespace-block buffer) so the reader can + // see we intentionally skipped writing into it — the split-node + // no longer contributes short-name procs there. + let _ = out; +} + +/// Build the XML-driven DictSchema map for a split-node's +/// Properties-shaped direct params. Runs once per split-node +/// emission; the returned map is keyed by raw IP-XACT parameter +/// name (matches the arg-decl / body lookup pattern). +/// +/// The shape-path passed to `DictSchema::from_paired_default` is +/// `::`, where `stem` is the un-indexed stem of +/// the node's label (`INTF0` → `intf`, `QUAD0_CH0` → `quad_ch`), +/// so overrides written against `intf::gt_settings` apply across +/// every `intf0`..`intf7` sibling. Matches the "one override entry +/// per family" convention documented in +/// [`crate::overrides::OverridesFile`]. +fn build_split_dict_schemas( + n: &Node<'_>, + _node_local: &str, + opts: &GenerateOptions, +) -> std::collections::HashMap { + let stem_lower = split_stem_lower(&n.label); + let mut out = std::collections::HashMap::new(); + + // Pass 1 — extract each Properties-shaped param's own schema + // from its default value. Empty results (from `0` / `""` / + // `NA NA` sentinel defaults) still land in the map so pass 2 + // can decide whether to synthesize from an anchor. + for p in &n.direct { + if !is_properties_shaped_param(p) { + continue; + } + let field_key = strip_prefix(&p.name, &n.label).to_string(); + let field_lower = lowercase_ident(&field_key); + let shape_path = if stem_lower.is_empty() { + field_lower + } else { + format!("{stem_lower}::{field_lower}") + }; + let schema = crate::DictSchema::from_paired_default( + p.value.default_value(), + &shape_path, + &opts.overrides, + ); + out.insert(field_key, schema); + } + + // Pass 2 — resolve schemas whose own default was uninformative. + // The gtwiz-versal LR{n}_SETTINGS / GT_SETTINGS / GT_INTERNAL + // shapes all share a field vocabulary that only lives inside + // TXRX_OPTIONAL_PORTS's `INTF_LR_SETTINGS.LR0_SETTINGS` payload + // (see the Explore report). Find that payload once, then use it + // to backfill: + // - each trivial-schema `LR{n}_SETTINGS` slot (16 of them), + // - the tcldict-tagged `GT_SETTINGS` / `GT_INTERNAL` params, + // synthesized as wrappers holding 16 LR sub-slots. + let anchor_lr = out + .get("TXRX_OPTIONAL_PORTS") + .and_then(|s| s.sub_schemas.get("INTF_LR_SETTINGS")) + .and_then(|s| s.sub_schemas.get("LR0_SETTINGS")) + .cloned(); + let params_by_key: std::collections::HashMap<&str, &&Parameter> = n + .direct + .iter() + .map(|p| (strip_prefix(&p.name, &n.label), p)) + .collect(); + if let Some(lr_template) = anchor_lr.as_ref() { + for i in 0..16 { + let key = format!("LR{i}_SETTINGS"); + // Only backfill if the slot exists on the node (LR0..15 + // are all declared params on gtwiz-versal's intf nodes). + if !params_by_key.contains_key(key.as_str()) { + continue; + } + // Always replace with the anchor template — the LR* + // params' own defaults are Xilinx sentinels (`NA NA` + // extracts to a bogus `NA=NA` field, hardly trivial by + // fields.is_empty() but useless as a schema). The + // TXRX_OPTIONAL_PORTS anchor carries the real field + // vocabulary that Vivado actually accepts. + let mut copy = lr_template.clone(); + // The anchor's fields were built under the txrx shape + // path; reapply the destination slot's overrides so + // `intf::lrN_settings` gets its own refinements. + let dst_shape_path = if stem_lower.is_empty() { + lowercase_ident(&key) + } else { + format!("{stem_lower}::{}", lowercase_ident(&key)) + }; + copy.reapply_overrides(&dst_shape_path, &opts.overrides); + out.insert(key, copy); + } + // Synthesize wrappers for tcldict GT_SETTINGS / + // GT_INTERNAL. Both take the same LR0..LR15 sub-slot shape + // (Vivado accepts `CONFIG.INTF*_GT_SETTINGS(LR_SETTINGS) + // {...}` — the parenthesized sub-key IS the wrapper slot). + for wrapper_key in ["GT_SETTINGS", "GT_INTERNAL"] { + let Some(p) = params_by_key.get(wrapper_key) else { + continue; + }; + if !p.has_parameter_type("tcldict") { + continue; + } + let existing = out.get(wrapper_key); + if existing.is_some_and(|s| !schema_is_trivial(s)) { + continue; + } + let mut sub = std::collections::BTreeMap::new(); + for i in 0..16 { + let mut copy = lr_template.clone(); + // Sub-slot's shape path is `::::lrN_settings` + // — refines overrides written for that specific path + // (e.g. `intf::gt_settings::lr0_settings`), separate + // from the top-level `intf::lrN_settings` slot. + let wrapper_lower = wrapper_key.to_ascii_lowercase(); + let lr_lower = format!("lr{i}_settings"); + let sub_shape_path = if stem_lower.is_empty() { + format!("{wrapper_lower}::{lr_lower}") + } else { + format!("{stem_lower}::{wrapper_lower}::{lr_lower}") + }; + copy.reapply_overrides(&sub_shape_path, &opts.overrides); + sub.insert(format!("LR{i}_SETTINGS"), copy); + } + out.insert( + wrapper_key.to_string(), + crate::DictSchema { + fields: Vec::new(), + sub_schemas: sub, + }, + ); + } + } + + // Drop trivial schemas — params whose default was `0` / `""` + // sentinels with no anchor available. Fall back to raw + // Properties for those; a bare typed slot with no fields + // would just clutter the LSP surface. + out.retain(|_, schema| !schema_is_trivial(schema)); + out +} + +/// A schema is "trivial" when it has no fields AND no sub-slots — +/// derived from an empty default (`0`, `""`) or a nonsense `NA NA` +/// where the parser found no meaningful structure. Callers use +/// this to decide whether to backfill from an anchor param. +fn schema_is_trivial(s: &crate::DictSchema) -> bool { + s.fields.is_empty() && s.sub_schemas.is_empty() +} + +/// Emit the dict-schema sub-procs (prelude + constructor + any +/// nested sub-slots) for each entry in `dict_schemas`, all under +/// `::::`. Deep recursion happens inside +/// `emit_dict_sub_proc` via `emit_dict_sub_schemas`, so multi-level +/// XML shapes (`INTF0_TXRX_OPTIONAL_PORTS` → `INTF_LR_SETTINGS` → +/// `LR0_SETTINGS`) unfold naturally. +fn emit_split_dict_sub_procs( + out: &mut String, + ip_name: &str, + node_local: &str, + node_label: &str, + params: &[&Parameter], + dict_schemas: &std::collections::HashMap, + opts: &GenerateOptions, +) { + // Emit in the same order params appear so review diffs are + // deterministic and consumers can visually pair the newtype + // with its Properties arg in the outer proc. + for p in params { + let stripped = strip_prefix(&p.name, node_label); + let Some(schema) = dict_schemas.get(stripped) else { + continue; + }; + writeln!(out).unwrap(); + emit_dict_props_prelude(out, ip_name, &[node_local], stripped); + writeln!(out).unwrap(); + emit_dict_sub_proc(out, ip_name, &[node_local], stripped, schema, opts); + } +} + +/// Strip the trailing digit(s) from a split-node label and +/// lowercase — `INTF0` → `"intf"`, `QUAD0_CH1` → `"quad_ch"`. +/// Used as the shape-path stem when looking up overrides so a +/// single override entry applies across every indexed instance in +/// the family. +fn split_stem_lower(label: &str) -> String { + let mut segs: Vec = Vec::new(); + for seg in label.split('_') { + // Strip a trailing digit run from each segment. Keeps + // multi-word stems (`QUAD_CH`) intact while collapsing + // `INTF0` → `INTF`, `QUAD0_CH1` → `QUAD_CH`. + let trimmed = seg.trim_end_matches(|c: char| c.is_ascii_digit()); + if trimmed.is_empty() { + continue; + } + segs.push(trimmed.to_ascii_lowercase()); + } + segs.join("_") +} + +/// Fully-qualified newtype name for a split-shape node. +/// `("dcmac", "MAC_PORT0_RX")` → `"dcmac::MacPort0RxProps"`. +fn split_props_name(ip_name: &str, label: &str) -> String { + format!("{ip_name}::{}", split_props_local(label)) +} + +fn split_props_local(label: &str) -> String { + let mut out = String::new(); + for seg in label.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out.push_str("Props"); + out +} + +/// Extended top-proc dict writer that also merges split-shape +/// value-constructor outputs into the atomic dict. Mirrors +/// [`write_set_property_dict`]'s structure but weaves the +/// split-node merges in between the top-level knobs, family +/// merges, and the finalization. +#[allow(clippy::too_many_arguments)] +fn write_dict_assembly_with_splits( + out: &mut String, + component: &Component, + parameters: &[&Parameter], + families: &[FamilyMerge<'_>], + dict_schema_newtypes: &std::collections::HashMap, + split_nodes: &[&Node<'_>], + ip_name: &str, +) { + writeln!(out, "set _vw_d [list]").unwrap(); + for p in parameters { + let arg = lowercase_ident(&p.name); + // Type-driven value unwrap: + // - Dict-schema newtype: the constructor stores bare-string + // values in a paired-list dict (see + // [`emit_dict_sub_proc`]), which is EXACTLY what Vivado + // expects at `CONFIG.`. So just unwrap the newtype + // via `::to` — do NOT pipe through `Properties::to_raw`, + // which would try to dispatch on `Property::Scalar`/ + // `Nested` tags our stored values don't carry. + // - Plain Properties (paired-dict-shaped default without a + // schema): assume the caller passed a properly-tagged + // Properties value; unwrap through `Properties::to_raw`. + // - Scalar: `$arg` as-is. + let value_expr = + if let Some(newtype) = dict_schema_newtypes.get(&p.name) { + format!("[{newtype}::to -v ${arg}]") + } else if is_properties_shaped_param(p) { + // Properties-typed args now arrive as tagged trees + // (from other IPs' `configure` procs, or extracted + // via `dict get $props CONFIG` + `Property::as_nested` + // by the caller). Unwrap tags to a raw paired list + // via `Properties::to_raw` so + // `Properties::from_dotted_pairs` (which lifts the + // whole `_vw_d` at the end of the configure body) + // sees consistent bare-string values across all + // sub-slots. Without this the tagged-tree elements + // would confuse the shim's structural lifter. + format!("[Properties::to_raw -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Marshal the HTCL bool literal to whichever + // lexical form the IP-XACT declaration expects: + // * `Pure` (`spirit:format="bool"`) → true/false + // * `Pseudo` (long+choice{0,1}) → 1/0 + // Uses `[if $arg …]` (not `[expr {$arg?…}]`) so + // `$arg` appears as a whole-word VarRef in the + // emitted AST — the brace-wrapped form would put + // the reference inside a braced-text part where + // the analyzer's usage walker can't see it, + // producing spurious "unused proc arg" warnings + // for every bool arg. Defense-in-depth for + // callers reaching here via `extern::` (analyzer + // bypassed): Tcl's `if` still coerces via + // `string is boolean` semantics on non-canonical + // input. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the merge in a `catch` prepending the HTCL arg + // name on error — mirrors the sub-proc emission in + // `emit_dict_sub_proc`. Only nested-Properties args + // actually risk erroring here (`Properties::to_raw` + // walks a tagged tree that might have a mistyped + // leaf); scalar/bool args store bare strings that + // can't fail at `lappend` time, so the catch is + // no-op-cheap for them. + writeln!( + out, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + lappend _vw_d CONFIG.{name} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n\ + }}", + name = p.name, + ) + .unwrap(); + } + // Composed-family merges. + for fam in families { + for idx in &fam.indices { + let arg = format!("{}{}", fam.stem_lower, idx); + let prefix = format!("CONFIG.{}{}_", fam.stem, idx); + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{}::to -v ${arg}] {{", + fam.newtype_qualified + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v") + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + } + // Split-shape merges — each provided value-constructor result + // gets its dict entries flattened into the atomic dict with + // the `CONFIG._` prefix so property names round- + // trip to Vivado exactly as they were in IP-XACT. + for n in split_nodes { + let arg = sanitize_ident(&n.label.to_ascii_lowercase()); + let prefix = format!("CONFIG.{}_", n.label); + let newtype = split_props_name(ip_name, &n.label); + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{newtype}::to -v ${arg}] {{" + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v").unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + // No finalization here — callers wrap the assembled `_vw_d` + // in a typed `::Config` value. The `set_property` finalize + // has moved to `emit_config_finalize` on the `::create` + // side, which unwraps the caller's `-config` param and applies. +} + +// --------------------------------------------------------------------------- +// Shared helpers. +// --------------------------------------------------------------------------- + +/// Break the concatenated generator output into one main file plus +/// zero-or-more sibling files. +/// +/// Peel rule: scan the aggregate for the split marker +/// `## ==== split-file: ==== ##` … (matching close marker). +/// The interior lands in `subfiles[]`; the open/close markers +/// in `main` become a `src ./.htcl` line so the main file's +/// downstream references still resolve at load time. +/// +/// Content without markers stays in `main` untouched. Callers that +/// don't want the split at all (unit tests, single-file consumers) +/// use `MultiFileOutput::into_single()` to re-flatten. +/// +/// Threshold-based auto-splitting layers on top: the emitters wrap +/// each large split-node in markers so this pass does the physical +/// separation. Small nodes don't get markers → they stay inline. +fn split_into_files(aggregate: String) -> MultiFileOutput { + const OPEN: &str = "## ==== split-file: "; + const CLOSE: &str = "## ==== end split-file: "; + let mut main = String::with_capacity(aggregate.len()); + let mut subfiles: Vec<(String, String)> = Vec::new(); + let mut rest = aggregate.as_str(); + while let Some(open_off) = rest.find(OPEN) { + // Everything before the marker stays in main. + main.push_str(&rest[..open_off]); + rest = &rest[open_off + OPEN.len()..]; + let Some(open_end) = rest.find('\n') else { + // Malformed marker — retain the rest in main and bail. + main.push_str(OPEN); + main.push_str(rest); + return MultiFileOutput { main, subfiles }; + }; + let name_line = rest[..open_end].trim(); + // Strip the trailing ` ==== ##` from the marker line. The + // suffix has interleaved whitespace, ``#``, and ``=``; strip + // them all in one pass so partial-suffix boundaries (like + // "==== " with the trailing space blocking the "====" strip) + // don't leak into the filename. + let name = name_line + .trim_end_matches(|c: char| { + c == '#' || c == '=' || c.is_whitespace() + }) + .trim(); + rest = &rest[open_end + 1..]; + // Find the matching close marker. + let close_needle = format!("{CLOSE}{name}"); + let Some(close_off) = rest.find(&close_needle) else { + // Unpaired open — restore in main and stop splitting. + main.push_str(OPEN); + main.push_str(name_line); + main.push('\n'); + main.push_str(rest); + return MultiFileOutput { main, subfiles }; + }; + let body = rest[..close_off].to_string(); + // Advance past the close marker's line (up to and + // including its newline). + let after = &rest[close_off..]; + let close_line_end = + after.find('\n').map(|n| n + 1).unwrap_or(after.len()); + rest = &after[close_line_end..]; + // Emit the src line into main so the loader still walks + // this subfile. Uses a relative `./` path — the CLI + // writes the subfile as a sibling of the main output. + writeln!(main, "src ./{name}").unwrap(); + subfiles.push((name.to_string(), body)); + } + main.push_str(rest); + MultiFileOutput { main, subfiles } +} + +fn emit_file_header(out: &mut String, component: &Component, vlnv: &str) { + // Pull in the whole `vivado-cmd` library — the body uses + // `vivado_cmd::create_bd_cell` and `vivado_cmd::set_property` + // alongside `ip::check`, so `src @vivado-cmd/ip` (just the + // ip sub-module) leaves those references unresolved. The + // analyzer reports the unbound calls; sourcing the full + // package brings everything the emitted body actually uses + // into scope. + writeln!(out, "src @vivado-cmd").unwrap(); + writeln!(out).unwrap(); + writeln!(out, "ip::check -name \"{vlnv}\"").unwrap(); + writeln!(out).unwrap(); + if let Some(desc) = + component.description.as_deref().filter(|s| !s.is_empty()) + { + // Split the IP-XACT description into a one-sentence summary + // plus body so an LSP client can show a short blurb on hover + // / completion without repeating it in the documentation + // popup. Same shape as the cmd-doc generator. + let raw: Vec = + desc.lines().map(|l| l.trim_end().to_string()).collect(); + let summary = vw_htcl::doc::brief(&raw); + let extended = vw_htcl::doc::extended(&raw); + if let Some(s) = summary { + for line in vw_htcl::doc::wrap_paragraph(&s, 78) { + writeln!(out, "## {line}").unwrap(); + } + } + if let Some(body) = extended { + for paragraph in body.split("\n\n") { + writeln!(out, "##").unwrap(); + for line in vw_htcl::doc::wrap_paragraph(paragraph, 78) { + writeln!(out, "## {line}").unwrap(); + } + } + } + writeln!(out, "##").unwrap(); + } + writeln!(out, "## Source IP-XACT: {vlnv}").unwrap(); +} + +/// Emit `proc { } ? { }` with the args +/// and body indented two spaces each. When `return_type` is Some, +/// emits it as the 4th htcl word between args and body. +fn emit_proc( + out: &mut String, + name: &str, + args: &Doc, + return_type: Option<&str>, + body: &str, +) { + let args_text = args.to_string(); + writeln!(out, "proc {name} {{").unwrap(); + for line in args_text.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + match return_type { + Some(ty) => { + let needs_brace = ty.chars().any(char::is_whitespace); + if needs_brace { + writeln!(out, "}} {{{ty}}} {{").unwrap(); + } else { + writeln!(out, "}} {ty} {{").unwrap(); + } + } + None => { + writeln!(out, "}} {{").unwrap(); + } + } + for line in body.lines() { + if line.is_empty() { + writeln!(out).unwrap(); + } else { + writeln!(out, " {line}").unwrap(); + } + } + writeln!(out, "}}").unwrap(); +} + +/// Emit the paired-list assembly loops that build `_vw_d` from +/// direct parameters + composed families + dict-schema newtypes. +/// Arg names are built by stripping `prefix_to_strip` from each +/// parameter's full IP-XACT name; the `CONFIG.` key keeps +/// the full name Vivado expects. +/// +/// Callers: `::configure`'s body (both single and split shapes). +/// The output is a plain `_vw_d` paired list — no finalize, no +/// `set_property` call. `configure` wraps the assembled dict in +/// `[::Config::from -v [Properties::from -v $_vw_d]]` and +/// returns it; `::create` unwraps and applies. See +/// [`emit_config_finalize`] for the corresponding apply side. +fn write_dict_assembly( + out: &mut String, + component: &Component, + parameters: &[&Parameter], + prefix_to_strip: &str, + // Composed families to merge atomically into the same dict. + // Empty for sub-procs; populated on the top proc when the + // generator has collapsed indexed sibling groups into + // `::` constructors + `-` + // kwargs. Each family's per-index kwarg is unwrapped through + // its newtype's `::to` + `Properties::to_raw` and merged into + // `_vw_d` with `CONFIG._` keys. + families: &[FamilyMerge<'_>], + // Dict-schema newtype names, keyed by IP-XACT param name + // (e.g. `PS_PMC_CONFIG` → `versal_cips::PsPmcConfig`). When a + // parameter here matches a top-level top-proc param, its + // value_expr gets the `[::to -v $arg]` unwrap injected + // BEFORE the `Properties::to_raw` step so the type-check + // passes at compile time. Empty for sub-procs / single-shape + // IPs without any schemas. + dict_schema_newtypes: &std::collections::HashMap, +) { + // Build the dict conditionally so only user-supplied args reach + // Vivado. See `emit_dict_proc` for the rationale — unconditionally + // setting all CONFIG.* properties re-validates the whole cell and + // Vivado rejects values whose declared defaults happen to be + // out-of-range for the cell's current state. The + // `__vw_kw__set` flag is set by `::vw::kwargs` (shim helper) + // only when the user passed a value for that arg. + writeln!(out, "set _vw_d [list]").unwrap(); + for p in parameters { + let arg = lowercase_ident(strip_prefix(&p.name, prefix_to_strip)); + // Type-driven value unwrap: + // - Dict-schema newtype (`versal_cips::PsPmcConfig` etc.): + // `[Properties::to_raw -v [::to -v $arg]]`. The extra + // `::to` step satisfies the type-checker at compile + // time; at runtime it's identity on the underlying + // Properties value. + // - Plain Properties (paired-dict-shaped default without + // a registered schema): `[Properties::to_raw -v $arg]`. + // - Scalar: `$arg`. + // Type-driven value unwrap: + // - Dict-schema newtype: the constructor stores bare-string + // values in a paired-list dict (see + // [`emit_dict_sub_proc`]), which is EXACTLY what Vivado + // expects at `CONFIG.`. So just unwrap the newtype + // via `::to` — do NOT pipe through `Properties::to_raw`, + // which would try to dispatch on `Property::Scalar`/ + // `Nested` tags our stored values don't carry. + // - Plain Properties (paired-dict-shaped default without a + // schema): assume the caller passed a properly-tagged + // Properties value; unwrap through `Properties::to_raw`. + // - Scalar: `$arg` as-is. + let value_expr = + if let Some(newtype) = dict_schema_newtypes.get(&p.name) { + format!("[{newtype}::to -v ${arg}]") + } else if is_properties_shaped_param(p) { + // Properties-typed args now arrive as tagged trees + // (from other IPs' `configure` procs, or extracted + // via `dict get $props CONFIG` + `Property::as_nested` + // by the caller). Unwrap tags to a raw paired list + // via `Properties::to_raw` so + // `Properties::from_dotted_pairs` (which lifts the + // whole `_vw_d` at the end of the configure body) + // sees consistent bare-string values across all + // sub-slots. Without this the tagged-tree elements + // would confuse the shim's structural lifter. + format!("[Properties::to_raw -v ${arg}]") + } else if let Some(kind) = bool_kind(component, p) { + // Marshal the HTCL bool literal to whichever + // lexical form the IP-XACT declaration expects: + // * `Pure` (`spirit:format="bool"`) → true/false + // * `Pseudo` (long+choice{0,1}) → 1/0 + // Uses `[if $arg …]` (not `[expr {$arg?…}]`) so + // `$arg` appears as a whole-word VarRef in the + // emitted AST — the brace-wrapped form would put + // the reference inside a braced-text part where + // the analyzer's usage walker can't see it, + // producing spurious "unused proc arg" warnings + // for every bool arg. Defense-in-depth for + // callers reaching here via `extern::` (analyzer + // bypassed): Tcl's `if` still coerces via + // `string is boolean` semantics on non-canonical + // input. + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + // Wrap the merge in a `catch` prepending the HTCL arg + // name on error — mirrors the sub-proc emission in + // `emit_dict_sub_proc`. Only nested-Properties args + // actually risk erroring here (`Properties::to_raw` + // walks a tagged tree that might have a mistyped + // leaf); scalar/bool args store bare strings that + // can't fail at `lappend` time, so the catch is + // no-op-cheap for them. + writeln!( + out, + "if {{${{__vw_kw_{arg}_set}}}} {{\n \ + if {{[catch {{\n \ + lappend _vw_d CONFIG.{name} {value_expr}\n \ + }} __vw_msg]}} {{\n \ + error \"{arg}.$__vw_msg\"\n \ + }}\n\ + }}", + name = p.name, + ) + .unwrap(); + } + // Composed-family merges — each provided `-` + // value gets unwrapped and its `FIELD → value` pairs merged + // into `_vw_d` with the `CONFIG._` prefix. Runs + // BEFORE the `if {llength} { set_property … }` finalization + // so the whole thing lands as ONE atomic call. + for fam in families { + for idx in &fam.indices { + let arg = format!("{}{}", fam.stem_lower, idx); + let prefix = format!("CONFIG.{}{}_", fam.stem, idx); + // Family constructors store bare-string values in + // their dict (see `emit_family_constructor`), so + // iterate the raw dict directly rather than going + // through `Properties::to_raw`. The latter would + // dispatch on `Property::Scalar`/`Nested` tags that + // our stored values don't carry — the constructor + // treats every field as a scalar and this loop has + // to match that convention. + writeln!(out, "if {{${{__vw_kw_{arg}_set}}}} {{").unwrap(); + writeln!( + out, + " foreach {{_vw_f _vw_v}} [{}::to -v ${arg}] {{", + fam.newtype_qualified + ) + .unwrap(); + writeln!(out, " lappend _vw_d \"{prefix}$_vw_f\" $_vw_v") + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); + } + } +} + +/// Emit the create-side finalize: unwrap `$config` through +/// `::Config::to`, flatten the nested tagged Properties tree +/// back into the flat `CONFIG. value` paired list +/// `set_property -dict` expects (via [`Properties::to_dotted_flat`]), +/// then apply against the cell handle. Two-mode `-bd` branch: +/// `bd=1` → target `$cell` directly (a bd_cell path); `bd=0` → +/// resolve via `[get_ips $name]` because `create_ip` returns an +/// XCI file path, not an IP handle. +fn emit_config_finalize(out: &mut String, ip_name: &str) { + let config_ty = config_name(ip_name); + writeln!( + out, + "set _dict [Properties::to_dotted_flat -v [{config_ty}::to -v $config]]" + ) + .unwrap(); + // Two-mode set_property target: + // - `-bd true`: `$handle` IS the `/…` bd_cell path, + // `set_property -objects` takes it directly. + // - `-bd false`: `$handle` is a bare module name; the IP + // object is fetched via `[get_ips $handle]`. + // Both branches are semantically identical up to the target + // resolution, matching the pre-collapse behavior. + writeln!(out, "if {{[llength $_dict] > 0}} {{").unwrap(); + writeln!(out, " if {{$bd}} {{").unwrap(); + writeln!( + out, + " vivado_cmd::set_property -dict $_dict -objects $handle" + ) + .unwrap(); + writeln!(out, " }} else {{").unwrap(); + writeln!( + out, + " vivado_cmd::set_property -dict $_dict -objects [get_ips $handle]" + ) + .unwrap(); + writeln!(out, " }}").unwrap(); + writeln!(out, "}}").unwrap(); +} + +// --------------------------------------------------------------------------- +// Indexed-family emission. +// --------------------------------------------------------------------------- + +/// Emit one arg-decl for a family constructor. Mirror of +/// [`emit_arg_decl`] with one addition: the `@enum(...)` list is +/// the **union** of enum choices across every provided member +/// (`per_member`), so the collapsed constructor accepts any value +/// any port takes. Defaults are shape-equal by construction (the +/// shape-match check requires it), so we use `shape_param`'s +/// default verbatim. +fn emit_arg_decl_family( + doc: &mut Doc, + component: &Component, + presets: &crate::presets::PresetMap, + shape_param: &Parameter, + per_member: &[&Parameter], + shape_member_label: &str, + opts: &GenerateOptions, +) { + if opts.include_descriptions { + if let Some(desc) = + shape_param.description.as_deref().filter(|s| !s.is_empty()) + { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + let mut words = Vec::new(); + // Union enum choices across every per-member Parameter that + // maps to this field. Preserve insertion order (IP-XACT first + // across members, presets after) so hover/completion shows a + // stable list. + let mut seen = std::collections::HashSet::new(); + let mut unioned: Vec = Vec::new(); + for p in per_member { + for v in enum_values_for(component, presets, p) { + if seen.insert(v.clone()) { + unioned.push(v); + } + } + } + if !unioned.is_empty() { + let formatted: Vec = + unioned.iter().map(|v| format_attribute_value(v)).collect(); + words.push(Word::Raw(format!("@enum({})", formatted.join(", ")))); + } + let default = shape_param.value.default_value(); + if !default.is_empty() { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } + let lowered = + lowercase_ident(strip_prefix(&shape_param.name, shape_member_label)); + let typed_name = if is_properties_shaped_param(shape_param) { + format!("{lowered}: Properties") + } else { + lowered + }; + words.push(Word::Bare(typed_name)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +/// Emit the newtype declaration and its `from`/`to`/`repr`/`empty` +/// helper procs for one family. Called from inside a +/// `namespace eval { … }` block, so identifiers use their +/// TOP-LEVEL forms — this fn is called at file top level, NOT +/// inside `namespace eval { … }`. Reason: the analyzer +/// double-prefixes qualified proc names declared inside a matching +/// namespace-eval block (`proc ::T::from` inside +/// `namespace eval { … }` becomes `::::T::from`), so +/// external references never resolve. Keeping the newtype block at +/// top level with fully-qualified names avoids that. Legal thanks +/// to this slice's `validate::reject_nested_qualified` update. +fn emit_family_prelude(out: &mut String, ip_name: &str, f: &IndexedFamily<'_>) { + let stem_props = stem_props_name(ip_name, &f.stem); + let stem_lower = lowercase_ident(&f.stem); + writeln!( + out, + "## Typed configuration slot for one [{ip_name}::{stem_lower}] on \ + [{ip_name}::create].", + ) + .unwrap(); + // Tcl requires the target namespace to exist before qualified + // proc names like `::::from` are legal. Nested + // `namespace eval` calls establish the whole chain. The outer + // `namespace eval {}` shape is idempotent — the proc-block + // emitted below re-enters the same namespace. + writeln!(out, "namespace eval {ip_name} {{}}").unwrap(); + writeln!(out, "namespace eval {stem_props} {{}}").unwrap(); + writeln!(out, "type {stem_props} = Properties").unwrap(); + writeln!( + out, + "proc {stem_props}::repr {{ v: {stem_props} }} string \ + {{ return [Properties::repr -v [{stem_props}::to -v $v]] }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::from {{ v: Properties }} {stem_props} \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::to {{ v: {stem_props} }} Properties \ + {{ return $v }}" + ) + .unwrap(); + writeln!( + out, + "proc {stem_props}::empty {{}} {stem_props} \ + {{ return [{stem_props}::from -v [Properties::empty]] }}" + ) + .unwrap(); +} + +/// Emit the family constructor `::` — a pure +/// value-builder that takes the same typed args the collapsed +/// per-N procs took, packs the supplied ones into a `Properties` +/// dict, and returns it wrapped in the newtype. NO Vivado side +/// effects; the atomic materialization happens later in +/// `::create`. +fn emit_family_constructor( + out: &mut String, + ip_name: &str, + component: &Component, + presets: &crate::presets::PresetMap, + opts: &GenerateOptions, + f: &IndexedFamily<'_>, +) { + // Emitted inside `namespace eval { … }` — use a bare + // proc name; Tcl resolves it to `::` at load + // time via the enclosing namespace. + let stem_lower = lowercase_ident(&f.stem); + let ret_ty = stem_props_name(ip_name, &f.stem); + + let mut doc = Doc::new(); + doc.push(Item::DocComment(format!( + "Configuration value for one of [{ip_name}::create]'s \ + `-{stem_lower}` slots. Composes into the top proc so \ + every provided slot's fields land in ONE atomic \ + `set_property -dict` call.", + ))); + doc.push(Item::Blank); + // Each field's `@enum(...)` values are the UNION of that + // field's per-member enum choices — the constructor has to + // accept any value any port can take. See ShapeSlot's docs + // for why choice_ref differences don't block the family. + for p in &f.shape { + let field_short = strip_prefix(&p.name, &f.shape_member_label); + let per_member: Vec<&Parameter> = f + .members_direct + .iter() + .zip(&f.member_labels) + .filter_map(|(direct, label)| { + direct + .iter() + .copied() + .find(|q| strip_prefix(&q.name, label) == field_short) + }) + .collect(); + emit_arg_decl_family( + &mut doc, + component, + presets, + p, + &per_member, + &f.shape_member_label, + opts, + ); + } + + // Body: build a dict from the supplied kwargs and wrap it. + let mut body = String::new(); + writeln!(body, "set _vw_d [dict create]").unwrap(); + for p in &f.shape { + let arg = lowercase_ident(strip_prefix(&p.name, &f.shape_member_label)); + // Post-strip key becomes the CONFIG._ suffix at + // top-proc merge time. Store un-prefixed field name here. + let field_key = strip_prefix(&p.name, &f.shape_member_label); + // Properties-typed sub-slots arrive as raw paired-list + // dicts; scalar sub-slots are plain strings. Both cases + // reduce to `$arg`. The old `Properties::to_raw` step + // required Property::Scalar/Nested tags that our new + // configure-based value flow no longer produces. + // + // Bool-typed sub-slots get the same marshaling as + // top-level bool args — the composed-family paired list + // ends up merged into the top proc's `_vw_d` via a + // `foreach`, so the canonical wire form (`true`/`false` + // for pure bool, `1`/`0` for pseudo bool per BoolKind) + // needs to be in place before that step. + let value_expr = if let Some(kind) = bool_kind(component, p) { + bool_value_expr(&arg, kind) + } else { + format!("${arg}") + }; + writeln!( + body, + "if {{${{__vw_kw_{arg}_set}}}} \ + {{ dict set _vw_d {field_key} {value_expr} }}" + ) + .unwrap(); + } + writeln!( + body, + "return [{ret_ty}::from -v [Properties::from -v $_vw_d]]" + ) + .unwrap(); + emit_proc(out, &stem_lower, &doc, Some(&ret_ty), &body); +} + +/// Reconstruct a family member's tree-node label from stem + index. +/// The `` naming (no separator) matches how +/// `tree::build_tree` produces `MAC_PORT0`, `MAC_PORT1`, …. +fn stem_index_label(stem: &str, idx: u32) -> String { + format!("{stem}{idx}") +} + +/// Fully-qualified newtype name from IP-name + uppercase-with- +/// underscores stem. `("dcmac", "MAC_PORT")` → `"dcmac::MacPortProps"`. +/// +/// Namespaced under the IP so `dcmac::` completion surfaces the +/// type alongside the constructor and top proc. Requires +/// vw-htcl's validator to accept qualified newtype names (landed +/// as part of this slice — see `validate::reject_nested_qualified`). +fn stem_props_name(ip_name: &str, stem: &str) -> String { + format!("{ip_name}::{}", stem_props_local(stem)) +} + +/// Local (unqualified) newtype segment — the PascalCase stem + +/// `Props`. Used inside `namespace eval { … }` where bare +/// names are preferred; the outer emission builds the qualified +/// form via [`stem_props_name`]. +fn stem_props_local(stem: &str) -> String { + let mut out = String::new(); + for seg in stem.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out.push_str("Props"); + out +} + +/// Fully-qualified newtype name for a dict-schema parameter's +/// composed value. `("versal_cips", "PS_PMC_CONFIG")` → +/// `"versal_cips::PsPmcConfig"`. Same compositional-value pattern +/// as the family-based [`stem_props_name`], applied to Xilinx's +/// `structured_tcldict` parameter surface so top-proc kwargs like +/// `-ps_pmc_config` get a typed newtype instead of raw Properties. +/// Fully-qualified newtype name for a dict-schema sub-proc. +/// +/// `("gtwiz_versal", ["intf", "gt_settings"], "LR0_SETTINGS")` → +/// `"gtwiz_versal::intf::gt_settings::Lr0Settings"`. The IP name +/// keeps its snake_case; namespace-prefix segments keep their +/// lowercase form; the leaf param name becomes PascalCase. +fn dict_props_name( + ip_name: &str, + namespace_prefix: &[&str], + param_name: &str, +) -> String { + let scope = proc_scope(ip_name, namespace_prefix); + format!("{scope}::{}", dict_props_local(param_name)) +} + +/// Local (unqualified) form of the dict-schema newtype name — +/// PascalCase of the param name. +fn dict_props_local(param_name: &str) -> String { + let mut out = String::new(); + for seg in param_name.split('_').filter(|s| !s.is_empty()) { + out.push_str(&pascal_case(seg)); + } + out +} + +/// Build the `param_name → qualified_newtype_name` map that +/// [`write_set_property_dict`] consults to inject the `::to` +/// unwrap step. Empty when the IP has no dict schemas. +fn build_dict_schema_newtypes( + ip_name: &str, + dict_schemas: &std::collections::HashMap, +) -> std::collections::HashMap { + dict_schemas + .keys() + // Top-level dict-schema newtype lookup for the top-proc + // arg-decl (`-ps_pmc_config: versal_cips::PsPmcConfig`). + // Nested newtypes referenced by sub-schemas aren't part of + // this map — they're referenced by the sub-procs directly. + .map(|k| (k.clone(), dict_props_name(ip_name, &[], k))) + .collect() +} + +/// Convert an identifier segment to PascalCase — first char upper, +/// rest lower. Non-ASCII passes through unchanged. +fn pascal_case(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + if let Some(c) = chars.next() { + out.push(c.to_ascii_uppercase()); + for c in chars { + out.push(c.to_ascii_lowercase()); + } + } + out +} + +/// Bundle of names needed to emit one family's per-index merge +/// block inside [`write_set_property_dict`]. Built at the top-proc +/// call site from an [`IndexedFamily`] + the IP's namespace. +struct FamilyMerge<'a> { + /// Uppercase stem (`"MAC_PORT"`). + stem: String, + /// Lowercase-ident stem for arg names (`"mac_port"`). + stem_lower: String, + /// Present indices (`[0, 1, 2, 3, 4, 5]`). + indices: Vec, + /// Fully-qualified newtype path for the `::to` unwrap + /// (`"dcmac::MacPortProps"`). + newtype_qualified: String, + #[allow(dead_code)] + marker: std::marker::PhantomData<&'a ()>, +} + +#[allow(clippy::too_many_arguments)] +/// Normalize an IP-XACT `spirit:format="bool"` default value into +/// the canonical HTCL literal `true` / `false`. XML defaults for +/// bool params come in a mix of shapes: quick-xml emits `true` / +/// `false` for boolean-typed values, but plenty of IPs still +/// stamp `1` / `0` even when `spirit:format="bool"` is declared, +/// and empty defaults mean "the IP has an internal default; if +/// the caller doesn't override, the wrapper's guard block skips +/// the assignment entirely." +/// +/// The `false` fallback for empty and unrecognized values is safe +/// because it flows into `@default(false)` — the runtime +/// `__vw_kw__set` guard only merges the value into the +/// property dict when the caller passed the flag explicitly, so +/// the default is inert unless referenced. +fn normalize_bool_default(default: &str) -> &'static str { + match default.trim() { + "1" | "true" | "TRUE" | "True" => "true", + // Empty defaults / unrecognized shapes get the safer + // `false` fallback — the `__vw_kw__set` guard makes + // this inert unless the caller sets the flag explicitly. + _ => "false", + } +} + +/// Which lexical form a bool-typed parameter takes on the wire to +/// Vivado's `set_property`. HTCL surface is the same either way +/// (bare `true` / `false`); the wrapper's marshaling picks the +/// lexical based on how the IP-XACT declares the underlying +/// parameter. +/// +/// See `bool_kind` for the classification rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BoolKind { + /// IP-XACT `spirit:format="bool"`. Vivado's parameter engine + /// normalizes this family and accepts the canonical `true` / + /// `false` string. Emit `[if $arg {list true} {list false}]`. + Pure, + /// Pseudo-boolean: IP-XACT `spirit:format="long"` bound to a + /// two-entry choice pair `(text="false" → 0, text="true" → + /// 1)`. Vivado STORES the numeric `0` / `1` — passing + /// `true` / `false` errors at `set_property` or fails later + /// at `validate_ip`. Emit `[if $arg {list 1} {list 0}]`. + /// + /// Extremely common on CoreGen-lineage IPs — cpm5's + /// component.xml alone has ~3,500 params of this shape. See + /// the audit in the pseudo-bool implementation planning. + Pseudo, +} + +/// Classify a parameter as bool-typed at the HTCL surface, and +/// return the wire form the wrapper should marshal to. `None` +/// when the parameter isn't bool-typed (a scalar string / int / +/// float / dict / enum / newtype-composed slot). +/// +/// The classification is deliberately schema-driven, not +/// heuristic: `Pure` requires the IP-XACT `spirit:format="bool"` +/// tag; `Pseudo` requires `format="long"` PLUS a `choiceRef` +/// resolving to a choice with EXACTLY two enumerations valued +/// `0` and `1` with labels `false` / `true` (case-insensitive). +/// The 8-choice audit of cpm5 confirmed this signature has no +/// false positives — string enums like `{REFCLK0=0, REFCLK1=1}` +/// or size enums like `{16KB=0, 32KB=1}` all miss the label +/// match and correctly stay untyped. +fn bool_kind(component: &Component, p: &Parameter) -> Option { + if p.is_bool() { + return Some(BoolKind::Pure); + } + if param_is_pseudo_bool(component, p) { + return Some(BoolKind::Pseudo); + } + None +} + +/// Detect pseudo-boolean parameters: `spirit:format="long"` with +/// a `choiceRef` pointing at a `{(text="false" → 0), (text="true" +/// → 1)}` choice. See `BoolKind::Pseudo` for the motivation. +fn param_is_pseudo_bool(component: &Component, p: &Parameter) -> bool { + if p.value.format.as_deref() != Some("long") { + return false; + } + let Some(choice_ref) = p.value.choice_ref.as_deref() else { + return false; + }; + let Some(choice) = component.find_choice(choice_ref) else { + return false; + }; + if choice.enumerations.len() != 2 { + return false; + } + let mut saw_true = false; + let mut saw_false = false; + for e in &choice.enumerations { + let label = e.label.as_deref().unwrap_or("").trim(); + match (label.to_ascii_lowercase().as_str(), e.value.trim()) { + ("true", "1") => saw_true = true, + ("false", "0") => saw_false = true, + _ => return false, + } + } + saw_true && saw_false +} + +/// The Tcl `value_expr` fragment for a bool-typed param's +/// marshaling site. Callers write it into their +/// `if {${__vw_kw__set}} { … }` template. +/// +/// `if`-form rather than `expr`-form so `$arg` appears as a +/// whole-word VarRef the analyzer's usage walker can see — +/// otherwise every bool arg produces a spurious "unused proc +/// arg" warning. +fn bool_value_expr(arg: &str, kind: BoolKind) -> String { + let (t, f) = match kind { + BoolKind::Pure => ("true", "false"), + BoolKind::Pseudo => ("1", "0"), + }; + format!("[if ${arg} {{list {t}}} {{list {f}}}]") +} + +// The arg count is intentionally high — this fn threads every +// input the analyzer needs to type-annotate and default a single +// proc arg (doc, component, presets, param, opts, prefix strip, +// ip name, dict schema map, dict namespace). Bundling them into +// a struct would just push the same coupling into the caller. +#[allow(clippy::too_many_arguments)] +fn emit_arg_decl( + doc: &mut Doc, + component: &Component, + presets: &crate::presets::PresetMap, + p: &Parameter, + opts: &GenerateOptions, + prefix_to_strip: &str, + ip_name: &str, + dict_schemas: &std::collections::HashMap, + // Namespace prefix under which any dict-schema newtype for `p` + // has been (or will be) emitted. Empty for top-proc args (the + // classic PS_PMC_CONFIG rail); non-empty for split-node procs + // that emit their dict-schema sub-procs under + // `::::`. Ignored when the param isn't + // dict-schema-backed. + dict_ns: &[&str], +) { + if opts.include_descriptions { + if let Some(desc) = p.description.as_deref().filter(|s| !s.is_empty()) { + for line in desc.lines() { + doc.push(Item::DocComment(line.trim_end().into())); + } + } + } + // Dict-schema-backed params get their composed newtype form + // instead of raw Properties. The IP-XACT default (a paired-list + // string) is unrepresentable as a newtype value, so we omit the + // `@default(...)` — the top-proc body's `__vw_kw__set` + // guard skips unset args, and callers explicitly compose the + // slot via the constructor when they want to override. + // + // Dict-schema map keys: split-node schemas are keyed by the + // STRIPPED slot name (`CHANNEL_MAP`) so the emitted newtype is + // `::intf0::ChannelMap` not `::intf0::Intf0ChannelMap`. + // Top-proc schemas use the raw param name (`PS_PMC_CONFIG`) as + // key with an empty prefix_to_strip → strip is a no-op. + let schema_key = strip_prefix(&p.name, prefix_to_strip); + let dict_schema = dict_schemas.get(schema_key); + if let Some(_schema) = dict_schema { + let ctor_scope = proc_scope(ip_name, dict_ns); + let ctor = format!("{ctor_scope}::{}", schema_key.to_ascii_lowercase()); + doc.push(Item::DocComment(format!("Composed via [{ctor}]."))); + } + let mut words = Vec::new(); + if dict_schema.is_none() { + // Bool-typed params (pure IP-XACT `format="bool"` OR + // pseudo-bool `format="long"` + `{false=0,true=1}` + // choice) get the HTCL `bool` type. Skip `@enum` + // emission — bool accepts exactly `true` / `false` and + // the analyzer's literal check enforces it structurally; + // an `@enum(true, false)` annotation would be redundant. + // Normalize the XML default to the canonical `true` / + // `false` lexical form so `@default(...)` matches the + // HTCL surface (the wire-form marshaling to `1`/`0` for + // pseudo-bool happens later at the set_property emit + // site). + if bool_kind(component, p).is_some() { + let default = normalize_bool_default(p.value.default_value()); + words.push(Word::Raw(format!("@default({default})"))); + } else { + let enum_values = enum_values_for(component, presets, p); + if !enum_values.is_empty() { + let formatted: Vec = enum_values + .iter() + .map(|v| format_attribute_value(v)) + .collect(); + words.push(Word::Raw(format!( + "@enum({})", + formatted.join(", ") + ))); + } + // Always emit `@default(...)` — a missing default would + // make the param required at the analyzer level, but every + // IP-XACT parameter is optional in Vivado's semantics (the + // IP uses its own internal default when the user doesn't + // override). Empty defaults happen when the XML's + // `` is whitespace-only (BOARD_PARAMETER, + // ANLT_PARAMETERS in gtwiz_versal, etc.) — quick-xml's + // `$text` strips leading/trailing whitespace, so we see + // `""`. Emit `@default("")` as the placeholder and let the + // runtime `__vw_kw__set` guard skip the merge loop + // when the caller didn't pass a value — same pattern as + // the family / split-node kwarg placeholders. + let default = p.value.default_value(); + if !default.is_empty() { + words.push(Word::Raw(format!( + "@default({})", + format_attribute_value(default) + ))); + } else { + words.push(Word::Raw("@default(\"\")".into())); + } + } + } else { + // Dict-schema-backed param: empty-string placeholder + // default. The runtime guard (`__vw_kw__set`) + // prevents the placeholder from ever being dereferenced + // through the newtype machinery. + words.push(Word::Raw("@default(\"\")".into())); + } + let lowered = lowercase_ident(strip_prefix(&p.name, prefix_to_strip)); + // Typing rules for the param: + // - Dict-schema-backed: use its typed newtype + // (`versal_cips::PsPmcConfig`) so `dcmac::` / `versal_cips::` + // completion surfaces the type alongside the constructor, + // and misuse (passing e.g. `CpmConfig` to `-ps_pmc_config`) + // is a compile-time error. + // - Otherwise Properties-shaped (paired-dict default without a + // registered schema): keep raw `Properties` — callers still + // compose these by hand, and the top-proc body unwraps via + // `Properties::to_raw` at the `set_property -dict` boundary + // (see [`write_set_property_dict`]). + // - Plain scalar: no type annotation. + let typed_name = if dict_schema.is_some() { + // Dict-schema slot — reference the newtype at whatever + // namespace it was emitted under. Top-proc args pass + // `dict_ns=&[]`; split-node args pass `dict_ns=&[node_local]`. + // Use the stripped key so the PascalCase newtype name isn't + // prefixed with the split-node's label (see schema_key). + format!( + "{lowered}: {}", + dict_props_name(ip_name, dict_ns, schema_key) + ) + } else if is_properties_shaped_param(p) { + format!("{lowered}: Properties") + } else if bool_kind(component, p).is_some() { + // HTCL `bool` primitive — same surface for pure + // (`format="bool"`) and pseudo (`format="long"` + + // false/true choice) bools. The analyzer's bool literal + // check gates values through this slot; wire-form + // marshaling picks true/false vs 1/0 at the + // `lappend _vw_d` site based on the same `bool_kind`. + format!("{lowered}: bool") + } else { + lowered + }; + words.push(Word::Bare(typed_name)); + doc.push(Item::Command(Command { + doc_comments: Vec::new(), + words, + body: None, + })); +} + +/// Union of the parameter's IP-XACT `` values and any +/// presets, in insertion order. Order is *IP-XACT first* (preserving +/// the vendor's intended ordering when both sources agree) followed +/// by preset-only values; duplicates are filtered. +fn enum_values_for( + component: &Component, + presets: &crate::presets::PresetMap, + p: &Parameter, +) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut out: Vec = Vec::new(); + if let Some(choice) = p + .value + .choice_ref + .as_deref() + .and_then(|name| component.find_choice(name)) + { + for e in &choice.enumerations { + if seen.insert(e.value.clone()) { + out.push(e.value.clone()); + } + } + } + if let Some(extra) = presets.get(&p.name) { + for v in extra { + if seen.insert(v.clone()) { + out.push(v.clone()); + } + } + } + out +} + +/// True when the default value parses as a paired-dict Tcl-list +/// shape — i.e. has an even number of whitespace-separated tokens +/// (≥ 2) with identifier-shaped keys at every even index. Used by +/// [`emit_arg_decl`] / [`write_set_property_dict`] to decide which +/// wrapper args get the typed `Properties` annotation + automatic +/// `Properties::to_raw` unwrap at the extern boundary. +/// +/// Vivado IP-XACT defaults for CONFIG.* dict slots typically look +/// like `"KEY1 VAL1 KEY2 VAL2"` (e.g. `CPM_PCIE0_MODES None`, +/// `SMON_ALARMS Set_Alarms_On SMON_ENABLE_TEMP_AVERAGING 0`), +/// while scalar params look like `Custom` or `0` or +/// `versal_cips_v3_4`. Two-pair-shaped strings whose keys happen +/// to be bare-identifier-shaped slip through as Properties even +/// when the IP author meant them as a scalar — unlikely enough +/// to be acceptable noise; the wrapper still works when the +/// caller passes a string-shaped raw value (it round-trips through +/// `Properties::to_raw` returning the same paired list). +/// Parameter-level Properties-shape decision. Prefers Xilinx's +/// authoritative vendor tag (`tcldict` +/// on ``) when present, then falls back to the +/// structural default-value heuristic below. +/// +/// Xilinx writes the default of a tcldict param as the bare +/// sentinel `0` (see `INTF0_GT_SETTINGS`, `INTF0_GT_INTERNAL`, +/// `INTF1_*` in the `gtwiz_versal_v1_0` component.xml), which the +/// structural heuristic reads as scalar and gets wrong. The +/// vendor tag captures the array-indexed compound-property +/// intent — Vivado's Tcl uses `CONFIG.INTF0_GT_SETTINGS(LR0_SETTINGS) +/// {…}` on those params — so params carrying it must be exposed +/// as Properties-typed args regardless of their default shape. +fn is_properties_shaped_param(p: &Parameter) -> bool { + p.has_parameter_type("tcldict") + || is_properties_shaped(p.value.default_value()) +} + +/// Look up the sibling `` whose name matches +/// `param_name` and return its default. Vivado sometimes hides the +/// paired-list schema for a user-facing scalar param in an +/// internally-flagged model parameter with the same name — the +/// `INTF_PARENT_PIN_LIST` case (top-level default `0`, model-param +/// default `QUAD0_RX0 undef QUAD0_RX1 undef …`). `None` when no +/// matching model param exists or when the two names diverge. +fn model_param_anchor_default( + component: &Component, + param_name: &str, +) -> Option { + component + .model_parameters() + .find(|mp| mp.name == param_name) + .map(|mp| mp.value.default_value().to_string()) +} + +/// Post-process an anchor-derived schema: if its keys follow the +/// `QUAD_(RX|TX)` pattern (Vivado's per-quad-per-channel +/// pin-map convention), replicate the slot set across every quad +/// the IP declares. +/// +/// **Key extrapolation.** Model-param anchors typically only +/// enumerate ONE quad's worth of slots as a template. Detect the +/// `QUAD_` prefix and, for each other quad `q` up to the IP's +/// max, clone the `QUAD0_*` slots as `QUAD_*`. The max is +/// derived from the parameter tree — every top-level param whose +/// name starts `QUAD_` counts, so gtwiz-versal's 5-quad layout +/// materializes 40 slots (5 × 8) from the 8-slot QUAD0 template. +/// +/// **No auto-enum on the values.** An earlier version of this +/// function attached `@enum(undef, /INTF__GT_IP_Interface_0, …)` +/// to each RX/TX slot. That was wrong: the `_GT_IP_Interface_0` +/// suffix is a specific BD pin name from one reference topology, +/// not a fixed Vivado convention — the actual pin path is +/// whatever the user's block-design names it, which is +/// arbitrary. Constraining the slot to a hard-coded name set +/// rejects legitimate values. We leave `enum_values` empty here +/// (any string accepted); when a workspace really wants a +/// vocabulary restriction it can add one via +/// `overrides.toml` at the shape's `intf::…::lane_map` (or +/// wherever) path, which layers on top a few lines below. +fn extrapolate_quad_schema( + schema: &mut crate::DictSchema, + component: &Component, + shape_path: &str, + opts: &GenerateOptions, +) { + // Detect the quad-prefix pattern on any existing field. + let has_quad_pattern = schema + .fields + .iter() + .any(|f| parse_quad_slot(&f.name).is_some()); + if !has_quad_pattern { + return; + } + let max_quads = count_indexed_prefix(component, "QUAD"); + + // Build a fresh field list: one per (quad, orig_local_slot) + // pair, ordered quad-major then original-order. + let template: Vec = std::mem::take(&mut schema.fields); + let mut fields = Vec::with_capacity(template.len() * max_quads.max(1)); + for q in 0..max_quads.max(1) { + for f in &template { + let Some((_orig_q, local)) = parse_quad_slot(&f.name) else { + // Non-QUAD-shaped keys stay as-is on the first quad + // iteration only, so we don't duplicate them. + if q == 0 { + fields.push(f.clone()); + } + continue; + }; + let name = format!("QUAD{q}_{local}"); + let field_lookup = lowercase_ident(&name); + let field_override = + opts.overrides.field(shape_path, &field_lookup); + let mut enum_values = f.enum_values.clone(); + let mut default = f.default.clone(); + let mut baseline = + f.baseline || opts.overrides.shape_baseline(shape_path); + if let Some(fo) = field_override { + if let Some(d) = &fo.default { + default = d.clone(); + } + if let Some(ev) = &fo.enum_values { + enum_values = ev.iter().cloned().collect(); + } + if fo.baseline { + baseline = true; + } + } + fields.push(crate::DictField { + name, + default, + description: f.description.clone(), + enum_values, + baseline, + }); + } + } + schema.fields = fields; +} + +/// Split a `QUAD_` slot name into `(n, REST)`. Returns +/// `None` when the prefix doesn't match. +fn parse_quad_slot(name: &str) -> Option<(usize, String)> { + let rest = name.strip_prefix("QUAD")?; + let digit_end = rest.find(|c: char| !c.is_ascii_digit())?; + let n: usize = rest[..digit_end].parse().ok()?; + let after = rest[digit_end..].strip_prefix('_')?; + Some((n, after.to_string())) +} + +/// Split an `RX` / `TX` slot local name into +/// `(direction, m)`. Direction is `"RX"` or `"TX"`. +/// Count how many top-level params share the `_` shape, +/// which gives us the max index for that family. Used to figure +/// out the IP's max quad count without hard-coding. +fn count_indexed_prefix(component: &Component, prefix: &str) -> usize { + let mut indices = std::collections::BTreeSet::new(); + for p in component.component_parameters() { + let Some(rest) = p.name.strip_prefix(prefix) else { + continue; + }; + let Some(digit_end) = rest.find(|c: char| !c.is_ascii_digit()) else { + continue; + }; + if rest[digit_end..].starts_with('_') { + if let Ok(n) = rest[..digit_end].parse::() { + indices.insert(n); + } + } + } + indices.iter().max().map(|n| n + 1).unwrap_or(0) +} + +fn is_properties_shaped(default: &str) -> bool { + // Fast path via the Tcl-aware paired-list tokenizer — catches + // defaults with `{…}`-grouped values (INTF*_TXRX_OPTIONAL_PORTS + // and other tcldict params carry braces in their inner + // `INTF_LR_SETTINGS {LR0_SETTINGS {…}}` payload). The naive + // `split_whitespace` heuristic below tokenizes those braces as + // separate items and misclassifies the default as non-paired. + if !crate::paired_list::parse_paired_list(default).is_empty() { + return true; + } + let tokens: Vec<&str> = default.split_whitespace().collect(); + if tokens.len() < 2 || !tokens.len().is_multiple_of(2) { + return false; + } + for (i, t) in tokens.iter().enumerate() { + if i % 2 != 0 { + continue; + } + let mut chars = t.chars(); + let first = match chars.next() { + Some(c) => c, + None => return false, + }; + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + for c in chars { + if !(c.is_ascii_alphanumeric() || c == '_' || c == '.') { + return false; + } + } + } + true +} + +/// Lowercase an IP-XACT parameter name into a valid htcl argument +/// name. The htcl proc-arg grammar is `/[a-zA-Z_][a-zA-Z0-9_]*/`, so +/// an empty result or a digit-leading result (which prefix-stripping +/// can produce — e.g. `64BIT` after stripping `CPM_PCIE1_PF0_BAR0_`) +/// gets a leading underscore to land back in the grammar. +fn lowercase_ident(name: &str) -> String { + let mut out = String::with_capacity(name.len() + 1); + for c in name.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c.to_ascii_lowercase()); + } else { + out.push('_'); + } + } + let needs_leading_underscore = out + .as_bytes() + .first() + .map(|b| b.is_ascii_digit()) + .unwrap_or(true); + if needs_leading_underscore { + out.insert(0, '_'); + } + out +} + +/// Sanitize an arbitrary string for use as an htcl identifier. +fn sanitize_ident(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if c.is_ascii_alphanumeric() || c == '_' { + out.push(c); + } else { + out.push('_'); + } + } + out +} + +/// Render an IP-XACT default value as it should appear inside an +/// `@default(...)` attribute. The htcl proc-args grammar accepts only +/// three attribute-value forms — `integer_literal`, `attribute_value_ident` +/// (`[a-zA-Z_][a-zA-Z0-9_]*`), and double-quoted strings. Anything that +/// isn't a clean ident or integer is double-quoted (with `"` escaped). +fn format_attribute_value(s: &str) -> String { + if is_integer_literal(s) || is_attribute_ident(s) { + s.to_string() + } else { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } +} + +fn is_integer_literal(s: &str) -> bool { + let body = s.strip_prefix('-').unwrap_or(s); + !body.is_empty() && body.bytes().all(|b| b.is_ascii_digit()) +} + +fn is_attribute_ident(s: &str) -> bool { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !(first.is_ascii_alphabetic() || first == '_') { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +#[cfg(test)] +mod tests { + use super::*; + use ipxact::{ + Choice, Choices, Component, Enumeration, ParamValue, Parameter, + Parameters, + }; + + fn mk_component() -> Component { + Component { + vendor: "acme".into(), + library: "ip".into(), + name: "demo".into(), + version: "1.0".into(), + description: Some("A demo IP.".into()), + parameters: Some(Parameters { + entries: vec![ + Parameter { + name: "BUS_WIDTH".into(), + description: Some("Bus width in bits.".into()), + value: ParamValue { + text: "32".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + Parameter { + name: "MODE".into(), + value: ParamValue { + text: "FAST".into(), + resolve: Some("user".into()), + choice_ref: Some("mode_choices".into()), + ..Default::default() + }, + ..Default::default() + }, + ], + }), + choices: Some(Choices { + entries: vec![Choice { + name: "mode_choices".into(), + enumerations: vec![ + Enumeration { + value: "FAST".into(), + ..Default::default() + }, + Enumeration { + value: "SLOW".into(), + ..Default::default() + }, + ], + }], + }), + ..Default::default() + } + } + + fn mk_split_component(n_per_group: usize) -> Component { + // Build a component with two big groups and a smattering of + // small ones, all above the split threshold. + let mut entries = Vec::new(); + for i in 0..n_per_group { + entries.push(Parameter { + name: format!("BIG_ONE_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + entries.push(Parameter { + name: format!("BIG_TWO_FIELD{i}"), + value: ParamValue { + text: "1".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + } + // A pair of tiny groups that should be collapsed into _misc. + for name in ["TINY_A_ONE", "TINY_B_ONE", "TINY_C_ONE", "STRAY_THING"] { + entries.push(Parameter { + name: name.into(), + value: ParamValue { + text: "x".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }); + } + Component { + vendor: "acme".into(), + library: "ip".into(), + name: "wide".into(), + version: "1.0".into(), + parameters: Some(Parameters { entries }), + ..Default::default() + } + } + + #[test] + fn single_mode_below_threshold() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Procs live inside `namespace eval { … }` and get + // the 2-space indent from `write_namespace_block`. Two + // procs: `configure` (typed value constructor) and + // `create` (cell instantiator + config applier). + let n_procs = out.matches("\n proc ").count(); + assert_eq!(n_procs, 2, "{out}"); + assert!(out.contains("proc configure")); + assert!(out.contains("proc create")); + assert!(out.contains("namespace eval demo {")); + // Top-level Config newtype ships outside the namespace + // block (see emit_config_prelude for why). + assert!(out.contains("type demo::Config = Properties")); + } + + #[test] + fn split_mode_emits_top_and_sub_procs() { + let component = mk_split_component(60); // 60 * 2 + 4 = 124 params > 100 + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + eprintln!("--- generated ---\n{out}\n--- end ---"); + assert!(out.contains("proc create")); + // Split-node procs use fully-qualified names now — they live + // outside the namespace-block so they can be extracted into + // sibling `.htcl` files by `split_into_files`. + assert!(out.contains("proc wide::big_one")); + assert!(out.contains("proc wide::big_two")); + let parsed = vw_htcl::parse(&out); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd + // (`ip::check`, `create_bd_cell`, `set_property`); + // those resolve when the wrapper is sourced through + // the loader, but this unit test runs the validator + // on the bare generated text. The unknown-call + // diagnostic is *expected* in that mode; we filter it + // out so the test catches real structural breakage. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "{errors:#?}"); + } + + #[test] + fn split_sub_procs_take_cell_as_first_arg() { + let component = mk_split_component(60); + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Sub-procs are pure value-constructors — no `cell:` + // first arg, no bd switch — and return the node's typed + // newtype instead of `bd_cell`. Assert the shape. + // Fully-qualified name shape after the split-file refactor. + assert!( + out.contains("proc wide::big_one {\n ## Configuration value"), + "{out}" + ); + assert!(out.contains("} wide::BigOneProps {"), "{out}"); + // The old `cell: bd_cell` mutator arg must NOT appear. + assert!(!out.contains("cell: bd_cell"), "{out}"); + } + + #[test] + fn tiny_groups_land_on_the_parent_proc() { + let component = mk_split_component(60); + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // None of the tiny prefix groups becomes its own proc... + for name in ["proc tiny_a ", "proc tiny_b ", "proc stray "] { + assert!(!out.contains(name), "unexpected {name} in:\n{out}"); + } + // ...and the params instead appear as args on the top + // proc's configure counterpart. Post-refactor `create` has + // just `-name`/`-bd`/`-config`; the full documented kwarg + // pile lives on `configure`. Slice the configure range so + // we can search its args without accidentally matching arg + // names embedded in one of the value-constructor sub-procs' + // bodies. + let cfg_start = + out.find("proc configure {").expect("no proc configure"); + let cfg_body = &out[cfg_start..]; + let cfg_end = cfg_body + .find("\n }\n") + .map(|e| e + 5) + .unwrap_or(cfg_body.len()); + let cfg_range = &cfg_body[..cfg_end]; + for arg in ["tiny_a_one", "tiny_b_one", "tiny_c_one", "stray_thing"] { + assert!( + cfg_range.contains(arg), + "{arg} missing from configure proc: {cfg_range}" + ); + } + } + + #[test] + fn arg_name_strips_node_prefix() { + // Two big groups whose internal arg names should be the + // segments *after* the group prefix, not the full name. + let entries = (0..10) + .flat_map(|i| { + [ + Parameter { + name: format!("GROUP_A_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + Parameter { + name: format!("GROUP_B_FIELD{i}"), + value: ParamValue { + text: "0".into(), + resolve: Some("user".into()), + ..Default::default() + }, + ..Default::default() + }, + ] + }) + .collect(); + let component = Component { + vendor: "acme".into(), + library: "ip".into(), + name: "demo".into(), + version: "1.0".into(), + parameters: Some(Parameters { entries }), + ..Default::default() + }; + let opts = GenerateOptions { + split_threshold: 5, + ..GenerateOptions::default() + }; + let out = generate( + &component, + &Default::default(), + &::std::collections::HashMap::new(), + &opts, + ) + .into_single(); + // Inside the GROUP_A proc, the arg names should be `field0`, + // not `group_a_field0`. + assert!(out.contains("@default(0) field0\n"), "{out}"); + assert!(!out.contains("group_a_field0"), "{out}"); + // The constructor stores the index-stripped key in its + // Properties dict… + assert!(out.contains("dict set _vw_d FIELD0 $field0"), "{out}"); + // …and the top proc's merge loop prefixes with + // `CONFIG.GROUP_A_` when composing atomically. The literal + // format uses `$_vw_f` at runtime, so we assert on the + // prefix pattern. + assert!(out.contains("CONFIG.GROUP_A_$_vw_f"), "{out}"); + } + + #[test] + fn generated_output_parses_back() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + } + + #[test] + fn bd_switch_arg_toggles_construction() { + // Every generated wrapper carries the typed `-bd` arg + // (real `bool`, defaulting to `false` = project-IP mode), + // and the body branches on it: `-bd true` calls + // `create_bd_cell`, `-bd false` calls `create_ip`. Both + // paths produce a plain-string handle so the wrapper's + // declared return type (`string`) covers either branch. + for out in [ + generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(), + generate( + &mk_split_component(6), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions { + split_threshold: 5, + ..GenerateOptions::default() + }, + ) + .into_single(), + ] { + assert!(out.contains("@default(false) bd: bool"), "{out}"); + assert!(out.contains("if {$bd} {"), "{out}"); + assert!(out.contains("create_bd_cell"), "{out}"); + assert!(out.contains("create_ip -vlnv"), "{out}"); + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "wrapped output should parse cleanly: {:?}", + parsed.errors + ); + } + } + + #[test] + fn includes_description_as_doc_comment() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + assert!(out.contains("## A demo IP."), "{out}"); + assert!(out.contains("## Bus width in bits."), "{out}"); + } + + #[test] + fn emits_default_and_enum_attributes() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + assert!(out.contains("@default(32) bus_width"), "{out}"); + assert!(out.contains("@enum(FAST, SLOW)"), "{out}"); + assert!(out.contains("@default(FAST) mode"), "{out}"); + } + + #[test] + fn emits_set_property_for_each_param() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + // Post-refactor these lappend lines live inside `configure`, + // not `create`. `create`'s body just unwraps `-config` and + // splats the resulting dict via `set_property -dict`. + assert!(out.contains("CONFIG.BUS_WIDTH $bus_width"), "{out}"); + assert!(out.contains("CONFIG.MODE $mode"), "{out}"); + } + + /// `configure`'s body is pure dict assembly + a Config wrap. + /// Any of the side-effecting Vivado calls appearing inside its + /// body would mean the seam wasn't cleanly cut. + #[test] + fn configure_returns_typed_config_no_side_effects() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let cfg_start = out.find("proc configure {").expect("no configure"); + let cfg_body = &out[cfg_start..]; + let cfg_end = cfg_body + .find("\n }\n") + .map(|e| e + 5) + .unwrap_or(cfg_body.len()); + let cfg_range = &cfg_body[..cfg_end]; + for forbidden in ["create_bd_cell", "create_ip", "set_property"] { + assert!( + !cfg_range.contains(forbidden), + "configure body should not contain `{forbidden}`:\n{cfg_range}" + ); + } + // But it SHOULD wrap the assembled dict as a Config value. + assert!(cfg_range.contains("demo::Config::from"), "{cfg_range}"); + } + + /// `create`'s arg surface is exactly `-name`, `-bd`, `-config` + /// under the post-refactor design. No documented-kwarg pile. + #[test] + fn create_takes_only_name_bd_config() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let create_start = out.find("proc create {").expect("no create"); + // Argspec ends at `} string {`. + let create_body = &out[create_start..]; + let argspec_end = create_body + .find("} string {") + .expect("create should return string"); + let argspec = &create_body[..argspec_end]; + for expected in ["name", "bd: bool", "config: demo::Config"] { + assert!( + argspec.contains(expected), + "expected `{expected}` in create argspec:\n{argspec}" + ); + } + // No leftover typed IP-param kwargs on create. + for forbidden in ["bus_width: int", "@enum(FAST, SLOW)"] { + assert!( + !argspec.contains(forbidden), + "IP-param kwarg `{forbidden}` leaked onto create:\n{argspec}" + ); + } + } + + /// `create`'s body unwraps `$config` through the Config newtype + /// and applies via `set_property -dict` in both `-bd 1` and + /// `-bd 0` branches. + #[test] + fn create_body_unwraps_config_and_applies() { + let out = generate( + &mk_component(), + &Default::default(), + &::std::collections::HashMap::new(), + &GenerateOptions::default(), + ) + .into_single(); + let create_start = out.find("proc create {").expect("no create"); + let create_range = &out[create_start..]; + // Unwrap step. + assert!( + create_range.contains("demo::Config::to -v $config"), + "create should unwrap $config via Config::to:\n{create_range}" + ); + // Guard for the empty-Config default (bracket-expr @default + // fallback pattern documented in build_single_create_body). + assert!( + create_range.contains("demo::Config::empty"), + "create should coerce empty-string default to Config::empty:\n{create_range}" + ); + // Both bd branches: `-bd true` targets `$handle` (the + // bd_cell path directly); `-bd false` fetches the IP + // object via `[get_ips $handle]`. + assert!( + create_range.contains("set_property -dict $_dict -objects $handle") + ); + assert!(create_range + .contains("set_property -dict $_dict -objects [get_ips $handle]")); + } + + // ------------------------------------------------------------------ + // is_properties_shaped_param — Xilinx vendor-tag routing. + // ------------------------------------------------------------------ + + use ipxact::{ParameterInfo, VendorExtensions}; + + fn mk_param(default: &str, tcldict: bool) -> Parameter { + Parameter { + name: "X".into(), + value: ParamValue { + text: default.into(), + ..Default::default() + }, + vendor_extensions: if tcldict { + Some(VendorExtensions { + xilinx_parameter_info: Some(ParameterInfo { + parameter_type: vec!["tcldict".into()], + }), + }) + } else { + None + }, + ..Default::default() + } + } + + #[test] + fn tcldict_tag_overrides_scalar_default() { + // The `INTF0_GT_SETTINGS` shape — vendor tag says + // `tcldict`, default is Xilinx's `0` sentinel that reads + // as scalar. The vendor tag must win. + let p = mk_param("0", true); + assert!(is_properties_shaped_param(&p)); + } + + #[test] + fn structural_shape_still_wins_without_tag() { + // The `INTF0_LR0_SETTINGS` shape — no vendor tag but the + // default's paired-list shape gives it away. + let p = mk_param("NA NA", false); + assert!(is_properties_shaped_param(&p)); + } + + #[test] + fn neither_tag_nor_shape_stays_scalar() { + // Plain scalar param. Was scalar before this change, + // stays scalar after. + let p = mk_param("0", false); + assert!(!is_properties_shaped_param(&p)); + } + + #[test] + fn dict_field_with_baseline_flag_emits_baseline_attr() { + // A `DictField` with `baseline: true` renders as + // `@baseline("")` in the emitted arg list, not + // `@default("...")`. Non-baseline fields still emit + // `@default(...)` as before. + let baseline = crate::DictField { + name: "RX_REFCLK_FREQUENCY".into(), + default: "156.25".into(), + description: None, + enum_values: Default::default(), + baseline: true, + }; + let normal = crate::DictField { + name: "RX_HD_EN".into(), + default: "0".into(), + description: None, + enum_values: Default::default(), + baseline: false, + }; + let opts = GenerateOptions::default(); + let mut doc = Doc::new(); + emit_dict_field_arg(&mut doc, &baseline, &opts); + emit_dict_field_arg(&mut doc, &normal, &opts); + let out = doc.to_string(); + assert!( + out.contains("@baseline(\"156.25\") rx_refclk_frequency"), + "expected baseline emission, got:\n{out}", + ); + assert!( + out.contains("@default(0) rx_hd_en"), + "expected default emission, got:\n{out}", + ); + // Sanity: the resulting attributes parse back through the + // HTCL grammar without errors — @baseline is treated as a + // first-class attribute the same as any @-prefixed name. + let wrapped = format!("proc t {{\n{out}}} unit {{ }}\n"); + let parsed = vw_htcl::parse(&wrapped); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + } + + #[test] + fn extrapolate_quad_schema_does_not_synthesize_pin_path_enum() { + // Regression: an earlier version hard-coded + // `@enum(undef, /INTF__GT_IP_Interface_0, …)` + // on every QUAD*_RX*/TX* slot. The `_GT_IP_Interface_0` + // suffix is a specific BD pin name from one reference + // topology, not a fixed Vivado convention — a real BD + // can name the pin anything, so the derived enum + // wrongly rejected legitimate values. The fix leaves + // `enum_values` untouched by the extrapolator; any + // per-workspace vocabulary lives in overrides.toml. + let component = mk_component(); + let mut schema = crate::DictSchema { + fields: vec![ + crate::DictField { + name: "QUAD0_RX0".into(), + default: String::new(), + description: None, + enum_values: Default::default(), + baseline: false, + }, + crate::DictField { + name: "QUAD0_TX0".into(), + default: String::new(), + description: None, + enum_values: Default::default(), + baseline: false, + }, + ], + sub_schemas: Default::default(), + }; + let opts = GenerateOptions::default(); + extrapolate_quad_schema(&mut schema, &component, "", &opts); + for f in &schema.fields { + assert!( + f.enum_values.is_empty(), + "field {} unexpectedly got auto-enum {:?} — pin paths \ + must be user-supplied, not baked into the wrapper", + f.name, + f.enum_values, + ); + } + } +} diff --git a/vw-ip/src/group.rs b/vw-ip/src/group.rs new file mode 100644 index 0000000..eaf3952 --- /dev/null +++ b/vw-ip/src/group.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Derive parameter groups from naming conventions. +//! +//! IP-XACT components published by Xilinx carry no machine-readable +//! grouping for their configuration parameters. The xgui Tcl scripts +//! that drive the GUI grouping are encrypted, so they're not a source +//! we can use. What we *can* use is the strong prefix structure of the +//! parameter names themselves: in CPM5, 4200 parameters start with +//! `CPM_PCIE0_`, another 4200 with `CPM_PCIE1_`, 136 with `CPM_CCIX_`, +//! and so on. That structure is the right grain for a sub-proc. +//! +//! The grouping strategy: +//! +//! 1. Split each parameter name on `_`. +//! 2. Take the first N segments as the group key. We pick N to balance +//! group cardinality vs. group size — small enough that there are +//! few groups (so each becomes a manageable proc), big enough that +//! no single group is so huge it's just a flat dump. +//! 3. Parameters with no underscore, or whose only group would be a +//! singleton, fall into a catch-all `_misc` group. + +use std::collections::BTreeMap; + +use ipxact::Parameter; + +#[derive(Clone, Debug)] +pub struct ParameterGroup<'a> { + /// Key used as the group name (e.g. `CPM_PCIE0`). + pub key: String, + /// Parameters in this group, in input order. + pub parameters: Vec<&'a Parameter>, +} + +/// Group parameters by their leading underscore-separated segments. +/// `prefix_segments` controls how many leading segments form the key: +/// 1 = `CPM`, 2 = `CPM_PCIE0`, etc. 2 is the right default for Xilinx's +/// big IPs; their first segment is a coarse domain (`CPM`, `PS`, `PMC`) +/// and the second names the controller / subsystem. +pub fn group_parameters<'a, I>( + parameters: I, + prefix_segments: usize, +) -> Vec> +where + I: IntoIterator, +{ + // BTreeMap keeps groups in a stable, readable order. + let mut groups: BTreeMap> = BTreeMap::new(); + for p in parameters { + let key = prefix_key(&p.name, prefix_segments); + groups.entry(key).or_default().push(p); + } + groups + .into_iter() + .map(|(key, parameters)| ParameterGroup { key, parameters }) + .collect() +} + +/// First `n` underscore-separated segments of `name`. If `name` has +/// fewer than `n` segments (or no underscores), returns the whole name. +/// Empty names map to the literal `_misc`. +fn prefix_key(name: &str, n: usize) -> String { + if name.is_empty() { + return "_misc".into(); + } + let mut out = String::new(); + for (i, seg) in name.split('_').enumerate().take(n) { + if i > 0 { + out.push('_'); + } + out.push_str(seg); + } + // If the name has fewer than `n` segments, we end up with the full + // name as the key — that's fine; it just means the group is named + // after the parameter itself. Singletons coalesce later if we want. + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(name: &str) -> Parameter { + Parameter { + name: name.into(), + ..Default::default() + } + } + + #[test] + fn groups_by_two_prefix_segments() { + let params = [ + p("CPM_PCIE0_FOO"), + p("CPM_PCIE0_BAR"), + p("CPM_PCIE1_BAZ"), + p("CPM_CCIX_QUX"), + ]; + let groups = group_parameters(¶ms, 2); + let by_key: Vec<_> = groups + .iter() + .map(|g| (g.key.clone(), g.parameters.len())) + .collect(); + assert_eq!( + by_key, + vec![ + ("CPM_CCIX".to_string(), 1), + ("CPM_PCIE0".to_string(), 2), + ("CPM_PCIE1".to_string(), 1), + ] + ); + } + + #[test] + fn names_with_fewer_segments_become_their_own_key() { + let params = [p("FOO"), p("FOO_BAR_BAZ")]; + let groups = group_parameters(¶ms, 2); + let keys: Vec<_> = groups.iter().map(|g| g.key.as_str()).collect(); + // "FOO" stays "FOO" (only one segment), "FOO_BAR_BAZ" becomes "FOO_BAR". + assert!(keys.contains(&"FOO")); + assert!(keys.contains(&"FOO_BAR")); + } +} diff --git a/vw-ip/src/lib.rs b/vw-ip/src/lib.rs new file mode 100644 index 0000000..1438502 --- /dev/null +++ b/vw-ip/src/lib.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! IP-XACT → htcl wrapper generation. +//! +//! Reads an IP-XACT component description (via the `ipxact` crate) and +//! emits an htcl instantiation proc for it — the "configuration +//! interface" layer described in the project plan: one top-level proc +//! per IP, with sub-procs for parameter groups when an IP's surface is +//! too large for a single proc to be tractable (CPM5 has ~8700 +//! parameters). +//! +//! Group recovery: IP-XACT itself carries no grouping metadata for +//! large Xilinx IPs (no `` etc., and the `xgui/*.tcl` +//! files that *do* carry the UI grouping are encrypted). Instead, we +//! derive groups from the convention Xilinx uses in parameter naming — +//! `CPM_PCIE0_*`, `CPM_PCIE1_*`, `PS_PMC_*` and so on are clear +//! prefix clusters. See [`group_parameters`]. + +pub mod cips_dict; +pub mod family; +pub mod generate; +pub mod group; +pub mod overrides; +pub mod paired_list; +pub mod presets; +pub mod summary; +pub mod targets; +pub mod tree; + +pub use cips_dict::{ + load_schemas as load_cips_dict_schemas, DictField, DictSchema, +}; +pub use family::{detect_families, DetectOptions, IndexedFamily}; +pub use generate::{generate, GenerateOptions}; +pub use group::{group_parameters, ParameterGroup}; +pub use presets::{ + discover_for as discover_presets, load_files as load_presets, PresetMap, +}; +pub use summary::Summary; +pub use tree::{build_tree, Node, TreeOptions}; + +use std::path::Path; + +use ipxact::Component; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("loading IP-XACT component: {0}")] + Ipxact(#[from] ipxact::Error), +} + +pub type Result = std::result::Result; + +/// Load an IP-XACT component from disk. +pub fn load(path: impl AsRef) -> Result { + Ok(Component::from_file(path)?) +} diff --git a/vw-ip/src/overrides.rs b/vw-ip/src/overrides.rs new file mode 100644 index 0000000..68fa4b3 --- /dev/null +++ b/vw-ip/src/overrides.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Per-IP TOML overrides for typed-constructor field emission. +//! +//! The generator's default source of field metadata is the IP-XACT +//! `` paired-list default — good for populating +//! `@default(...)` but silent on bounded vocabularies (Vivado wants +//! `RX_PAM_SEL` to be one of `NRZ` / `PAM4`, but the XML default is +//! just a bare string). An `overrides.toml` file colocated with each +//! IP's `regenerate.sh` refines the emitted surface where the XML +//! is silent: attach `@enum(…)` restrictions to specific fields, +//! override the XML default, etc. +//! +//! Discovery: the CLI accepts `--overrides ` and threads the +//! parsed [`OverridesFile`] through `GenerateOptions`. When no +//! override file exists, the generator falls back to XML-only +//! defaults (schema shape derived entirely from ``). +//! +//! File shape: +//! +//! ```toml +//! [shapes."intf::gt_settings::lr0_settings"] +//! fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +//! fields.rx_refclk_source = { enum = ["R0", "R1", "R2", "R3", "R4", "R5", "ERR"] } +//! fields.rx_line_rate = { default = "10.3125" } +//! ``` +//! +//! `shape_path` uses `::`-separated segments matching the emitted +//! proc's namespace path *below* the IP name — i.e. how a caller +//! writes `gtwiz_versal::intf::gt_settings::lr0_settings` refers to +//! shape `"intf::gt_settings::lr0_settings"`. Per-N sub-newtypes +//! (`intf0` vs `intf1`) share their shape's overrides — the emitter +//! matches on the stem (`intf`), not the indexed instance. + +use std::collections::HashMap; +use std::path::Path; + +use serde::Deserialize; + +/// Errors produced by [`OverridesFile::load_from`]. +#[derive(Debug, thiserror::Error)] +pub enum OverridesError { + /// The path exists but couldn't be read (permissions, IO error). + #[error("reading overrides file {path}: {source}")] + Read { + path: String, + #[source] + source: std::io::Error, + }, + /// The file exists but doesn't parse as valid TOML matching the + /// override schema. + #[error("parsing overrides file {path}: {source}")] + Parse { + path: String, + #[source] + source: toml::de::Error, + }, +} + +/// The parsed overrides file. `shapes["intf::gt_settings::lr0_settings"]` +/// looks up refinements for one emitted proc's field list. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct OverridesFile { + #[serde(default)] + pub shapes: HashMap, +} + +impl OverridesFile { + /// Load an overrides file from `path`. Missing file is not an + /// error — returns an empty overrides object so callers can pass + /// the result through unconditionally. + pub fn load_from(path: impl AsRef) -> Result { + let path = path.as_ref(); + if !path.exists() { + return Ok(Self::default()); + } + let text = std::fs::read_to_string(path).map_err(|source| { + OverridesError::Read { + path: path.display().to_string(), + source, + } + })?; + toml::from_str(&text).map_err(|source| OverridesError::Parse { + path: path.display().to_string(), + source, + }) + } + + /// Look up a field's override in `shape_path`. `None` when the + /// shape isn't listed, or the shape is listed but the field + /// isn't. Callers merge with the XML-derived default when present. + /// + /// Field names are matched exactly as the emitter writes them + /// (lowercase form of the XML key — e.g. XML `RX_PAM_SEL` + /// becomes field key `rx_pam_sel`). + pub fn field( + &self, + shape_path: &str, + field_name: &str, + ) -> Option<&FieldOverride> { + self.shapes.get(shape_path)?.fields.get(field_name) + } + + /// True when the shape at `shape_path` sets shape-wide + /// `baseline = true`. Missing shape → false. Callers use this + /// to force `@baseline(...)` emission on every field in the + /// shape, regardless of per-field configuration. + pub fn shape_baseline(&self, shape_path: &str) -> bool { + self.shapes.get(shape_path).is_some_and(|s| s.baseline) + } + + /// True when this override set is empty — no shapes registered. + /// Callers use this to skip the whole override-application pass + /// when there's nothing to do (fast path for IPs without an + /// override file). + pub fn is_empty(&self) -> bool { + self.shapes.is_empty() + } +} + +/// Overrides scoped to one emitted proc / shape. +/// +/// `baseline` at this level is the shape-wide switch: when true, +/// every field in the shape gets `@baseline(...)` instead of +/// `@default(...)`. Use it when the containing constructor takes a +/// `-preset` (or other mutation source) that can shift ANY of the +/// shape's fields — enumerating each field with per-field +/// `baseline = true` would be tedious and easy to leave stale as +/// new fields are generated. Per-field `baseline = false` cannot +/// override the shape-level `true` (there's no "opt-out" — if the +/// shape is baseline-wide, individual fields are too). +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ShapeOverrides { + #[serde(default)] + pub fields: HashMap, + /// Shape-wide baseline switch — see the type-level docs. + #[serde(default)] + pub baseline: bool, +} + +/// Per-field refinement applied on top of the XML-derived schema. +/// +/// - `enum_values`: attach `@enum(v1, v2, …)` to the emitted arg, +/// restricting valid callsite values. Overrides absent → no +/// enum annotation. +/// - `default`: replace the XML default. Rare — the XML is usually +/// authoritative, but Xilinx sometimes ships defaults that are +/// sentinels rather than useful values (e.g. `NA NA`). Overrides +/// absent → use XML value. +/// - `baseline`: emit `@baseline()` instead of `@default()`. +/// The field's runtime shape is unchanged (the wrapper uses the same +/// `if kw_set { dict set }` gate either way), but the annotation +/// documents that the value is the ParamInfo baseline — the value a +/// FRESH IP would report, which a preset / board_interface / prior +/// configuration can shift out from under omission. Callers that +/// explicitly pass the baseline value are NOT flagged by the +/// redundant-default lint under this variant. Use for scalar knobs +/// downstream of a `-preset`-carrying constructor where omitting the +/// flag doesn't pin the value. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FieldOverride { + #[serde(default, rename = "enum")] + pub enum_values: Option>, + #[serde(default)] + pub default: Option, + #[serde(default)] + pub baseline: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + use std::io::Write; + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(contents.as_bytes()).unwrap(); + f + } + + #[test] + fn missing_file_yields_empty() { + let path = + std::path::Path::new("/nonexistent-vw-ip-test-overrides.toml"); + assert!(!path.exists()); + let ov = OverridesFile::load_from(path).unwrap(); + assert!(ov.is_empty()); + } + + #[test] + fn empty_file_yields_empty_shapes_map() { + let f = write_tmp(""); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.is_empty()); + } + + #[test] + fn single_shape_single_field_enum() { + let f = write_tmp( + r#" +[shapes."intf::gt_settings::lr0_settings"] +fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let field = ov + .field("intf::gt_settings::lr0_settings", "rx_pam_sel") + .expect("field present"); + assert_eq!( + field.enum_values.as_deref(), + Some(&["NRZ".to_string(), "PAM4".to_string()][..]) + ); + assert!(field.default.is_none()); + } + + #[test] + fn field_default_override() { + let f = write_tmp( + r#" +[shapes."intf::lr0_settings"] +fields.rx_line_rate = { default = "10.3125" } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let field = ov + .field("intf::lr0_settings", "rx_line_rate") + .expect("field present"); + assert_eq!(field.default.as_deref(), Some("10.3125")); + assert!(field.enum_values.is_none()); + } + + #[test] + fn field_baseline_flag_parses() { + let f = write_tmp( + r#" +[shapes."intf::gt_settings::lr0_settings"] +fields.rx_refclk_frequency = { baseline = true } +fields.tx_refclk_frequency = { baseline = true, enum = ["156.25", "161.13"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let rx = ov + .field("intf::gt_settings::lr0_settings", "rx_refclk_frequency") + .expect("rx field present"); + assert!(rx.baseline); + assert!(rx.default.is_none()); + assert!(rx.enum_values.is_none()); + let tx = ov + .field("intf::gt_settings::lr0_settings", "tx_refclk_frequency") + .expect("tx field present"); + assert!(tx.baseline); + assert_eq!( + tx.enum_values.as_deref(), + Some(&["156.25".to_string(), "161.13".to_string()][..]) + ); + } + + #[test] + fn shape_level_baseline_flag_parses() { + // The shape-wide baseline switch flips every field in the + // shape without needing per-field entries. Useful when the + // containing constructor takes a `-preset` that can shift + // any of the shape's fields. + let f = write_tmp( + r#" +[shapes."intf::lr0_settings"] +baseline = true +fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.shape_baseline("intf::lr0_settings")); + assert!(!ov.shape_baseline("intf::other_shape")); + // Per-field override still parses alongside the shape flag. + let field = ov.field("intf::lr0_settings", "rx_pam_sel").unwrap(); + assert!(field.enum_values.is_some()); + } + + #[test] + fn shape_baseline_defaults_to_false() { + let f = write_tmp( + r#" +[shapes."intf::lr0_settings"] +fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(!ov.shape_baseline("intf::lr0_settings")); + } + + #[test] + fn baseline_defaults_to_false() { + // Existing override files that don't set `baseline` continue + // to emit `@default(...)` unchanged. + let f = write_tmp( + r#" +[shapes."intf::lr0_settings"] +fields.rx_pam_sel = { enum = ["NRZ", "PAM4"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + let field = ov.field("intf::lr0_settings", "rx_pam_sel").unwrap(); + assert!(!field.baseline); + } + + #[test] + fn missing_shape_returns_none() { + let f = write_tmp( + r#"[shapes."intf::gt_settings"] +fields.dummy = { enum = ["A", "B"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.field("intf::channel_map", "dummy").is_none()); + } + + #[test] + fn missing_field_within_present_shape_returns_none() { + let f = write_tmp( + r#"[shapes."intf::gt_settings"] +fields.dummy = { enum = ["A", "B"] } +"#, + ); + let ov = OverridesFile::load_from(f.path()).unwrap(); + assert!(ov.field("intf::gt_settings", "not_dummy").is_none()); + } + + #[test] + fn malformed_toml_errors_with_path_context() { + let f = write_tmp("this is not [ valid toml"); + let err = OverridesFile::load_from(f.path()).unwrap_err(); + // The error message should mention the path so a user seeing + // it in `vw` output can find the file that needs fixing. + assert!( + err.to_string().contains(&f.path().display().to_string()), + "err: {err}" + ); + } +} diff --git a/vw-ip/src/paired_list.rs b/vw-ip/src/paired_list.rs new file mode 100644 index 0000000..8bb86e6 --- /dev/null +++ b/vw-ip/src/paired_list.rs @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Parse Xilinx IP-XACT paired-list default values into a nested tree. +//! +//! Many Xilinx IP-XACT `` defaults are Tcl-list-shaped +//! paired dicts — `KEY VAL KEY VAL …`, where a `VAL` can itself be a +//! braced Tcl list holding another paired dict. The gtwiz-versal +//! `INTF0_TXRX_OPTIONAL_PORTS` param is the canonical example: ~300 +//! flat fields at the top level, terminating in +//! `INTF_LR_SETTINGS {LR0_SETTINGS {~80 fields} LR1_SETTINGS { } … }`. +//! +//! That structure IS the field schema. The generator uses it to emit +//! typed constructor procs with named args — no more asking the caller +//! to hand-populate `Properties::from -v {…}` from memory. See +//! `cips_dict::DictSchema::from_paired_default` for the extractor that +//! turns a [`PairedValue`] tree into a `DictSchema`. +//! +//! Semantics match Tcl list tokenization: whitespace separates tokens; +//! `{…}` groups a token, stripping only the outermost braces; nested +//! `{…}` are preserved verbatim inside the outer token. Nothing else is +//! interpreted — `$var` / `[cmd]` substitution isn't touched, since +//! IP-XACT defaults arrive with literal text only. + +use std::fmt; + +/// A parsed value inside a paired-list dict. A `Scalar` is a single +/// bare token (or a braced token whose interior isn't itself a +/// paired list). A `Nested` value is a braced token whose interior +/// parses as another paired list — recursively. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PairedValue { + Scalar(String), + Nested(Vec<(String, PairedValue)>), +} + +impl PairedValue { + /// True when this value is a `Scalar`. Convenience for callers + /// that iterate paired lists and treat scalar / nested slots + /// differently. + pub fn is_scalar(&self) -> bool { + matches!(self, Self::Scalar(_)) + } + + /// The underlying string for a `Scalar`, or `None` for `Nested`. + pub fn as_scalar(&self) -> Option<&str> { + match self { + Self::Scalar(s) => Some(s), + _ => None, + } + } +} + +impl fmt::Display for PairedValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Scalar(s) => write!(f, "{s}"), + Self::Nested(pairs) => { + write!(f, "{{")?; + for (i, (k, v)) in pairs.iter().enumerate() { + if i > 0 { + write!(f, " ")?; + } + write!(f, "{k} {v}")?; + } + write!(f, "}}") + } + } + } +} + +/// Parse a Tcl-list-shaped paired-dict default string into pairs. +/// +/// Returns an empty list when the input doesn't tokenize as an even- +/// count sequence — that's the "not really a paired dict" signal +/// callers use to fall back to treating the whole default as scalar. +/// Empty input returns empty pairs (a valid empty dict). +pub fn parse_paired_list(input: &str) -> Vec<(String, PairedValue)> { + let cleaned = fix_wrapped_tokens(input); + let tokens = tokenize(&cleaned); + if tokens.is_empty() || !tokens.len().is_multiple_of(2) { + return Vec::new(); + } + // Bail on non-ident-shaped keys at even indices. A dict whose + // "keys" don't look like idents is almost certainly a random + // scalar default that happened to have an even token count — + // treating it as pairs would produce garbage schema fields. + for (i, tok) in tokens.iter().enumerate() { + if i % 2 == 0 && !is_ident_shaped(tok) { + return Vec::new(); + } + } + let mut pairs = Vec::with_capacity(tokens.len() / 2); + let mut iter = tokens.into_iter(); + while let (Some(k), Some(v)) = (iter.next(), iter.next()) { + pairs.push((k, classify_value(&v))); + } + pairs +} + +/// Classify a token as `Scalar` or `Nested` by attempting to re-parse +/// its content as another paired list. If the re-parse yields at +/// least one valid pair, it's `Nested`; otherwise it's a `Scalar` +/// carrying the original text verbatim. +/// +/// An empty string classifies as `Nested([])` — an empty inner dict, +/// which is exactly the shape `LR1_SETTINGS { }` … `LR15_SETTINGS { }` +/// take in the `INTF_LR_SETTINGS` payload. Preserving the nested +/// classification for empties matters so the generator sees a +/// consistent tree shape across every LRn slot even when the anchor +/// only populates LR0. +fn classify_value(text: &str) -> PairedValue { + if text.is_empty() || text.chars().all(char::is_whitespace) { + return PairedValue::Nested(Vec::new()); + } + let inner = parse_paired_list(text); + if inner.is_empty() { + PairedValue::Scalar(text.to_string()) + } else { + PairedValue::Nested(inner) + } +} + +/// True when `s` is shaped like a bare identifier — leading letter or +/// underscore, then letters / digits / underscore / dot. Matches the +/// key form Xilinx uses everywhere in paired-list defaults +/// (`RX_REFCLK_FREQUENCY`, `ch_txdata`, `CONFIG.CPM_PCIE0_MODES`). +fn is_ident_shaped(s: &str) -> bool { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.') +} + +/// Repair Xilinx-style mid-identifier line wraps in IP-XACT +/// `` payloads. Their XML formatter wraps long lines +/// by inserting `\n` INSIDE a token (`ch_txpr\necursor3` instead of +/// `ch_txprecursor3`) — the standard Tcl-list tokenizer then splits +/// the token in half, blowing the paired-list even-count invariant. +/// +/// The repair: when `\n` sits between two identifier characters, +/// drop it — reconstructs the original token. Newlines that are +/// legitimate whitespace between tokens (adjacent to other +/// whitespace or non-identifier chars) survive unchanged. +fn fix_wrapped_tokens(input: &str) -> String { + let bytes = input.as_bytes(); + let mut out = String::with_capacity(input.len()); + for i in 0..bytes.len() { + let c = bytes[i] as char; + if c == '\n' { + let prev_ok = i > 0 && is_ident_byte(bytes[i - 1]); + let next_ok = i + 1 < bytes.len() && is_ident_byte(bytes[i + 1]); + if prev_ok && next_ok { + // Wrapped mid-token — drop. + continue; + } + } + out.push(c); + } + out +} + +fn is_ident_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b == b'.' +} + +/// Tokenize a Tcl list. Whitespace separates tokens; `{` opens a +/// braced token that swallows characters up to the matching `}`, +/// with nested `{…}` preserved verbatim (only the OUTER braces are +/// stripped from the emitted token). +/// +/// Unbalanced braces silently truncate — sufficient for well-formed +/// IP-XACT defaults and simpler than a full error type. Callers that +/// need to detect malformed input can compare tokenized length to +/// expected pair count. +fn tokenize(input: &str) -> Vec { + let mut tokens = Vec::new(); + let mut chars = input.chars().peekable(); + loop { + while chars.next_if(|c| c.is_whitespace()).is_some() {} + let Some(&c) = chars.peek() else { break }; + let mut tok = String::new(); + if c == '{' { + chars.next(); + let mut depth = 1_usize; + for ch in chars.by_ref() { + if ch == '{' { + depth += 1; + tok.push(ch); + } else if ch == '}' { + depth -= 1; + if depth == 0 { + break; + } + tok.push(ch); + } else { + tok.push(ch); + } + } + } else { + while let Some(&ch) = chars.peek() { + if ch.is_whitespace() { + break; + } + tok.push(ch); + chars.next(); + } + } + tokens.push(tok); + } + tokens +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scalar(s: &str) -> PairedValue { + PairedValue::Scalar(s.into()) + } + + fn nested(pairs: Vec<(&str, PairedValue)>) -> PairedValue { + PairedValue::Nested( + pairs.into_iter().map(|(k, v)| (k.into(), v)).collect(), + ) + } + + #[test] + fn flat_paired_list() { + // The `intf::channel_map` shape — 4 pairs, all scalar values. + let src = "INTF0_RX0 QUAD0_RX0 INTF0_TX0 QUAD0_TX0"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].0, "INTF0_RX0"); + assert_eq!(pairs[0].1, scalar("QUAD0_RX0")); + assert_eq!(pairs[1].0, "INTF0_TX0"); + assert_eq!(pairs[1].1, scalar("QUAD0_TX0")); + } + + #[test] + fn one_level_nested_lr_settings() { + // The `INTF_LR_SETTINGS` payload shape — one nested value. + let src = "LR0_SETTINGS {RX_REFCLK_FREQUENCY 156.25 TX_REFCLK_FREQUENCY 156.25}"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].0, "LR0_SETTINGS"); + assert_eq!( + pairs[0].1, + nested(vec![ + ("RX_REFCLK_FREQUENCY", scalar("156.25")), + ("TX_REFCLK_FREQUENCY", scalar("156.25")), + ]), + ); + } + + #[test] + fn two_level_nesting_matches_txrx_optional_ports_shape() { + // The `INTF0_TXRX_OPTIONAL_PORTS` anchor shape — flat outer + // pairs terminating in a nested INTF_LR_SETTINGS block whose + // inner values are themselves paired dicts. + let src = "GT_TYPE GTY GT_DIRECTION DUPLEX INTF_LR_SETTINGS \ + {LR0_SETTINGS {RX_HD_EN 0 TX_HD_EN 0} LR1_SETTINGS { }}"; + let pairs = parse_paired_list(src); + assert_eq!(pairs.len(), 3); + assert_eq!(pairs[0], ("GT_TYPE".into(), scalar("GTY"))); + assert_eq!(pairs[1], ("GT_DIRECTION".into(), scalar("DUPLEX"))); + let PairedValue::Nested(intf_lr) = &pairs[2].1 else { + panic!("expected nested INTF_LR_SETTINGS payload"); + }; + assert_eq!(intf_lr.len(), 2); + assert_eq!(intf_lr[0].0, "LR0_SETTINGS"); + assert_eq!( + intf_lr[0].1, + nested(vec![("RX_HD_EN", scalar("0")), ("TX_HD_EN", scalar("0")),]), + ); + // LR1_SETTINGS { } — empty braces classify as an empty + // Nested dict, not a Scalar with empty content. Critical: + // otherwise the schema for LR1 would come out as "one field + // named ''" instead of "nested slot, presently empty". + assert_eq!(intf_lr[1].0, "LR1_SETTINGS"); + assert_eq!(intf_lr[1].1, PairedValue::Nested(Vec::new())); + } + + #[test] + fn odd_token_count_yields_empty_pairs() { + // Signal to the caller: "this isn't a paired dict, treat the + // whole default as scalar." The `intf::gt_settings` param has + // default `0` (a single token) — must NOT parse as a pair. + assert!(parse_paired_list("0").is_empty()); + assert!(parse_paired_list("Custom").is_empty()); + assert!(parse_paired_list("A B C").is_empty()); + } + + #[test] + fn empty_input_yields_empty_pairs() { + assert!(parse_paired_list("").is_empty()); + assert!(parse_paired_list(" ").is_empty()); + } + + #[test] + fn non_ident_keys_reject_pair_interpretation() { + // `NA NA` — the current-generator misfire that would parse as + // one pair with key=NA (ident-shaped) value=NA. That IS + // ident-shaped so it DOES parse. Test the reject path with + // something that shouldn't. + assert!(parse_paired_list("32.0 GT/s").is_empty(), "leading digit"); + assert!(parse_paired_list("--foo bar").is_empty(), "leading punct"); + } + + #[test] + fn scalar_value_with_dot() { + // Frequencies (`156.25`, `10.3125`) tokenize as ident-shaped + // under our rules (dot allowed). Match Xilinx's usage — a + // pair like `RX_LINE_RATE 10.3125` should classify the value + // as scalar even though `10.3125` passes `is_ident_shaped`. + // classify_value tries to re-parse; a single token doesn't + // form a pair; falls back to Scalar. Belt-and-suspenders test. + let pairs = parse_paired_list("RX_LINE_RATE 10.3125"); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].1, scalar("10.3125")); + } + + #[test] + fn deeply_nested_braces_preserved() { + // Tokenizer strips ONLY the outer braces — inner `{…}` are + // kept verbatim so downstream re-parses see the same shape. + let toks = tokenize("A {B {C D} E} F G"); + assert_eq!(toks, vec!["A", "B {C D} E", "F", "G"]); + } + + #[test] + fn empty_braces_tokenize_as_empty_string() { + // `LR1_SETTINGS { }` — the value tokenizes to `""` (or `" "` + // then trimmed by classify_value). Either way, classifies as + // an empty nested dict, not a lone Scalar. + let toks = tokenize("K { }"); + assert_eq!(toks.len(), 2); + assert_eq!(toks[0], "K"); + assert!(toks[1].chars().all(char::is_whitespace) || toks[1].is_empty()); + } +} diff --git a/vw-ip/src/presets.rs b/vw-ip/src/presets.rs new file mode 100644 index 0000000..710abdb --- /dev/null +++ b/vw-ip/src/presets.rs @@ -0,0 +1,268 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Out-of-band parameter value sources for IP-XACT components. +//! +//! Some Xilinx IPs (notably CIPS / CPM5) ship the bulk of their +//! parameter enumerations *outside* the IP-XACT XML, in +//! `cpm_preset*.xml` files Vivado bundles under `data/versal/ps_pmc/`. +//! Without them, parameters like `CPM_PCIE1_PF0_BASE_CLASS_MENU` would +//! only carry their declared default in the generated `@enum(...)`, +//! and there's no other principled signal to recover the legal values +//! from. This module reads those files into a flat map the generator +//! can merge against the IP-XACT `` lists. +//! +//! The XML shape is uniform across the files I've seen: +//! +//! ```xml +//! +//! +//! +//! ... +//! +//! ``` + +use std::collections::{BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("reading preset file {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("parsing preset file {path}: {source}")] + Xml { + path: PathBuf, + #[source] + source: quick_xml::DeError, + }, +} + +/// `param_name → set of valid values`. `BTreeSet` keeps the iteration +/// order stable so generated `@enum(...)` lists are deterministic. +pub type PresetMap = HashMap>; + +#[derive(Debug, Default, Deserialize)] +struct Root { + #[serde(default, rename = "preset")] + entries: Vec, +} + +#[derive(Debug, Deserialize)] +struct Entry { + #[serde(rename = "@param")] + param: String, + #[serde(rename = "@name")] + name: String, +} + +/// Load one preset XML file into a fresh map. +pub fn load_file(path: &Path) -> Result { + let xml = fs::read_to_string(path).map_err(|source| Error::Io { + path: path.to_path_buf(), + source, + })?; + let root: Root = + quick_xml::de::from_str(&xml).map_err(|source| Error::Xml { + path: path.to_path_buf(), + source, + })?; + let mut map = PresetMap::new(); + for e in root.entries { + map.entry(e.param).or_default().insert(e.name); + } + Ok(map) +} + +/// Load several preset XML files and merge their entries into one map. +pub fn load_files(paths: I) -> Result +where + I: IntoIterator, + I::Item: AsRef, +{ + let mut merged = PresetMap::new(); + for p in paths { + let map = load_file(p.as_ref())?; + for (param, values) in map { + merged.entry(param).or_default().extend(values); + } + } + Ok(merged) +} + +/// Try to find sibling preset files for the IP whose +/// `component.xml` lives at `component_path`. +/// +/// Walks up from the component file looking for a Vivado-style +/// `data/` ancestor directory and then peeks at +/// `data/versal/ps_pmc//`. Any `*preset*.xml` found there +/// (recursively) is returned. Returns an empty vector — not an error — +/// when the layout doesn't match; the caller should treat it as a +/// best-effort hint. +pub fn discover_for(component_path: &Path) -> Vec { + let Some(data_root) = data_root_of(component_path) else { + return Vec::new(); + }; + let ip_name = ip_name_from(component_path); + let Some(ip_name) = ip_name else { + return Vec::new(); + }; + let ip_dir = data_root.join("versal").join("ps_pmc").join(&ip_name); + if !ip_dir.is_dir() { + return Vec::new(); + } + let mut out = Vec::new(); + collect_preset_files(&ip_dir, &mut out); + out.sort(); + out +} + +/// Recurse through `dir` collecting any `*preset*.xml` file paths. +fn collect_preset_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_preset_files(&path, out); + continue; + } + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + continue; + }; + if name.contains("preset") && name.ends_with(".xml") { + out.push(path); + } + } +} + +/// Walk up `component_path`'s ancestors looking for a directory +/// literally named `data` (Vivado's install root convention). +fn data_root_of(component_path: &Path) -> Option { + for ancestor in component_path.ancestors() { + if ancestor.file_name().and_then(|s| s.to_str()) == Some("data") { + return Some(ancestor.to_path_buf()); + } + } + None +} + +/// Recover an IP's short name from a Vivado-style versioned directory +/// (`cpm5_v1_0` → `cpm5`, `axi_dma_v7_1` → `axi_dma`). The trailing +/// `_v_` suffix is the convention Xilinx uses across IPs. +fn ip_name_from(component_path: &Path) -> Option { + let ip_dir = component_path.parent()?; + let name = ip_dir.file_name()?.to_str()?; + Some(strip_version_suffix(name).to_string()) +} + +fn strip_version_suffix(name: &str) -> &str { + // Find a trailing `_v_` and trim it. + let bytes = name.as_bytes(); + let mut end = bytes.len(); + // Trailing digits (minor) + while end > 0 && bytes[end - 1].is_ascii_digit() { + end -= 1; + } + if end == 0 || bytes[end - 1] != b'_' { + return name; + } + let after_minor = end; + end -= 1; // skip the `_` + while end > 0 && bytes[end - 1].is_ascii_digit() { + end -= 1; + } + let after_major_digits = end; + if end < 1 || &bytes[end.saturating_sub(2)..end] != b"_v" { + // Doesn't end in `_v_` — leave as-is. + return name; + } + let _ = after_minor; + let _ = after_major_digits; + &name[..end - 2] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_preset_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("p.xml"); + fs::write( + &path, + r#" + + + + "#, + ) + .unwrap(); + let m = load_file(&path).unwrap(); + let a: Vec<&str> = m["A"].iter().map(String::as_str).collect(); + assert_eq!(a, vec!["x", "y"]); + assert!(m["B"].contains("z")); + } + + #[test] + fn merges_multiple_files() { + let dir = tempfile::tempdir().unwrap(); + let p1 = dir.path().join("a.xml"); + fs::write(&p1, r#""#) + .unwrap(); + let p2 = dir.path().join("b.xml"); + fs::write(&p2, r#""#) + .unwrap(); + let m = load_files(&[p1, p2]).unwrap(); + let v: Vec<&str> = m["K"].iter().map(String::as_str).collect(); + assert_eq!(v, vec!["1", "2"]); + } + + #[test] + fn strips_xilinx_version_suffix() { + assert_eq!(strip_version_suffix("cpm5_v1_0"), "cpm5"); + assert_eq!(strip_version_suffix("axi_dma_v7_1"), "axi_dma"); + // No version → unchanged. + assert_eq!(strip_version_suffix("foo_bar"), "foo_bar"); + // Almost-but-not version → unchanged. + assert_eq!(strip_version_suffix("foo_v1"), "foo_v1"); + } + + #[test] + fn discovers_under_data_layout() { + let dir = tempfile::tempdir().unwrap(); + // Mimic Xilinx layout: data/ip/xilinx/_v1_0/component.xml + let data = dir.path().join("data"); + let ip = data.join("ip").join("xilinx").join("widget_v2_3"); + fs::create_dir_all(&ip).unwrap(); + let component = ip.join("component.xml"); + fs::write(&component, "").unwrap(); + // And sibling: data/versal/ps_pmc/widget/p.xml + let preset_dir = data.join("versal").join("ps_pmc").join("widget"); + fs::create_dir_all(&preset_dir).unwrap(); + let preset = preset_dir.join("my_preset.xml"); + fs::write(&preset, "").unwrap(); + // Unrelated file shouldn't be picked up. + fs::write(preset_dir.join("README.md"), "ignored").unwrap(); + + let found = discover_for(&component); + assert_eq!(found, vec![preset]); + } + + #[test] + fn discovery_empty_when_layout_doesnt_match() { + let dir = tempfile::tempdir().unwrap(); + let component = dir.path().join("loose.xml"); + fs::write(&component, "").unwrap(); + assert!(discover_for(&component).is_empty()); + } +} diff --git a/vw-ip/src/summary.rs b/vw-ip/src/summary.rs new file mode 100644 index 0000000..665eabd --- /dev/null +++ b/vw-ip/src/summary.rs @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! A small, human-friendly summary of an IP-XACT component — used by +//! `vw ip generate` to print what it's processing before emitting code. + +use ipxact::Component; + +#[derive(Clone, Debug)] +pub struct Summary { + pub vlnv: String, + pub description: Option, + pub parameter_count: usize, + pub user_parameter_count: usize, + pub model_parameter_count: usize, + pub port_count: usize, + pub choice_count: usize, +} + +impl Summary { + pub fn of(c: &Component) -> Self { + let parameters: Vec<_> = c.component_parameters().collect(); + let user_parameter_count = parameters + .iter() + .filter(|p| p.value.is_user_configurable()) + .count(); + Self { + vlnv: c.vlnv(), + description: c.description.clone(), + parameter_count: parameters.len(), + user_parameter_count, + model_parameter_count: c.model_parameters().count(), + port_count: c.ports().count(), + choice_count: c.choices().count(), + } + } +} diff --git a/vw-ip/src/targets.rs b/vw-ip/src/targets.rs new file mode 100644 index 0000000..b01ba8d --- /dev/null +++ b/vw-ip/src/targets.rs @@ -0,0 +1,312 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `[targets]` extraction for the generated `vw.toml`. +//! +//! Reads `` out of an IP's `component.xml`, +//! CATEGORIZES each entry by its `xilinx:lifeCycle` attribute, and +//! returns two brace-form pattern lists ready to be written into a +//! workspace `vw.toml` under `[targets]`: +//! +//! - `supported = [...]` — entries whose lifeCycle is `Production`, +//! `Beta`, or `Pre-Production`. Xilinx has blessed the IP for the +//! listed parts. +//! - `not-supported = [...]` — entries with `lifeCycle="Not-Supported"`. +//! Xilinx has attested the IP is NOT usable on those parts. +//! +//! The split matters because Vivado's IP catalog is not gated by +//! `` — `get_ipdefs` returns an IP even +//! for families not in the list. So a static "reject if the target +//! isn't listed" rule produces false positives (e.g. clk_wizard_v1_0 +//! has EVERY entry marked Not-Supported yet works fine on `xcvp1202` +//! in real projects). The list is a lifeCycle-tagged compatibility +//! matrix, not a filter — `vw check` uses it to distinguish +//! definitively-forbidden combinations (error) from merely-unblessed +//! ones (warning). See `vw_lib::TargetMismatchKind` for the check +//! side. +//! +//! Normalization rules per entry: +//! - Entries already in brace form (`versal{xcvm3(.*)}`) pass through +//! verbatim. +//! - Bare-family entries (`artix7`) get widened to `{.+}` — a +//! permissive placeholder that says "we support any part in this +//! family, but we haven't narrowed the pattern here." A future +//! extension can query Vivado at `vw ip generate` time to derive +//! precise regexes for each legacy family. +//! +//! The extractor is regex-driven rather than XML-parsed to avoid +//! adding an XML dep to `vw-ip`. The `` elements have a +//! tightly constrained shape in every component.xml we've observed; +//! the regex covers each cleanly. + +use std::path::Path; + +/// The two `[targets]` lists produced by [`extract_targets`], each +/// already normalized to brace form and ready for TOML upsert. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ExtractedTargets { + /// `Production` / `Beta` / `Pre-Production` entries. + pub supported: Vec, + /// `Not-Supported` entries. + pub not_supported: Vec, +} + +/// Read the `` entries from `component_path` and +/// split them by lifeCycle. Returns empty lists when the file has +/// no `` section (older or +/// non-family-aware component.xml). I/O errors and malformed XML +/// both surface as an empty result — the generator wraps this +/// with a warning when both lists are empty on an IP that clearly +/// should have families. +pub fn extract_targets(component_path: &Path) -> ExtractedTargets { + let Ok(xml) = std::fs::read_to_string(component_path) else { + return ExtractedTargets::default(); + }; + extract_targets_from_xml(&xml) +} + +/// Same as [`extract_targets`] but takes the XML text directly — +/// used by unit tests to feed known snippets without disk I/O. +pub fn extract_targets_from_xml(xml: &str) -> ExtractedTargets { + // `versal{xcvm3(.*)}` + // (with optional attributes on the opening tag). We care both + // about the attribute cluster (to sniff out `xilinx:lifeCycle`) + // and the interior text. + let entry_re = regex::Regex::new( + r#"(?s)]*)>\s*([^<]*?)\s*"#, + ) + .expect("family entry regex must compile"); + let lifecycle_re = + regex::Regex::new(r#"(?i)xilinx:lifeCycle\s*=\s*"([^"]*)""#) + .expect("lifecycle regex must compile"); + let mut out = ExtractedTargets::default(); + for cap in entry_re.captures_iter(xml) { + let attrs = &cap[1]; + let raw = cap[2].trim(); + if raw.is_empty() { + continue; + } + let normalized = normalize_family_entry(raw); + let lifecycle = lifecycle_re + .captures(attrs) + .map(|c| c[1].to_string()) + .unwrap_or_default(); + // Anything explicitly "Not-Supported" goes into the ban + // list. Every other value — Production, Beta, + // Pre-Production, or missing — is treated as blessed. + // Missing lifeCycle is the common case for older / + // non-annotated component.xml files; blessing is the safer + // default since a NON-match on the blessed list produces + // a warning, not an error. + if lifecycle.eq_ignore_ascii_case("Not-Supported") { + out.not_supported.push(normalized); + } else { + out.supported.push(normalized); + } + } + // Lift `ARCHITECTURE=` clauses out of the IP's + // `` and add a family-wide + // `{.+}` pattern per unique architecture. This + // captures the intent of IPs like `clk_wizard_v1_0` whose + // filter is `((ARCHITECTURE=versal)&&(MMCM>0))` — the IP is + // blessed for the entire versal architecture; the `MMCM > 0` + // and per-family `not-supported` entries then prune specific + // parts out. Without this lift, an IP whose supportedFamilies + // list is empty (or entirely Not-Supported) looks like it has + // no blessed patterns at all, which fires spurious "not + // blessed" warnings on parts Vivado will happily instantiate. + // + // We don't try to evaluate the boolean expression as a whole + // (capability clauses like `MMCM > 0` need per-part device + // properties we don't have statically). Extracting the pure + // architecture predicates is enough for the "blessed vs. not" + // question — capabilities and per-part bans still narrow at + // check time. + for arch in extract_architectures_from_filter(xml) { + let pat = format!("{arch}{{.+}}"); + if !out.supported.contains(&pat) { + out.supported.push(pat); + } + } + out +} + +/// Pull every `ARCHITECTURE=` clause out of any +/// `` block in `xml`. The +/// filter is a boolean expression written in a Xilinx-specific +/// mini-language, with entity-encoded operators (`&&`, +/// `>`). We only care about the architecture predicates; +/// capability clauses (`MMCM > 0`, `CPM5 > 0`, …) are ignored. +fn extract_architectures_from_filter(xml: &str) -> Vec { + let block_re = regex::Regex::new( + r#"(?s)\s*([^<]*?)\s*"#, + ) + .expect("autoDevicePropertiesFilter block regex must compile"); + let arch_re = + regex::Regex::new(r#"(?i)ARCHITECTURE\s*=\s*([A-Za-z0-9_]+)"#) + .expect("architecture predicate regex must compile"); + let mut out = Vec::new(); + for cap in block_re.captures_iter(xml) { + for a in arch_re.captures_iter(&cap[1]) { + let name = a[1].to_string(); + if !out.contains(&name) { + out.push(name); + } + } + } + out +} + +/// Normalize one raw `` payload into +/// `{}` form. Entries already in brace form pass +/// through verbatim. +fn normalize_family_entry(raw: &str) -> String { + if raw.contains('{') && raw.ends_with('}') { + return raw.to_string(); + } + // Bare family — widen to `family{.+}`. Any part name matches; + // the check effectively becomes "does the consumer target + // string exist at all," which is a strict subset of what + // Vivado would allow but never rejects a genuinely-supported + // part. + format!("{raw}{{.+}}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn brace_form_without_lifecycle_is_blessed() { + // No `xilinx:lifeCycle` attribute → treat as blessed + // (`supported`), since a missing lifecycle is the common + // shape in older component.xml files. + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn production_lifecycle_lands_in_supported() { + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn not_supported_lifecycle_lands_in_ban_list() { + let out = extract_targets_from_xml( + r#"versal{xcvm3(.*)}"#, + ); + assert_eq!(out.not_supported, vec!["versal{xcvm3(.*)}"]); + assert!(out.supported.is_empty()); + } + + #[test] + fn bare_family_widens_to_placeholder_regex() { + let out = extract_targets_from_xml( + r#"artix7"#, + ); + assert_eq!(out.supported, vec!["artix7{.+}"]); + } + + #[test] + fn mixed_lifecycles_are_split() { + let out = extract_targets_from_xml( + r#" + + versal{xcvm3(.*)} + artix7{xc7a35t(.*)} + versal{xcvp1202(.*)} + + "#, + ); + assert_eq!( + out.supported, + vec!["versal{xcvm3(.*)}", "versal{xcvp1202(.*)}"], + ); + assert_eq!(out.not_supported, vec!["artix7{xc7a35t(.*)}"]); + } + + #[test] + fn missing_section_returns_empty() { + let out = + extract_targets_from_xml("no families here"); + assert!(out.supported.is_empty()); + assert!(out.not_supported.is_empty()); + } + + #[test] + fn architecture_filter_widens_blessed_list() { + // clk_wizard_v1_0's exact shape: every family entry is + // Not-Supported, but the filter blesses the whole versal + // architecture. Result: `versal{.+}` in supported; + // specific parts still in not_supported. + let out = extract_targets_from_xml( + r#" + + versal{xa2ve3288(.*)} + versal{xc2ve3(.*)} + + ((ARCHITECTURE=versal)&&(MMCM>0)) + "#, + ); + assert_eq!(out.supported, vec!["versal{.+}"]); + assert_eq!( + out.not_supported, + vec!["versal{xa2ve3288(.*)}", "versal{xc2ve3(.*)}"], + ); + } + + #[test] + fn architecture_filter_dedupes_against_existing_supported() { + // If the supported list already carries a versal entry, + // adding another family-wide one would be redundant. But + // `versal{.+}` and `versal{xcvm3(.*)}` are DIFFERENT + // patterns; both belong. We only dedupe on exact-string + // equality so the same regex isn't repeated. + let out = extract_targets_from_xml( + r#" + + versal{xcvm3(.*)} + + (ARCHITECTURE=versal) + "#, + ); + assert_eq!(out.supported, vec!["versal{xcvm3(.*)}", "versal{.+}"]); + } + + #[test] + fn capability_only_filter_adds_nothing_to_supported() { + // dcmac_v3_0 / cpm5_v1_0 have capability-only filters — + // no ARCHITECTURE clause. Nothing to lift; the family + // list alone drives the blessed set. + let out = extract_targets_from_xml( + r#" + + versal{xcvp1202(.*)} + + (CPM5 > 0) + "#, + ); + assert_eq!(out.supported, vec!["versal{xcvp1202(.*)}"]); + } + + #[test] + fn multiple_architecture_clauses_each_lift() { + // Rare but possible: an IP that supports several + // architectures. Each named `ARCHITECTURE=` clause + // becomes its own family-wide pattern. + let out = extract_targets_from_xml( + r#" + ((ARCHITECTURE=versal) || (ARCHITECTURE=zynquplus)) + "#, + ); + assert_eq!(out.supported, vec!["versal{.+}", "zynquplus{.+}"]); + } +} diff --git a/vw-ip/src/tree.rs b/vw-ip/src/tree.rs new file mode 100644 index 0000000..98dc90c --- /dev/null +++ b/vw-ip/src/tree.rs @@ -0,0 +1,268 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Recursive prefix tree over parameter names. +//! +//! Big Xilinx IPs encode their configuration hierarchy in parameter +//! names, not in IP-XACT structure: `CPM_PCIE1_PF0_BAR0_64BIT` lives +//! under PCIE1 → PF0 → BAR0, but that's all conveyed by underscores. +//! A flat depth-1 grouping leaves PCIE1 with ~4200 args, which is +//! useless in an LSP. We recurse: at each depth, partition by the next +//! segment; subgroups bigger than `min_split_size` become children +//! that recurse again; everything smaller absorbs into the current +//! node as direct parameters. Generation walks the tree and emits one +//! proc per node, which keeps every proc small enough to navigate by +//! flag completion. + +use std::collections::BTreeMap; + +use ipxact::Parameter; + +#[derive(Clone, Debug)] +pub struct TreeOptions { + /// Don't split a subgroup into its own child node unless it has at + /// least this many parameters. Smaller subgroups stay as direct + /// args of the parent — keeps singleton segments from becoming + /// their own procs. + pub min_split_size: usize, +} + +impl Default for TreeOptions { + fn default() -> Self { + Self { min_split_size: 8 } + } +} + +#[derive(Clone, Debug)] +pub struct Node<'a> { + /// Full underscore-joined prefix that names this node + /// (e.g. `CPM_PCIE1_PF0`). Empty for the root. + pub label: String, + /// Underscore-separated depth: 0 at root, 1 for `CPM`, 2 for + /// `CPM_PCIE1`, 3 for `CPM_PCIE1_PF0`, ... + pub depth: usize, + /// Parameters whose proc-level args belong on *this* node. Their + /// arg names are derived by stripping the node's prefix. + pub direct: Vec<&'a Parameter>, + /// Child nodes, keyed by their additional segment. + pub children: Vec>, +} + +impl<'a> Node<'a> { + /// Total parameters reachable from this node, including children. + pub fn total_params(&self) -> usize { + self.direct.len() + + self.children.iter().map(Node::total_params).sum::() + } + + /// Number of nodes in this subtree, including self. + pub fn node_count(&self) -> usize { + 1 + self.children.iter().map(Node::node_count).sum::() + } + + /// Pre-order walk: visit self, then each child recursively. + pub fn walk(&self, f: &mut impl FnMut(&Node<'a>)) { + f(self); + for c in &self.children { + c.walk(f); + } + } + + /// True when this node has no sub-children — its whole + /// configuration surface lives in `direct`. Used by the + /// indexed-family collapse pass to gate whether a set of + /// sibling nodes can flatten into one constructor: nodes with + /// their own sub-trees carry more shape than a flat + /// `` value can capture. + pub fn is_leaf_only(&self) -> bool { + self.children.is_empty() + } + + /// Collect references to every node in this subtree in pre-order. + /// Used by code-gen, which needs to iterate the tree twice (once + /// for the header summary, once to emit procs) without re-walking + /// through a closure that can't escape `&Node` references. + pub fn collect<'t>(&'t self) -> Vec<&'t Node<'a>> { + let mut out = Vec::new(); + self.collect_into(&mut out); + out + } + + fn collect_into<'t>(&'t self, out: &mut Vec<&'t Node<'a>>) { + out.push(self); + for c in &self.children { + c.collect_into(out); + } + } +} + +/// Build the prefix tree from a flat parameter list. +pub fn build_tree<'a, I>(params: I, opts: &TreeOptions) -> Node<'a> +where + I: IntoIterator, +{ + build_node(0, String::new(), params.into_iter().collect(), opts) +} + +fn build_node<'a>( + depth: usize, + label: String, + params: Vec<&'a Parameter>, + opts: &TreeOptions, +) -> Node<'a> { + let mut direct: Vec<&'a Parameter> = Vec::new(); + let mut subgroups: BTreeMap> = BTreeMap::new(); + + for p in params { + let segs: Vec<&str> = p.name.split('_').collect(); + if depth + 1 >= segs.len() { + // No further segments to split on — this parameter belongs + // to the current node directly. + direct.push(p); + } else { + // Group by the segment at position `depth` — the next one + // not yet absorbed into the label. + subgroups + .entry(segs[depth].to_string()) + .or_default() + .push(p); + } + } + + let mut children = Vec::new(); + for (seg, group) in subgroups { + // A subgroup that's smaller than the split threshold isn't + // worth its own proc — keep its parameters at this level. + if group.len() < opts.min_split_size { + direct.extend(group); + continue; + } + let child_label = if label.is_empty() { + seg.clone() + } else { + format!("{label}_{seg}") + }; + children.push(build_node(depth + 1, child_label, group, opts)); + } + + Node { + label, + depth, + direct, + children, + } +} + +/// Return the portion of `param_name` after the node's `label_prefix` +/// (and the underscore separating them). Used so arg names inside a +/// node's proc don't redundantly repeat the prefix. +pub fn strip_prefix<'a>(param_name: &'a str, label_prefix: &str) -> &'a str { + if label_prefix.is_empty() { + return param_name; + } + if let Some(rest) = param_name.strip_prefix(label_prefix) { + rest.strip_prefix('_').unwrap_or(rest) + } else { + param_name + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(name: &str) -> Parameter { + Parameter { + name: name.into(), + ..Default::default() + } + } + + #[test] + fn empty_input_returns_empty_root() { + let tree = + build_tree(Vec::<&Parameter>::new(), &TreeOptions::default()); + assert_eq!(tree.label, ""); + assert_eq!(tree.direct.len(), 0); + assert_eq!(tree.children.len(), 0); + } + + #[test] + fn singletons_stay_at_root() { + let params = [p("A"), p("B"), p("C")]; + let opts = TreeOptions::default(); + let tree = build_tree(params.iter(), &opts); + // Each is one segment, no subgroups; all direct at root. + assert_eq!(tree.direct.len(), 3); + assert!(tree.children.is_empty()); + } + + #[test] + fn splits_when_subgroup_exceeds_threshold() { + let mut params: Vec = + (0..10).map(|i| p(&format!("CPM_PCIE1_FIELD{i}"))).collect(); + params.extend((0..10).map(|i| p(&format!("CPM_PCIE0_FIELD{i}")))); + let opts = TreeOptions { min_split_size: 5 }; + let tree = build_tree(params.iter(), &opts); + // Root has one child `CPM`; under `CPM`, children `PCIE0` and + // `PCIE1`, each with 10 direct params. + assert_eq!(tree.children.len(), 1); + let cpm = &tree.children[0]; + assert_eq!(cpm.label, "CPM"); + assert_eq!(cpm.children.len(), 2); + for c in &cpm.children { + assert_eq!(c.direct.len(), 10); + } + } + + #[test] + fn nested_hierarchy_splits_recursively() { + // Mimic PCIE1's structure: a bunch of PF0/PF1/PF2 sub-trees, + // each with BARs and CAPs. + let mut params: Vec = Vec::new(); + for pf in 0..3 { + for bar in 0..6 { + for f in 0..10 { + params.push(p(&format!( + "CPM_PCIE1_PF{pf}_BAR{bar}_FIELD{f}" + ))); + } + } + for cap in 0..3 { + for f in 0..5 { + params.push(p(&format!( + "CPM_PCIE1_PF{pf}_CAP{cap}_FIELD{f}" + ))); + } + } + } + let opts = TreeOptions { min_split_size: 5 }; + let tree = build_tree(params.iter(), &opts); + // Drill into the tree: root → CPM → PCIE1 → PF0/PF1/PF2. + let cpm = &tree.children[0]; + let pcie1 = &cpm.children[0]; + assert_eq!(pcie1.label, "CPM_PCIE1"); + assert_eq!(pcie1.children.len(), 3); // PF0, PF1, PF2 + let pf0 = &pcie1.children[0]; + // PF0 should have BAR0..BAR5 + CAP0..CAP2 as children. + let bar_count = pf0 + .children + .iter() + .filter(|c| c.label.contains("BAR")) + .count(); + assert_eq!(bar_count, 6, "{pf0:#?}"); + } + + #[test] + fn strip_prefix_returns_local_name() { + assert_eq!( + strip_prefix("CPM_PCIE1_PF0_BAR0_ENABLED", "CPM_PCIE1_PF0_BAR0"), + "ENABLED" + ); + // No prefix: returns the name unchanged. + assert_eq!(strip_prefix("FOO", ""), "FOO"); + // Prefix doesn't match: returns unchanged (defensive). + assert_eq!(strip_prefix("FOO_BAR", "BAZ"), "FOO_BAR"); + } +} diff --git a/vw-ip/tests/load_real_files.rs b/vw-ip/tests/load_real_files.rs new file mode 100644 index 0000000..243c3ef --- /dev/null +++ b/vw-ip/tests/load_real_files.rs @@ -0,0 +1,239 @@ +// Smoke tests that load the actual Xilinx IP-XACT files from the local +// Vivado install. Skipped automatically when the files aren't present +// so this still passes in CI without a Vivado install. + +use std::path::Path; + +use vw_ip::{generate, group_parameters, load, GenerateOptions, Summary}; + +const CIPS: &str = + "/home/ry/Xilinx/2025.1/data/rsb/iprepos/versal_cips_v3_4/component.xml"; +const CPM5: &str = + "/home/ry/Xilinx/2025.1/data/ip/xilinx/cpm5_v1_0/component.xml"; + +fn skip_if_missing(p: &str) -> bool { + if Path::new(p).exists() { + false + } else { + eprintln!("skipping: {p} not present"); + true + } +} + +#[test] +fn loads_cips_component() { + if skip_if_missing(CIPS) { + return; + } + let component = load(CIPS).expect("load CIPS"); + let summary = Summary::of(&component); + eprintln!("CIPS summary: {summary:#?}"); + assert!(summary.vlnv.contains("versal_cips")); +} + +#[test] +fn loads_cpm5_component() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let summary = Summary::of(&component); + eprintln!("CPM5 summary: {summary:#?}"); + assert!(summary.vlnv.contains("cpm5")); + // CPM5 should be huge. + assert!( + summary.parameter_count > 1000, + "expected many parameters, got {}", + summary.parameter_count + ); +} + +#[test] +fn generates_cips_wrapper_that_reparses() { + if skip_if_missing(CIPS) { + return; + } + let component = load(CIPS).expect("load CIPS"); + let out = generate( + &component, + &Default::default(), + &Default::default(), + &GenerateOptions::default(), + ) + .into_single(); + eprintln!("--- generated CIPS wrapper (first 60 lines) ---"); + for line in out.lines().take(60) { + eprintln!("{line}"); + } + eprintln!("--- ({} lines total) ---", out.lines().count()); + + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + + // Validate the generated wrapper against its own signature using + // the same validator the LSP runs. + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd (`ip::check`, + // `create_bd_cell`, `set_property`); those resolve when + // the wrapper is sourced through the loader, but this + // integration test validates the bare generated text. + // The unknown-call diagnostic is expected in that mode. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "validator errors: {errors:#?}"); +} + +#[test] +fn generates_cpm5_wrapper_in_split_mode() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let out = generate( + &component, + &Default::default(), + &Default::default(), + &GenerateOptions::default(), + ) + .into_single(); + + // Walk the generated source and measure per-proc arg counts so we + // can assert nothing is anywhere near the 4200-arg PCIE1 disaster + // we started with. + let mut proc_sizes: Vec<(String, usize)> = Vec::new(); + let mut current: Option<(String, usize)> = None; + let mut in_args = false; + // Track the indent of the current proc — procs live inside + // `namespace eval { … }` and get a 2-space prefix. + let mut proc_indent = 0usize; + for line in out.lines() { + let trimmed = line.trim_start(); + let indent = line.len() - trimmed.len(); + if let Some(name) = trimmed + .strip_prefix("proc ") + .and_then(|s| s.split_once(' ').map(|(n, _)| n)) + { + current = Some((name.to_string(), 0)); + in_args = true; + proc_indent = indent; + } else if trimmed == "} {" + || (trimmed.starts_with("} ") && trimmed.ends_with(" {")) + { + // `} {` is the old (untyped) body opener; `} TYPE {` + // (e.g. `} bd_cell {`, `} unit {`) is the new + // type-annotated form. Match on the exact indent so + // an inner `} {` in the body doesn't close the outer. + if indent == proc_indent { + if let Some(c) = current.take() { + proc_sizes.push(c); + } + in_args = false; + } + } else if in_args + && indent > proc_indent + && !trimmed.starts_with("##") + && !trimmed.is_empty() + { + if let Some(c) = current.as_mut() { + c.1 += 1; + } + } + } + proc_sizes.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); + let (max_name, max_size) = proc_sizes[0].clone(); + let total_procs = proc_sizes.len(); + eprintln!( + "CPM5 wrapper: {} procs, {} lines, biggest is {} ({} args)", + total_procs, + out.lines().count(), + max_name, + max_size + ); + for (n, s) in proc_sizes.iter().take(8) { + eprintln!(" {n:>40} = {s} args"); + } + + // Under the flat compositional model the top-level `create` + // proc IS the single composition point — one kwarg per split + // sub-tree constructor. For a giant IP like CPM5 that's a lot + // of kwargs but each is typed, LSP-navigable, and users + // typically only pass a handful. Cap at 1000 to guard against + // runaway growth without pretending we're back to the + // 200-arg hierarchical shape. + assert!( + max_size <= 1000, + "biggest proc {max_name} has {max_size} args; \ + even the compositional top proc shouldn't exceed 1000" + ); + // And the overall proc count should reflect that we *are* splitting. + assert!( + total_procs > 50, + "only {total_procs} procs — hierarchy isn't being built" + ); + + // Under the compositional model + namespace-eval wrapping: + // - Newtype preludes emit at file top level (outside the block). + // - Inside `namespace eval cpm5 { … }`: value constructors, + // the top-level `create` proc, all with bare names and + // 2-space indent. + // The exact ordering of constructors depends on tree traversal; + // just assert the wrapping block exists and expected procs + // appear inside. + assert!( + out.contains("namespace eval cpm5 {"), + "{}", + &out[..out.len().min(1200)] + ); + assert!(out.contains(" proc create {")); + // Split-node procs use fully-qualified names now — they live + // OUTSIDE the namespace-block (in sibling `.htcl` files after + // the split pass) so they can be individually loaded without + // Tcl doubling the namespace prefix. + assert!(out.contains("proc cpm5::cpm_pcie0 ")); + assert!(out.contains("proc cpm5::cpm_pcie1 ")); + + let parsed = vw_htcl::parse(&out); + assert!( + parsed.errors.is_empty(), + "parse errors: {:?}", + parsed.errors + ); + let diags = vw_htcl::validate(&parsed.document, &out); + let errors: Vec<_> = diags + .iter() + .filter(|d| d.severity == vw_htcl::Severity::Error) + // The generator emits calls into vivado-cmd (`ip::check`, + // `create_bd_cell`, `set_property`); those resolve when + // the wrapper is sourced through the loader, but this + // integration test validates the bare generated text. + // The unknown-call diagnostic is expected in that mode. + .filter(|d| !d.message.starts_with("undefined proc")) + .collect(); + assert!(errors.is_empty(), "validator errors: {errors:#?}"); +} + +#[test] +fn groups_cpm5_parameters_into_handful_of_buckets() { + if skip_if_missing(CPM5) { + return; + } + let component = load(CPM5).expect("load CPM5"); + let params: Vec<_> = component.component_parameters().collect(); + let groups = group_parameters(params.iter().copied(), 2); + eprintln!("CPM5 has {} groups at prefix=2:", groups.len()); + for g in groups.iter().take(20) { + eprintln!(" {:>32} = {} params", g.key, g.parameters.len()); + } + eprintln!(" ... ({} total)", groups.len()); + // We expect a manageable number of groups (not one giant flat list, + // not thousands of singletons). + assert!(groups.len() < 200, "too many groups: {}", groups.len()); + assert!(groups.len() > 2, "too few groups: {}", groups.len()); +} diff --git a/vw-lib/Cargo.toml b/vw-lib/Cargo.toml index d05133f..f334ccc 100644 --- a/vw-lib/Cargo.toml +++ b/vw-lib/Cargo.toml @@ -9,6 +9,8 @@ keywords = ["vhdl", "workspace", "dependency-management"] categories = ["development-tools"] [dependencies] +vw-core = { path = "../vw-core" } +anodizer.workspace = true serde.workspace = true serde_json.workspace = true toml.workspace = true @@ -23,7 +25,8 @@ url.workspace = true glob.workspace = true petgraph.workspace = true git2 = "0.18" -vhdl_lang = "0.86" +vhdl_lang.workspace = true +# Used by sim/bridge.rs to codegen the Rust<->VHDL cosim bridge. quote = "1" proc-macro2 = "1" syn = "2" diff --git a/vw-lib/src/lib.rs b/vw-lib/src/lib.rs index 7e3ab38..d07edda 100644 --- a/vw-lib/src/lib.rs +++ b/vw-lib/src/lib.rs @@ -19,7 +19,7 @@ //! let workspace_dir = Utf8Path::new("."); //! //! // Initialize a new workspace -//! init_workspace(workspace_dir, "my_project".to_string())?; +//! init_workspace(workspace_dir, "my_project".to_string(), None)?; //! //! // Update dependencies //! update_workspace(workspace_dir).await?; @@ -28,187 +28,693 @@ //! ``` use std::cell::RefCell; -use std::collections::{hash_map::Entry, HashMap, HashSet, VecDeque}; +use std::collections::{hash_map::Entry, HashMap, HashSet}; +use std::fs; use std::path::{Path, PathBuf}; -use std::{fmt, fs}; use camino::{Utf8Path, Utf8PathBuf}; +use petgraph::graph::{DiGraph, NodeIndex}; use serde::{Deserialize, Serialize}; -use vhdl_lang::{VHDLParser, VHDLStandard}; -use petgraph::{ - algo::toposort, - graph::{DiGraph, NodeIndex}, -}; - -use crate::mapping::{FileData, SymbolKind, VwSymbol, VwSymbolFinder}; use crate::nvc_helpers::{run_nvc_analysis, run_nvc_elab, run_nvc_sim}; -use crate::visitor::walk_design_file; +use vw_core::parse_entities; -pub mod mapping; -pub mod nvc_helpers; +pub mod parts; pub mod sim; -pub mod visitor; -const BUILD_DIR: &str = "vw_build"; +// The low-level VHDL-analysis + nvc primitives now live in `vw-core`. +// Re-export them so existing `crate::…` / `vw_lib::…` paths (in this crate's +// remaining workflow code and in downstream crates) keep resolving. +pub use vw_core::{ + analyze_ext_libraries, find_referenced_files, load_existing_vhdl_ls_config, + sort_files_by_dependencies, FileCache, RecordProcessor, Result, + VhdlLsConfig, VhdlLsLibrary, VhdlStandard, VwError, +}; +pub use vw_core::{mapping, nvc_helpers, visitor}; + +/// Workspace-relative directory for vw's own testbench simulation build (the +/// nvc `work` + dependency libraries). Kept under `target/` so all generated +/// output lives there. (anodizer's separate build is `target/anodizer/build`.) +pub const BUILD_DIR: &str = "target/sim"; // ============================================================================ -// Error Types +// Configuration Structures // ============================================================================ -#[derive(Debug)] -pub enum VwError { - Config { message: String }, - Dependency { message: String }, - Git { message: String }, - FileSystem { message: String }, - Testbench { message: String }, - NvcSimulation { command: String }, - NvcElab { command: String }, - NvcAnalysis { library: String, command: String }, - CodeGen { message: String }, - Simulation { message: String }, - Io(std::io::Error), - Serialization(toml::ser::Error), - Deserialization(toml::de::Error), - Regex(regex::Error), +#[derive(Debug, Deserialize, Serialize)] +pub struct WorkspaceConfig { + #[allow(dead_code)] + pub workspace: WorkspaceInfo, + #[serde(default)] + pub dependencies: HashMap, + /// Test-only dependencies. Only the ENTRY workspace's + /// `[test-dependencies]` are honored — a transitive dep's + /// test-deps are private to itself. Cargo-parity semantic for + /// `dev-dependencies`. Consumed by `vw test` via + /// [`transitive_dep_cache_paths_with_test`]. + #[serde(default, rename = "test-dependencies")] + pub test_dependencies: HashMap, + /// Library-scope: the set of Vivado device families/parts the + /// files in this workspace support. Populated by `vw ip + /// generate` from the underlying IP's ``, normalized so every entry has a brace-form + /// regex against the raw part name. + /// + /// Project workspaces (those that declare `[workspace] + /// target-part`) omit this — projects consume libraries, they + /// don't publish their own supported-parts list. + #[serde(default)] + pub targets: Option, + #[serde(default)] + pub tools: Option, } -impl std::error::Error for VwError {} -impl From for VwError { - fn from(err: std::io::Error) -> Self { - VwError::Io(err) - } +#[derive(Debug, Deserialize, Serialize)] +pub struct WorkspaceInfo { + #[allow(dead_code)] + pub name: String, + #[allow(dead_code)] + pub version: String, + /// Project-scope: every Vivado device part this workspace can + /// target. Each entry is a full Vivado specifier (e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`) — package and speed grade + /// matter for downstream implementation / timing analysis + /// even if the family alone is enough for IP-availability + /// checks. + /// + /// One entry must be marked `default = true` when the list + /// has more than one; the default drives the auto-project + /// on `vw run` / `vw repl` / `vw test`. The CLI's + /// `--part ` flag selects a non-default + /// entry. + /// + /// Empty for library workspaces (they publish `[targets]` + /// instead — see [`TargetsConfig`]). + /// + /// Mutually exclusive with [`variants`](Self::variants) — a + /// workspace declares one shape or the other, never both. + /// Variants own their parts inline; `[[target-parts]]` is for + /// projects whose parts are truly interchangeable and don't + /// change what source files compile. + #[serde(default, rename = "target-parts")] + pub target_parts: Vec, + /// Project-scope: named feature-flag-style variants. Each + /// variant declares its own part inline and an optional list + /// of `exclusive` file paths (workspace-relative globs) that + /// are ONLY compiled when that variant is active. + /// + /// Mutually exclusive with [`target_parts`](Self::target_parts) + /// — see the docstring there. Empty for the common + /// "no variants" case, in which case part selection is + /// driven purely by `[[target-parts]]`. + /// + /// One entry must be marked `default = true` when the list + /// has more than one; the default drives the auto-project on + /// `vw run` / `vw repl` / `vw test`. The CLI's + /// `--variant ` flag selects a non-default entry. + #[serde(default)] + pub variants: Vec, + /// Project-scope: default top-entity name. Consumed by + /// `vw::synth` (as the fallback when `-top` isn't passed) and + /// by `vw::_resolve_top` (used by `vw::place` / `vw::route` / + /// `vw::report` to derive the DCP / report paths). + /// + /// A variant with its own `top` overrides this — see + /// [`Variant::top`]. When neither this nor the active + /// variant's `top` is set, callers fall back to fileset TOP / + /// current_design NAME / an explicit `-top` from the user. + /// + /// Typical usage: a workspace with a single top-level entity + /// sets this once at project scope. Multi-variant workspaces + /// with per-board tops (e.g. `top_vpk120` / `top_metro`) set + /// it per-variant instead. + #[serde(default)] + pub top: Option, } -impl From for VwError { - fn from(err: toml::ser::Error) -> Self { - VwError::Serialization(err) - } +/// One entry in a workspace's `[[target-parts]]` list. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TargetPart { + /// Full Vivado part identifier, e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`. + pub part: String, + /// `true` marks this entry as the default target part. Exactly + /// one entry must set this when the list has more than one + /// entry. A single-entry list may omit the flag — the sole + /// entry is implicitly the default. + #[serde(default)] + pub default: bool, } -impl From for VwError { - fn from(err: toml::de::Error) -> Self { - VwError::Deserialization(err) - } +/// Errors surfaced when validating or selecting a target part +/// from a workspace's `[[target-parts]]` list. Wrapped by the +/// `WorkspaceInfo` accessors so callers can render precise +/// diagnostics. +#[derive(Debug, thiserror::Error)] +pub enum TargetSelectError { + #[error( + "workspace has {count} `[[target-parts]]` entries but none are marked \ + `default = true`; add `default = true` to exactly one entry" + )] + NoDefault { count: usize }, + #[error( + "workspace has multiple `[[target-parts]]` entries marked \ + `default = true` ({defaults:?}); only one may be default" + )] + MultipleDefaults { defaults: Vec }, + #[error("no `[[target-parts]]` entry matches `{query}`")] + NoMatch { query: String }, + #[error( + "`{query}` matches multiple `[[target-parts]]` entries ({matches:?}); \ + disambiguate with a longer substring or the full part ID" + )] + Ambiguous { query: String, matches: Vec }, } -impl From for VwError { - fn from(err: regex::Error) -> Self { - VwError::Regex(err) - } +/// One entry in a workspace's `[[workspace.variants]]` list. +/// +/// A variant is a feature-flag-style selection that: +/// - pins a specific Vivado part (inline, not via cross-reference) +/// - optionally names an `exclusive` list of source-file globs +/// (workspace-relative) that ONLY compile when this variant +/// is the active one. +/// +/// Files NOT listed in any variant's `exclusive` set are shared — +/// they always contribute to `vhdl_design_sources` regardless +/// of the active variant. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Variant { + /// Human-facing selector, e.g. `vpk120` or `metro`. Matched + /// exactly by `--variant `. Must be unique within a + /// workspace's variants list. + pub name: String, + /// Full Vivado part identifier, e.g. + /// `xcvp1202-vsva2785-2MHP-e-S`. Drives the auto-project + /// when this variant is active. + pub part: String, + /// `true` marks this entry as the default variant. Exactly + /// one entry must set this when the list has more than one. + /// A single-entry list may omit the flag — the sole entry + /// is implicitly the default. + #[serde(default)] + pub default: bool, + /// Workspace-relative globs (e.g. `"hdl/ethernet-vpk120.vhd"` + /// or `"hdl/board-vpk120/**/*.vhd"`) matching files that + /// ONLY compile when this variant is active. Files that + /// don't match any variant's exclusive set are always shared. + #[serde(default)] + pub exclusive: Vec, + /// Per-variant top-entity name. Overrides + /// [`WorkspaceInfo::top`] when this variant is active. + /// The typical multi-variant shape is + /// `top_` — one wrapper per board. + #[serde(default)] + pub top: Option, } -pub type Result = std::result::Result; +/// Errors surfaced when validating or selecting a variant from a +/// workspace's `[[variants]]` list. Same shape as +/// [`TargetSelectError`] with variant-flavored messages. +#[derive(Debug, thiserror::Error)] +pub enum VariantSelectError { + #[error( + "workspace declares both `[[target-parts]]` and \ + `[[workspace.variants]]` — they're mutually exclusive; \ + variants own their parts inline, so remove `[[target-parts]]`" + )] + BothPartsAndVariants, + #[error( + "workspace has {count} `[[workspace.variants]]` entries but \ + none are marked `default = true`; add `default = true` to \ + exactly one entry" + )] + NoDefault { count: usize }, + #[error( + "workspace has multiple `[[workspace.variants]]` entries \ + marked `default = true` ({defaults:?}); only one may be default" + )] + MultipleDefaults { defaults: Vec }, + #[error("no `[[workspace.variants]]` entry named `{query}`")] + NoMatch { query: String }, + #[error( + "duplicate variant name `{name}` in \ + `[[workspace.variants]]` — variant names must be unique" + )] + DuplicateName { name: String }, +} -impl fmt::Display for VwError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VwError::NvcSimulation { command } => { - writeln!(f, "NVC simulation failed")?; - writeln!(f, "command:")?; - writeln!(f, "{command}")?; - Ok(()) - } - VwError::NvcElab { command } => { - writeln!(f, "NVC elaboration failed")?; - writeln!(f, "command:")?; - writeln!(f, "{command}")?; - Ok(()) - } - VwError::NvcAnalysis { library, command } => { - writeln!(f, "NVC analysis failed for library '{library}'")?; - writeln!(f, "command:")?; - writeln!(f, "{command}")?; - Ok(()) - } - VwError::CodeGen { message } => { - write!(f, "Code generation failed: {message}") - } - VwError::Simulation { message } => { - write!(f, "Simulation error: {message}") - } - VwError::Config { message } => { - write!(f, "Configuration error: {message}") - } - VwError::Dependency { message } => { - write!(f, "Dependency error: {message}") - } - VwError::Git { message } => { - write!(f, "Git operation failed: {message}") - } - VwError::FileSystem { message } => { - write!(f, "File system error: {message}") +impl WorkspaceInfo { + /// Return the default target part, if any. Empty list yields + /// `Ok(None)`. Single-entry list yields that entry as the + /// implicit default (regardless of the `default` flag). + /// Multi-entry list requires exactly one `default = true`. + pub fn default_target_part( + &self, + ) -> std::result::Result, TargetSelectError> { + match self.target_parts.len() { + 0 => Ok(None), + 1 => Ok(Some(self.target_parts[0].part.as_str())), + _ => { + let defaults: Vec<&TargetPart> = + self.target_parts.iter().filter(|p| p.default).collect(); + match defaults.len() { + 1 => Ok(Some(defaults[0].part.as_str())), + 0 => Err(TargetSelectError::NoDefault { + count: self.target_parts.len(), + }), + _ => Err(TargetSelectError::MultipleDefaults { + defaults: defaults + .iter() + .map(|p| p.part.clone()) + .collect(), + }), + } } - VwError::Testbench { message } => { - write!(f, "Testbench error: {message}") + } + } + + /// Resolve a CLI `--part ` selector against the + /// workspace's target parts. `None` returns the default (via + /// [`default_target_part`](Self::default_target_part)). `Some` + /// matches by exact part ID first; failing that, by unique + /// substring. Multiple substring matches, or zero matches, + /// error out. + pub fn select_target_part( + &self, + query: Option<&str>, + ) -> std::result::Result, TargetSelectError> { + let Some(q) = query else { + return self.default_target_part(); + }; + if let Some(exact) = self.target_parts.iter().find(|p| p.part == q) { + return Ok(Some(exact.part.as_str())); + } + let matches: Vec<&TargetPart> = self + .target_parts + .iter() + .filter(|p| p.part.contains(q)) + .collect(); + match matches.len() { + 1 => Ok(Some(matches[0].part.as_str())), + 0 => Err(TargetSelectError::NoMatch { + query: q.to_string(), + }), + _ => Err(TargetSelectError::Ambiguous { + query: q.to_string(), + matches: matches.iter().map(|p| p.part.clone()).collect(), + }), + } + } + + /// Return the default variant, if any. Empty list yields + /// `Ok(None)` (workspaces without variants). Single-entry + /// list yields that entry as the implicit default + /// (regardless of the `default` flag). Multi-entry list + /// requires exactly one `default = true`. + pub fn default_variant( + &self, + ) -> std::result::Result, VariantSelectError> { + match self.variants.len() { + 0 => Ok(None), + 1 => Ok(Some(&self.variants[0])), + _ => { + let defaults: Vec<&Variant> = + self.variants.iter().filter(|v| v.default).collect(); + match defaults.len() { + 1 => Ok(Some(defaults[0])), + 0 => Err(VariantSelectError::NoDefault { + count: self.variants.len(), + }), + _ => Err(VariantSelectError::MultipleDefaults { + defaults: defaults + .iter() + .map(|v| v.name.clone()) + .collect(), + }), + } } - VwError::Io(e) => write!(f, "IO error: {e}"), - VwError::Serialization(e) => write!(f, "Serialization error: {e}"), - VwError::Deserialization(e) => { - write!(f, "Deserialization error: {e}") + } + } + + /// Resolve a CLI `--variant ` selector against the + /// workspace's variants. `None` returns the default (via + /// [`default_variant`](Self::default_variant)). `Some` + /// matches by exact name only — unlike `--part`, no + /// substring fallback: variant names are short enough that + /// substring matching would be more confusing than useful. + pub fn select_variant( + &self, + query: Option<&str>, + ) -> std::result::Result, VariantSelectError> { + let Some(q) = query else { + return self.default_variant(); + }; + self.variants + .iter() + .find(|v| v.name == q) + .map(Some) + .ok_or_else(|| VariantSelectError::NoMatch { + query: q.to_string(), + }) + } + + /// Resolve the top-entity name for the given active variant. + /// Precedence: variant's `top` (when a variant is active and + /// sets one) wins over the workspace-level `top`. Returns + /// `None` when neither is configured — callers (`vw::synth` + /// via the `top` RPC + `vw::top` proc) then decide whether to + /// error or fall back to other sources like the fileset TOP + /// property. + /// + /// `active_variant`: the variant name the caller resolved + /// (typically via `select_variant`). Pass `None` for + /// no-variant workspaces or when the caller hasn't picked + /// one yet. + pub fn resolve_top(&self, active_variant: Option<&str>) -> Option { + if let Some(vname) = active_variant { + if let Some(v) = self.variants.iter().find(|v| v.name == vname) { + if let Some(t) = &v.top { + return Some(t.clone()); + } } - VwError::Regex(e) => write!(f, "Regex error: {e}"), } + self.top.clone() } } -// ============================================================================ -// VHDL Standard -// ============================================================================ +/// Library-scope target metadata. Populated by `vw ip generate` +/// so downstream `vw check` can verify a consumer's target-part +/// is supported by every transitive dep without ever touching +/// Vivado at check time. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TargetsConfig { + /// Blessed patterns — `` entries whose + /// `lifeCycle` is `Production`, `Beta`, or `Pre-Production`. + /// A target-part that matches any of these is a clean pass; + /// the check emits no diagnostic for that dep. + /// + /// Every string is in the form `{}` + /// — see [`parse_target_pattern`] for the parse rules. Bare + /// family names are rejected; `vw ip generate` normalizes them + /// before writing. + #[serde(default)] + pub supported: Vec, + /// Explicitly-unsupported patterns — `` entries + /// with `lifeCycle="Not-Supported"`. Xilinx has attested that + /// the IP is NOT usable on parts matching these patterns; if + /// the target-part matches one, `vw check` fires an ERROR. + /// + /// Note: an entire component.xml with only Not-Supported + /// entries (e.g. an experimental IP still in incubation) + /// leaves `supported = []` and populates only this list. In + /// that case the check treats a NON-match here as "unblessed + /// but not forbidden" — a warning, not an error. + #[serde(default, rename = "not-supported")] + pub not_supported: Vec, +} + +/// A parsed `family{regex}` target pattern. The regex is +/// pre-compiled at parse time so hot paths (validator, LSP +/// diagnostics) don't re-compile per-check. +#[derive(Debug, Clone)] +pub struct TargetPattern { + /// The family word out front — `versal`, `artix7`, etc. Kept + /// verbatim for diagnostics: "clk-wizard supports the versal + /// family; your target `xcvm3358…` isn't in that family." + pub family: String, + /// The compiled regex from inside the braces. Applied against + /// the consumer's target-part string; a match means the + /// library supports the target. + pub regex: regex::Regex, + /// Original source text (`versal{xcvm3(.*)}`) — used to + /// point diagnostics back at the exact vw.toml line. + pub raw: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum TargetParseError { + #[error("target pattern `{raw}` is missing the `{{regex}}` part")] + MissingBraces { raw: String }, + #[error("target pattern `{raw}`: {source}")] + BadRegex { + raw: String, + #[source] + source: regex::Error, + }, +} -#[derive(Clone, Copy, Debug)] -pub enum VhdlStandard { - Vhdl2008, - Vhdl2019, +/// Parse one entry of `[targets].supported` — the form +/// `{}`. Bare-family entries (no braces) are +/// rejected with [`TargetParseError::MissingBraces`]; the +/// generator's job is to normalize them into brace form before +/// they reach downstream consumers. +/// Snapshot of per-dep target-support metadata used by the +/// project-vs-dep compatibility check. Populated from each dep's +/// `[targets] supported` list at the entry workspace's transitive +/// walk time; used later by [`check_target_compatibility`] to +/// verify a given target-part is supported. +/// +/// Deps with no `[targets]` at all are `Vec::new()` — the check +/// treats an empty list as "universal / no constraint," so +/// non-IP libraries (@vw, @test) don't need to declare anything. +/// +/// Parse errors during pattern compile are turned into +/// `(dep_name, error)` entries in the second field so callers +/// can surface them as diagnostics without dropping the whole +/// dep's info. +#[derive(Debug, Default)] +pub struct DepTargets { + /// Blessed (`Production`/`Beta`/`Pre-Production`) patterns + /// per dep. A target-part matching any of these clears the + /// check clean. + pub per_dep: HashMap>, + /// Explicitly `Not-Supported` patterns per dep. A target-part + /// matching any of these is an ERROR — Xilinx has attested + /// the IP is not usable there. + pub per_dep_not_supported: HashMap>, + pub errors: Vec<(String, TargetParseError)>, } -impl From for VHDLStandard { - fn from(val: VhdlStandard) -> Self { - match val { - VhdlStandard::Vhdl2008 => VHDLStandard::VHDL2008, - VhdlStandard::Vhdl2019 => VHDLStandard::VHDL2019, +/// Walk the entry workspace's transitive deps and collect each +/// dep's `[targets].supported` patterns. Returns a +/// [`DepTargets`] snapshot ready to feed into +/// [`check_target_compatibility`]. +/// +/// Skips deps whose `vw.toml` doesn't load — they contribute +/// nothing (treated as universal). This matches the "empty = +/// universal" policy the compat check applies. +pub fn collect_dep_targets(entry_workspace_dir: &Utf8Path) -> DepTargets { + let mut out = DepTargets::default(); + let Ok(paths) = transitive_dep_cache_paths(entry_workspace_dir) else { + return out; + }; + for (name, path) in paths { + let Ok(path_utf8) = Utf8PathBuf::from_path_buf(path) else { + continue; + }; + let Ok(cfg) = load_workspace_config(&path_utf8) else { + continue; + }; + let Some(targets) = cfg.targets else { + out.per_dep.insert(name.clone(), Vec::new()); + out.per_dep_not_supported.insert(name, Vec::new()); + continue; + }; + let mut compiled: Vec = Vec::new(); + for raw in &targets.supported { + match parse_target_pattern(raw) { + Ok(p) => compiled.push(p), + Err(e) => out.errors.push((name.clone(), e)), + } + } + let mut compiled_not_supported: Vec = Vec::new(); + for raw in &targets.not_supported { + match parse_target_pattern(raw) { + Ok(p) => compiled_not_supported.push(p), + Err(e) => out.errors.push((name.clone(), e)), + } } + out.per_dep.insert(name.clone(), compiled); + out.per_dep_not_supported + .insert(name, compiled_not_supported); } + out +} + +/// Severity of a target-compatibility mismatch. The two-tier +/// treatment reflects what `` actually +/// means in Vivado's IP catalog: the family list is a lifeCycle- +/// tagged compatibility MATRIX, not a hard availability filter. +/// Vivado exposes IPs even for families that aren't listed. So: +/// +/// - `NotSupported` — target-part matched a Not-Supported entry. +/// Xilinx explicitly attests the IP won't work here. Error. +/// - `Unblessed` — target-part matched no entry at all (neither +/// supported nor not-supported). Warning: the IP MAY work but +/// Xilinx hasn't blessed the combination. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetMismatchKind { + NotSupported, + Unblessed, } -impl fmt::Display for VhdlStandard { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VhdlStandard::Vhdl2008 => write!(f, "2008"), - VhdlStandard::Vhdl2019 => write!(f, "2019"), +/// One target-compatibility observation. Emitted by +/// [`check_target_compatibility`]; the caller uses `kind` to +/// choose diagnostic severity (error vs warning). +#[derive(Debug, Clone)] +pub struct TargetMismatch { + /// Dep name (e.g. `clk-wizard`). + pub dep: String, + /// The target part string that was checked. + pub target_part: String, + /// Family cues gathered from the dep's BLESSED patterns — + /// its `[targets] supported` list. Empty when the dep has no + /// blessed patterns (e.g. clk-wizard v1.0, which has only + /// `not-supported` entries). The diagnostic message uses this + /// to say "clk-wizard blesses parts in the following families; + /// yours isn't among them." + pub supported_families: Vec, + /// Family cues gathered from the dep's BAN LIST — its + /// `[targets] not-supported` list. Reported separately from + /// `supported_families` because the semantics differ: + /// families named in `supported` are blessed for use, families + /// named ONLY in `not-supported` are ones Xilinx has attested + /// don't work (at least for specific parts). Reporting them + /// as "declared families" without qualification is misleading + /// (see #vw-check clarity). + pub not_supported_families: Vec, + /// Whether Xilinx explicitly forbids this combination or + /// simply hasn't blessed it. + pub kind: TargetMismatchKind, +} + +/// Return each dep's target-compatibility observation against +/// `target_part`. Deps with no `[targets]` at all (`supported` +/// AND `not_supported` both empty) contribute nothing — treated +/// as universal. When `target_part` is `None` (library workspace, +/// no project target) the check is a no-op. +/// +/// Decision matrix per dep: +/// - matches a Not-Supported pattern → `NotSupported` (error). +/// Trumps a supported match — an explicit ban wins. +/// - matches a Supported pattern → clean pass, no observation. +/// - matches neither, but the dep has SOME patterns declared → +/// `Unblessed` (warning). +pub fn check_target_compatibility( + target_part: Option<&str>, + dep_targets: &DepTargets, +) -> Vec { + let Some(part) = target_part else { + return Vec::new(); + }; + let mut out = Vec::new(); + for (dep, supported) in &dep_targets.per_dep { + let not_supported = dep_targets + .per_dep_not_supported + .get(dep) + .map(|v| v.as_slice()) + .unwrap_or(&[]); + if supported.is_empty() && not_supported.is_empty() { + continue; + } + let sup_families = dedup_families(supported); + let ns_families = dedup_families(not_supported); + let hit_not_supported = + not_supported.iter().any(|p| p.regex.is_match(part)); + if hit_not_supported { + out.push(TargetMismatch { + dep: dep.clone(), + target_part: part.to_string(), + supported_families: sup_families, + not_supported_families: ns_families, + kind: TargetMismatchKind::NotSupported, + }); + continue; + } + let hit_supported = supported.iter().any(|p| p.regex.is_match(part)); + if hit_supported { + continue; } + out.push(TargetMismatch { + dep: dep.clone(), + target_part: part.to_string(), + supported_families: sup_families, + not_supported_families: ns_families, + kind: TargetMismatchKind::Unblessed, + }); } + out } -// ============================================================================ -// Configuration Structures -// ============================================================================ +fn dedup_families(patterns: &[TargetPattern]) -> Vec { + let mut families: Vec = + patterns.iter().map(|p| p.family.clone()).collect(); + families.sort(); + families.dedup(); + families +} -#[derive(Debug, Deserialize, Serialize)] -pub struct WorkspaceConfig { - #[allow(dead_code)] - pub workspace: WorkspaceInfo, - pub dependencies: HashMap, - #[serde(default)] - pub tools: Option, +pub fn parse_target_pattern( + raw: &str, +) -> std::result::Result { + let Some((family, rest)) = raw.split_once('{') else { + return Err(TargetParseError::MissingBraces { + raw: raw.to_string(), + }); + }; + let Some(regex_src) = rest.strip_suffix('}') else { + return Err(TargetParseError::MissingBraces { + raw: raw.to_string(), + }); + }; + // Anchor at start: `xcvm3(.*)` should match ONLY parts that + // begin with `xcvm3`, not any string containing `xcvm3` mid- + // way. End anchor is optional because the source patterns + // themselves use `(.*)` to slop up the suffix. + let anchored = format!("^{regex_src}"); + let regex = regex::Regex::new(&anchored).map_err(|e| { + TargetParseError::BadRegex { + raw: raw.to_string(), + source: e, + } + })?; + Ok(TargetPattern { + family: family.to_string(), + regex, + raw: raw.to_string(), + }) } -#[derive(Debug, Deserialize, Serialize)] -pub struct WorkspaceInfo { - #[allow(dead_code)] - pub name: String, - #[allow(dead_code)] - pub version: String, +/// How a workspace dependency identifies its source. Currently a git +/// repo or a local filesystem path; the natural future addition is a +/// registry-resolved variant (`Registry { name, version }`) once a +/// crates.io-like index exists. +/// +/// `#[serde(untagged)]` keeps the `vw.toml` ergonomics that came +/// before: an entry with `repo = "..."` reads as `Git`, an entry with +/// `path = "..."` reads as `Path`. New variants need new +/// non-ambiguous required keys for serde to discriminate cleanly. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum DependencySource { + Git { + repo: String, + #[serde(default)] + branch: Option, + #[serde(default)] + commit: Option, + #[serde(default)] + submodules: bool, + }, + Path { + path: PathBuf, + }, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct Dependency { - pub repo: String, - #[serde(default)] - pub branch: Option, - #[serde(default)] - pub commit: Option, + #[serde(flatten)] + pub source: DependencySource, #[serde(default)] pub src: Vec, #[serde(default)] @@ -216,17 +722,57 @@ pub struct Dependency { #[serde(default)] pub sim_only: bool, #[serde(default)] - pub submodules: bool, - #[serde(default)] pub exclude: Vec, } +impl Dependency { + pub fn is_local(&self) -> bool { + matches!(self.source, DependencySource::Path { .. }) + } + + /// Git-only accessor: the upstream repo URL. + pub fn repo(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { repo, .. } => Some(repo), + DependencySource::Path { .. } => None, + } + } + + pub fn branch(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { branch, .. } => branch.as_deref(), + DependencySource::Path { .. } => None, + } + } + + pub fn commit(&self) -> Option<&str> { + match &self.source { + DependencySource::Git { commit, .. } => commit.as_deref(), + DependencySource::Path { .. } => None, + } + } + + pub fn submodules(&self) -> bool { + match &self.source { + DependencySource::Git { submodules, .. } => *submodules, + DependencySource::Path { .. } => false, + } + } + + pub fn local_path(&self) -> Option<&Path> { + match &self.source { + DependencySource::Path { path } => Some(path.as_path()), + DependencySource::Git { .. } => None, + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct LockFile { pub dependencies: HashMap, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct LockedDependency { pub repo: String, pub commit: String, @@ -243,15 +789,6 @@ pub struct LockedDependency { pub exclude: Vec, } -#[derive(Debug, Serialize, Deserialize)] -pub struct VhdlLsConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub standard: Option, - pub libraries: HashMap, - #[serde(skip_serializing_if = "Option::is_none")] - pub lint: Option>, -} - #[derive(Deserialize, Debug)] struct CargoToml { package: CargoPackage, @@ -262,15 +799,6 @@ struct CargoPackage { name: String, } -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct VhdlLsLibrary { - pub files: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub exclude: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_third_party: Option, -} - // ============================================================================ // Tool Configuration (workspace-wide [tools] section) // ============================================================================ @@ -391,6 +919,48 @@ pub fn get_access_credentials_from_netrc( Ok(None) } +/// Look up netrc credentials for a git-repository URL. Returns +/// `None` when the URL has no parseable host, or the host has no +/// entry in `~/.netrc`. Never errors — a missing / malformed +/// netrc is treated as "no credentials," matching the "unauthenticated +/// clone" path. +/// +/// Callers: `vw-cli::Commands::Update`, `vw-vivado`'s +/// auto-update RPC handler. +pub fn get_access_credentials_for_repo(repo_url: &str) -> Option { + let hostname = extract_hostname_from_repo_url(repo_url).ok()?; + get_access_credentials_from_netrc(&hostname).ok().flatten() +} + +/// Scan the workspace's declared git dependencies for the first +/// one that has netrc credentials. Cheap and pragmatic: one set +/// of credentials feeds the whole `update_workspace_with_token` +/// pass (all deps to the same host share the same login), and +/// most workspaces target a single provider (github, gitea, …) +/// so the first match is usually the right one. +/// +/// `include_test = true` also scans `[test-dependencies]`, +/// matching the same shape [`vhdl_dependency_sources_with_test`] +/// uses. Returns `None` when no git dep has creds — e.g. when +/// every git URL is a public repo. +pub fn get_access_credentials_for_workspace( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Option { + let cfg = load_workspace_config(workspace_dir).ok()?; + for dep in cfg + .dependencies + .values() + .chain(cfg.test_dependencies.values().filter(|_| include_test)) + { + let Some(repo) = dep.repo() else { continue }; + if let Some(creds) = get_access_credentials_for_repo(repo) { + return Some(creds); + } + } + None +} + /// Get access token for a given host from the netrc file. /// /// This function reads the user's .netrc file and looks for credentials @@ -438,7 +1008,11 @@ pub fn extract_hostname_from_repo_url(repo_url: &str) -> Result { // ============================================================================ /// Initialize a new workspace with the given name. -pub fn init_workspace(workspace_dir: &Utf8Path, name: String) -> Result<()> { +pub fn init_workspace( + workspace_dir: &Utf8Path, + name: String, + target_part: Option, +) -> Result<()> { let config_path = workspace_dir.join("vw.toml"); if config_path.exists() { return Err(VwError::Config { @@ -446,16 +1020,59 @@ pub fn init_workspace(workspace_dir: &Utf8Path, name: String) -> Result<()> { }); } + let target_parts = target_part + .map(|part| { + vec![TargetPart { + part, + default: true, + }] + }) + .unwrap_or_default(); + let config = WorkspaceConfig { workspace: WorkspaceInfo { name, version: "0.1.0".to_string(), + target_parts, + variants: Vec::new(), + top: Some("top".to_string()), }, dependencies: HashMap::new(), + test_dependencies: HashMap::new(), + targets: None, tools: None, }; save_workspace_config(workspace_dir, &config)?; + scaffold_top_vhd(workspace_dir)?; + Ok(()) +} + +/// Scaffold a minimal `hdl/top.vhd` alongside a fresh `vw.toml` so +/// `vw run` and the analyzer have something to elaborate against +/// out of the box. +fn scaffold_top_vhd(workspace_dir: &Utf8Path) -> Result<()> { + let hdl_dir = workspace_dir.join("hdl"); + std::fs::create_dir_all(&hdl_dir).map_err(|e| VwError::Config { + message: format!("failed to create {hdl_dir}: {e}"), + })?; + let top_path = hdl_dir.join("top.vhd"); + if top_path.exists() { + return Ok(()); + } + let contents = "\ +library ieee; +use ieee.std_logic_1164.all; + +entity top is + port ( + clk: in std_logic + ); +end top; +"; + std::fs::write(&top_path, contents).map_err(|e| VwError::Config { + message: format!("failed to write {top_path}: {e}"), + })?; Ok(()) } @@ -487,164 +1104,383 @@ pub async fn update_workspace_with_token( workspace_dir: &Utf8Path, credentials: Option, ) -> Result { - let config = load_workspace_config(workspace_dir)?; - let deps_dir = deps_directory()?; + // Resolve and fetch the WHOLE transitive dependency graph. Cargo + // model: the entry's `vw.lock` pins every dep — direct AND + // deps-of-deps — so a transitive import like `src @vivado-cmd` from + // inside `@clk-wizard` resolves without the consumer redeclaring + // it. Missing git deps are downloaded as the graph is built. + let creds = credentials + .as_ref() + .map(|c| (c.username.as_str(), c.password.as_str())); + let graph = build_dependency_graph(workspace_dir, true, creds).await?; let mut lock_file = LockFile { dependencies: HashMap::new(), }; - - let mut vhdl_ls_config = VhdlLsConfig { - standard: None, - libraries: HashMap::new(), - lint: None, - }; - let mut update_info = Vec::new(); - - for (name, dep) in &config.dependencies { - // Use credentials passed from caller - let creds = credentials - .as_ref() - .map(|c| (c.username.as_str(), c.password.as_str())); - - let commit_sha = resolve_dependency_commit( - &dep.repo, - &dep.branch, - &dep.commit, - creds, - ) - .await - .map_err(|e| VwError::Dependency { - message: format!( - "Failed to resolve commit for dependency '{name}': {e}" - ), - })?; - - let dep_path = deps_dir.join(format!("{name}-{commit_sha}")); - - let was_cached = dep_path.exists(); - - if !was_cached { - download_dependency( - &dep.repo, - &commit_sha, - &dep.src, - &dep_path, - dep.recursive, - &dep.exclude, - dep.submodules, - creds, - ) - .await - .map_err(|e| VwError::Dependency { - message: format!("Failed to download dependency '{name}': {e}"), - })?; - } - - update_info.push(DependencyUpdateInfo { - name: name.clone(), - commit: commit_sha.clone(), - was_cached, - }); - - lock_file.dependencies.insert( - name.clone(), - LockedDependency { - repo: dep.repo.clone(), - commit: commit_sha.clone(), - src: dep.src.clone(), - path: PathBuf::from(format!("{name}-{commit_sha}")), - recursive: dep.recursive, - sim_only: dep.sim_only, - submodules: dep.submodules, - exclude: dep.exclude.clone(), - }, - ); - - // Find VHDL files in the cached dependency directory - let vhdl_files = - find_vhdl_files(&dep_path, dep.recursive, &dep.exclude)?; - if !vhdl_files.is_empty() { - let portable_files = - vhdl_files.into_iter().map(make_path_portable).collect(); - vhdl_ls_config.libraries.insert( - name.clone(), - VhdlLsLibrary { - files: portable_files, - exclude: None, - is_third_party: None, - }, - ); + for idx in graph.node_indices() { + let node = &graph[idx]; + let Some(name) = node.name.clone() else { + continue; // the entry workspace itself — not a dependency + }; + match &node.locked { + // Git dep: record its pin in the entry lock. + Some(locked) => { + update_info.push(DependencyUpdateInfo { + name: name.clone(), + commit: locked.commit.clone(), + was_cached: node.was_cached, + }); + lock_file.dependencies.insert(name, locked.clone()); + } + // Path dep: no commit to pin, so no lock entry. + None => { + update_info.push(DependencyUpdateInfo { + name, + commit: "local".into(), + was_cached: true, + }); + } } } write_lock_file(workspace_dir, &lock_file)?; - write_vhdl_ls_config(workspace_dir, &vhdl_ls_config)?; + warn_stale_vhdl_ls_toml(workspace_dir); Ok(UpdateResult { dependencies: update_info, }) } -/// Add a new dependency to the workspace configuration. -#[allow(clippy::too_many_arguments)] -pub async fn add_dependency( - workspace_dir: &Utf8Path, - repo: String, - branch: Option, - commit: Option, - src: Option, +/// One workspace in the dependency graph built by +/// [`build_dependency_graph`]: the entry, a git dep, or a path dep. +#[derive(Debug, Clone)] +struct DepGraphNode { + /// The dependency name this node was reached as; `None` for the + /// entry workspace (which nothing depends on). name: Option, - recursive: bool, - sim_only: bool, -) -> Result<()> { - add_dependency_with_token( - workspace_dir, - repo, - branch, - commit, - src, - name, - recursive, - sim_only, - None, - ) - .await + /// On-disk root — a git dep's cache dir, a path dep's source tree, + /// or the entry's own directory. + root: Utf8PathBuf, + /// Lock entry to record for a git dependency; `None` for the entry + /// and for path deps (no commit to pin). + locked: Option, + /// For a git dep: whether its cache dir was already present rather + /// than freshly downloaded. Always `true` for non-git nodes. + was_cached: bool, } -/// Add a new dependency with optional credentials for private repositories. +/// Build the transitive dependency graph rooted at `entry`, downloading +/// any missing git dependencies into the per-user cache as it walks. /// -/// # Arguments -/// * `workspace_dir` - Path to the workspace directory -/// * `repo` - Git repository URL -/// * `branch` - Optional branch name -/// * `commit` - Optional commit hash -/// * `src` - Optional source path within the repository -/// * `name` - Optional dependency name -/// * `recursive` - Whether to recursively include VHDL files -/// * `sim_only` - Whether this dependency is only for simulation (excluded from deps.tcl) -/// * `credentials` - Optional credentials for authentication -#[allow(clippy::too_many_arguments)] -pub async fn add_dependency_with_token( - workspace_dir: &Utf8Path, - repo: String, - branch: Option, - commit: Option, - src: Option, - name: Option, - recursive: bool, - sim_only: bool, - _credentials: Option, -) -> Result<()> { - let mut config = - load_workspace_config(workspace_dir).unwrap_or_else(|_| { - WorkspaceConfig { - workspace: WorkspaceInfo { +/// petgraph gives cycle-safe traversal: a dev-dependency cycle (e.g. +/// `vw` test-depends on `testlib`, which depends back on `vw`) collapses +/// to a single shared node visited once, so the walk terminates. It +/// also matches the Cargo resolution model the import resolver already +/// assumes — the entry workspace pins the whole graph: the first time a +/// dep name is seen (starting from the entry) fixes its node; a later +/// occurrence of the same name only adds an edge, never a re-fetch. +/// +/// Only the entry's `[test-dependencies]` are followed (when +/// `include_test`); a transitive dep's dev-deps stay private to it, +/// mirroring [`transitive_dep_cache_paths_with_test`]. +async fn build_dependency_graph( + entry: &Utf8Path, + include_test: bool, + credentials: Option<(&str, &str)>, +) -> Result> { + let deps_dir = deps_directory()?; + let mut graph: DiGraph = DiGraph::new(); + // First-seen (entry-wins) node per dep name; also the cycle guard. + let mut node_by_name: HashMap = HashMap::new(); + + let entry_root = entry + .canonicalize_utf8() + .unwrap_or_else(|_| entry.to_path_buf()); + let entry_idx = graph.add_node(DepGraphNode { + name: None, + root: entry_root.clone(), + locked: None, + was_cached: true, + }); + + // Worklist of (parent node, workspace root, is_entry). + let mut queue = vec![(entry_idx, entry_root, true)]; + while let Some((parent, ws, is_entry)) = queue.pop() { + let Ok(config) = load_workspace_config(&ws) else { + continue; + }; + // The entry contributes its dev-deps too; transitive deps only + // propagate their regular `[dependencies]`. + let mut deps: Vec<(String, Dependency)> = + config.dependencies.into_iter().collect(); + if is_entry && include_test { + deps.extend(config.test_dependencies); + } + for (name, dep) in deps { + // Entry-wins: a name already resolved keeps its node — just + // wire the edge so the graph stays complete — and is never + // re-fetched or re-queued (this is also the cycle guard). + if let Some(&existing) = node_by_name.get(&name) { + graph.update_edge(parent, existing, ()); + continue; + } + let node = match &dep.source { + DependencySource::Git { + repo, + branch, + commit, + submodules, + } => { + let sha = resolve_dependency_commit( + repo, + branch, + commit, + credentials, + ) + .await + .map_err(|e| VwError::Dependency { + message: format!( + "Failed to resolve commit for dependency \ + '{name}': {e}" + ), + })?; + let root = deps_dir.join(format!("{name}-{sha}")); + // A dir left by a PARTIAL/failed prior download + // (created but empty) must not count as cached. + let was_cached = root.exists() + && fs::read_dir(&root) + .map(|mut d| d.next().is_some()) + .unwrap_or(false); + if !was_cached { + if root.exists() { + let _ = fs::remove_dir_all(&root); + } + download_dependency( + repo, + &sha, + &dep.src, + &root, + dep.recursive, + &dep.exclude, + *submodules, + credentials, + None, + ) + .await + .map_err(|e| { + VwError::Dependency { + message: format!( + "Failed to download dependency '{name}': {e}" + ), + } + })?; + } + let root = + Utf8PathBuf::from_path_buf(root).map_err(|p| { + VwError::FileSystem { + message: format!( + "dependency cache path is not UTF-8: {}", + p.display() + ), + } + })?; + DepGraphNode { + name: Some(name.clone()), + root, + was_cached, + locked: Some(LockedDependency { + repo: repo.clone(), + commit: sha.clone(), + src: dep.src.clone(), + path: PathBuf::from(format!("{name}-{sha}")), + recursive: dep.recursive, + sim_only: dep.sim_only, + submodules: *submodules, + exclude: dep.exclude.clone(), + }), + } + } + DependencySource::Path { .. } => { + let Some(p) = dep.local_path() else { + continue; + }; + let root = resolve_local_dep_path(&ws, p); + let root = + Utf8PathBuf::from_path_buf(root).map_err(|p| { + VwError::FileSystem { + message: format!( + "path dependency is not UTF-8: {}", + p.display() + ), + } + })?; + DepGraphNode { + name: Some(name.clone()), + root, + locked: None, + was_cached: true, + } + } + }; + let root = node.root.clone(); + // Recurse only into deps that are themselves htcl + // workspaces (a leaf dep is just files). + let recurse = root.join("vw.toml").exists(); + let idx = graph.add_node(node); + node_by_name.insert(name, idx); + graph.add_edge(parent, idx, ()); + if recurse { + queue.push((idx, root, false)); + } + } + } + Ok(graph) +} + +/// Whether every dependency in this workspace's transitive closure is +/// already materialized. Cheap and fully offline — reads only +/// `vw.toml`, `vw.lock`, and the cache dir, never the network — so it +/// can gate `vw check` the way `cargo check` transparently fetches +/// absent deps instead of face-planting on an unresolved `src @dep`. +/// +/// Path deps need no cache, so a workspace with only path deps is +/// always "present". Returns `false` when a declared git dep has no +/// lock entry (added but never `vw update`d) or its cache dir is +/// missing/empty (never fetched, or `vw clear`ed) — the signal that a +/// fetch is needed. Both `[dependencies]` and `[test-dependencies]` are +/// considered, matching what `vw update` materializes. +pub fn dependencies_present(workspace_dir: &Utf8Path) -> bool { + let Ok(config) = load_workspace_config(workspace_dir) else { + // Missing/unreadable vw.toml — nothing we can assert is + // missing; let the check itself surface any real problem. + return true; + }; + // (a) Every DECLARED git dep must have a lock entry. A dep freshly + // added to vw.toml but never `vw update`d won't appear in the + // transitive walk below (which reads the lock), so catch it here. + let declared_git: Vec = config + .dependencies + .iter() + .chain(config.test_dependencies.iter()) + .filter(|(_, d)| matches!(d.source, DependencySource::Git { .. })) + .map(|(n, _)| n.clone()) + .collect(); + if !declared_git.is_empty() { + match load_lock_file(workspace_dir) { + Ok(lock) => { + if declared_git + .iter() + .any(|n| !lock.dependencies.contains_key(n)) + { + return false; + } + } + Err(_) => return false, // git deps declared, no lock at all + } + } + // (b) Every dep in the transitive closure must be materialized on + // disk. `transitive_dep_cache_paths_with_test` walks the whole + // graph (direct + deps-of-deps, via each dep's bundled lock); a + // `vw clear`ed or partially-fetched cache dir counts as missing. + let Ok(paths) = transitive_dep_cache_paths_with_test(workspace_dir, true) + else { + return true; + }; + for (_name, path) in paths { + // Path deps resolve to real source trees (always present); git + // deps resolve into the cache and may be absent or empty. + let present = path.exists() + && fs::read_dir(&path) + .map(|mut d| d.next().is_some()) + .unwrap_or(false); + if !present { + return false; + } + } + true +} + +/// One-time migration nudge on `vw update`: if the workspace still +/// has a `vhdl_ls.toml` sitting at its root, print a warning to +/// stderr. vw no longer writes or reads that file — both the sim +/// path and `vw-analyzer` compute their VHDL config in memory from +/// live workspace state. +fn warn_stale_vhdl_ls_toml(workspace_dir: &Utf8Path) { + let path = workspace_dir.join("vhdl_ls.toml"); + if path.exists() { + eprintln!( + "warning: {path} is no longer consumed by vw; the LSP \ + and sim paths render config in memory. Remove the file \ + or ignore it — any user-added libraries there won't be \ + picked up." + ); + } +} + +/// Add a new dependency to the workspace configuration. +#[allow(clippy::too_many_arguments)] +pub async fn add_dependency( + workspace_dir: &Utf8Path, + repo: String, + branch: Option, + commit: Option, + src: Option, + name: Option, + recursive: bool, + sim_only: bool, +) -> Result<()> { + add_dependency_with_token( + workspace_dir, + repo, + branch, + commit, + src, + name, + recursive, + sim_only, + None, + ) + .await +} + +/// Add a new dependency with optional credentials for private repositories. +/// +/// # Arguments +/// * `workspace_dir` - Path to the workspace directory +/// * `repo` - Git repository URL +/// * `branch` - Optional branch name +/// * `commit` - Optional commit hash +/// * `src` - Optional source path within the repository +/// * `name` - Optional dependency name +/// * `recursive` - Whether to recursively include VHDL files +/// * `sim_only` - Whether this dependency is only for simulation (excluded from deps.tcl) +/// * `credentials` - Optional credentials for authentication +#[allow(clippy::too_many_arguments)] +pub async fn add_dependency_with_token( + workspace_dir: &Utf8Path, + repo: String, + branch: Option, + commit: Option, + src: Option, + name: Option, + recursive: bool, + sim_only: bool, + _credentials: Option, +) -> Result<()> { + let mut config = + load_workspace_config(workspace_dir).unwrap_or_else(|_| { + WorkspaceConfig { + workspace: WorkspaceInfo { name: "workspace".to_string(), version: "0.1.0".to_string(), + target_parts: Vec::new(), + variants: Vec::new(), + top: None, }, dependencies: HashMap::new(), + test_dependencies: HashMap::new(), + targets: None, tools: None, } }); @@ -660,13 +1496,15 @@ pub async fn add_dependency_with_token( let src_paths = vec![src.unwrap_or_else(|| ".".to_string())]; let dependency = Dependency { - repo: repo.clone(), - branch, - commit, + source: DependencySource::Git { + repo: repo.clone(), + branch, + commit, + submodules: false, + }, src: src_paths, recursive, sim_only, - submodules: false, exclude: Vec::new(), }; @@ -721,12 +1559,14 @@ pub fn clear_cache(workspace_dir: &Utf8Path) -> Result> { Ok(cleared) } -/// List all dependencies in the workspace. +/// List all dependencies in the workspace (both regular and +/// test-dependencies). Callers that want to render them in +/// separate sections can filter on [`DependencyInfo::is_test`]. pub fn list_dependencies( workspace_dir: &Utf8Path, ) -> Result> { let config = load_workspace_config(workspace_dir)?; - if config.dependencies.is_empty() { + if config.dependencies.is_empty() && config.test_dependencies.is_empty() { return Ok(Vec::new()); } @@ -734,44 +1574,50 @@ pub fn list_dependencies( let lock_file = load_lock_file(workspace_dir).ok(); let mut deps = Vec::new(); - for (name, dep) in &config.dependencies { - let version_info = match &lock_file { - Some(lock) => { - if let Some(locked_dep) = lock.dependencies.get(name) { - VersionInfo::Locked { - commit: locked_dep.commit.clone(), - } - } else { - // Not yet resolved, show branch/commit from config - match (&dep.branch, &dep.commit) { - (Some(branch), None) => VersionInfo::Branch { - branch: branch.clone(), - }, - (None, Some(commit)) => VersionInfo::Commit { - commit: commit.clone(), - }, - _ => VersionInfo::Unknown, - } - } + for (name, dep, is_test) in config + .dependencies + .iter() + .map(|(n, d)| (n, d, false)) + .chain(config.test_dependencies.iter().map(|(n, d)| (n, d, true))) + { + let (source_label, version_info) = match &dep.source { + DependencySource::Path { path } => { + (path.display().to_string(), VersionInfo::Local) } - None => { - // No lock file, show branch/commit from config - match (&dep.branch, &dep.commit) { - (Some(branch), None) => VersionInfo::Branch { - branch: branch.clone(), - }, - (None, Some(commit)) => VersionInfo::Commit { - commit: commit.clone(), + DependencySource::Git { + repo, + branch, + commit, + .. + } => { + let from_config = + || match (branch.as_deref(), commit.as_deref()) { + (Some(b), None) => { + VersionInfo::Branch { branch: b.into() } + } + (None, Some(c)) => { + VersionInfo::Commit { commit: c.into() } + } + _ => VersionInfo::Unknown, + }; + let version = match &lock_file { + Some(lock) => match lock.dependencies.get(name) { + Some(locked_dep) => VersionInfo::Locked { + commit: locked_dep.commit.clone(), + }, + None => from_config(), }, - _ => VersionInfo::Unknown, - } + None => from_config(), + }; + (repo.clone(), version) } }; deps.push(DependencyInfo { name: name.clone(), - repo: dep.repo.clone(), + source: source_label, version: version_info, + is_test, }); } @@ -781,15 +1627,29 @@ pub fn list_dependencies( #[derive(Debug, Clone)] pub struct DependencyInfo { pub name: String, - pub repo: String, + /// User-facing source description: the repo URL for git deps, + /// the local path for path deps. + pub source: String, pub version: VersionInfo, + /// True when this entry came from `[test-dependencies]` rather + /// than `[dependencies]`. Test-deps only affect `vw test`; other + /// commands see them but don't act on them. + pub is_test: bool, } #[derive(Debug, Clone)] pub enum VersionInfo { - Branch { branch: String }, - Commit { commit: String }, - Locked { commit: String }, + Branch { + branch: String, + }, + Commit { + commit: String, + }, + Locked { + commit: String, + }, + /// Local filesystem dependency — no commit to pin. + Local, Unknown, } @@ -984,1637 +1844,5728 @@ pub struct TestbenchInfo { pub path: PathBuf, } -pub struct RecordProcessor { - pub vhdl_std: VhdlStandard, - pub symbols: HashMap, - pub symbol_to_file: HashMap, - pub tagged_names: HashSet, - pub file_info: HashMap, - pub target_attr: String, +/// Recursively enumerate every `*.htcl` file under +/// `/test/`. Skips hidden directories (`.git`, +/// `.vscode`, etc.) and any directory literally named `target` +/// (the vw-standard build-output location, matches the shape used +/// by `vw::make_wrapper`). Returns file paths sorted +/// lexicographically for deterministic test order. +/// +/// Returns an empty vec when `/test/` doesn't exist +/// — matches `vw test`'s expected "no tests found" UX rather than +/// erroring. +/// One VHDL source shipped by a dep: the target VHDL library name +/// it should compile into, plus the absolute on-disk path. +/// +/// Library name is derived from the dep name with hyphens replaced +/// by underscores — the same rule the `vhdl_ls.toml` generator +/// already uses, matching NVC/Vivado convention. This will move to +/// a dep-controlled override later; for now the rule is uniform. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VhdlDepSource { + pub library: String, + pub path: PathBuf, } -const RECORD_PARSE_ATTRIBUTE: &str = "serialize_rust"; -impl RecordProcessor { - pub fn new(std: VhdlStandard) -> Self { - Self { - vhdl_std: std, - symbols: HashMap::new(), - symbol_to_file: HashMap::new(), - tagged_names: HashSet::new(), - file_info: HashMap::new(), - target_attr: RECORD_PARSE_ATTRIBUTE.to_string(), - } - } +/// Enumerate every VHDL source published by any transitive dep +/// of `workspace_dir`. Files are absolute paths pointing at the +/// dep's materialized cache (or its local path, for +/// `path = "..."` deps). +/// +/// A dep only contributes when it explicitly declares a `src` +/// field in its vw.toml entry. Path deps for htcl-only libraries +/// (e.g. `[dependencies.vw] path = "..."`) omit `src` and +/// therefore publish no VHDL — otherwise a recursive scan would +/// happily pick up the library's OWN `target/`, `test/`, and +/// other non-shipped subtrees. +/// +/// For each `src` entry we honor the dep's `recursive` / +/// `exclude` config, matching what `vw update` uses when it +/// populates the cache from a git dep. For path deps the same +/// filtering runs at read time (no copy step) so both dep kinds +/// present identical surfaces. +/// +/// Depends on the deps being present on disk — call after +/// `vw update`. Missing dep caches are silently skipped so a +/// half-updated workspace still gives a partial result rather +/// than erroring mid-enumeration. +/// +/// Sort order: library name, then path within library. Callers +/// that need topological order do their own downstream analysis. +pub fn vhdl_dependency_sources( + workspace_dir: &Utf8Path, +) -> Result> { + vhdl_dependency_sources_ext(workspace_dir, false, false) } -// ============================================================================ -// File Cache - Reduces redundant file reads during build -// ============================================================================ +/// Detect whether the workspace has any git deps declared in +/// vw.toml but missing from vw.lock — the state where the user +/// hasn't yet run `vw update`, or the lockfile has been +/// truncated / wiped. Cheap check (loads the config + lockfile +/// once, no network); consumers use it to decide whether to +/// auto-invoke [`update_workspace`]. +/// +/// `include_test` mirrors the same flag on the enumeration side +/// so a test-deps-only unlocked entry is caught when the caller +/// intends to enumerate test-deps too. +pub fn workspace_has_unlocked_git_deps( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result { + let cfg = load_workspace_config(workspace_dir)?; + let git_names: Vec<&str> = cfg + .dependencies + .iter() + .chain(cfg.test_dependencies.iter().filter(|_| include_test)) + .filter(|(_, dep)| matches!(dep.source, DependencySource::Git { .. })) + .map(|(name, _)| name.as_str()) + .collect(); + if git_names.is_empty() { + return Ok(false); + } + match load_lock_file(workspace_dir) { + Ok(lock) => Ok(git_names + .iter() + .any(|n| !lock.dependencies.contains_key(*n))), + // No lockfile at all: every git dep is unlocked. + Err(_) => Ok(true), + } +} -/// Cache for parsed file data to avoid redundant parsing during builds. -/// Only caches parsed results, not raw file contents. -pub struct FileCache { - dependencies: HashMap>, - provided_symbols: HashMap>, - entities: HashMap>, +/// Same as [`vhdl_dependency_sources`] but optionally includes +/// the entry workspace's `[test-dependencies]`. Cargo-parity +/// semantic for `dev-dependencies`: test-deps are private to the +/// workspace that declares them. Recursed-into workspaces are +/// walked with `include_test = false` so a dep's own test-deps +/// aren't pulled into your consumer. +/// +/// The test runner uses `include_test = true` so htcl tests +/// under `test/` can enumerate `[test-dependencies]` VHDL +/// alongside regular deps. Production `vw run` uses `false` so +/// test-only VHDL doesn't sneak into a synth flow. +pub fn vhdl_dependency_sources_with_test( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + vhdl_dependency_sources_ext(workspace_dir, include_test, false) } -impl FileCache { - pub fn new() -> Self { - Self { - dependencies: HashMap::new(), - provided_symbols: HashMap::new(), - entities: HashMap::new(), +/// Full-shape enumeration primitive. `exclude_sim_only = true` +/// drops every dep whose vw.toml sets `sim_only = true` (unisim, +/// xpm, etc.) — used by synth flows that need the design-only +/// surface. `include_test` mirrors the same knob on the sibling +/// wrapper. +pub fn vhdl_dependency_sources_ext( + workspace_dir: &Utf8Path, + include_test: bool, + exclude_sim_only: bool, +) -> Result> { + // Walk the entry workspace's deps + transitive deps. We need + // each dep's Dependency config (for src/recursive/exclude), + // so `transitive_dep_cache_paths` (name → path only) isn't + // enough — walk the graph ourselves. + let mut out = Vec::new(); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + let mut queue: Vec<(Utf8PathBuf, bool)> = + vec![(workspace_dir.to_path_buf(), include_test)]; + while let Some((ws, want_test)) = queue.pop() { + if !visited.insert(ws.as_std_path().to_path_buf()) { + continue; } - } - - /// Get cached file dependencies, reading and parsing file if not cached. - pub fn get_dependencies(&mut self, path: &Path) -> Result<&Vec> { - match self.dependencies.entry(path.to_path_buf()) { - Entry::Occupied(e) => Ok(e.into_mut()), - Entry::Vacant(e) => { - let content = fs::read_to_string(path).map_err(|e| { - VwError::FileSystem { - message: format!("Failed to read file {path:?}: {e}"), - } - })?; - let deps = parse_file_dependencies(&content)?; - Ok(e.insert(deps)) + let Ok(cfg) = load_workspace_config(&ws) else { + continue; + }; + // Combine regular + test deps for this level. + let deps: Vec<(String, Dependency)> = cfg + .dependencies + .into_iter() + .chain(cfg.test_dependencies.into_iter().filter(|_| want_test)) + .collect(); + for (name, dep) in deps { + // Skip sim-only deps when the caller wants a + // synth-clean surface. Filter happens BEFORE the + // transitive-workspace push, so if a dep is a + // workspace whose only purpose is sim glue, we + // don't descend and pick up its transitive deps + // either. + if exclude_sim_only && dep.sim_only { + continue; } - } - } - - /// Get cached provided symbols (packages and entities), reading and parsing if not cached. - pub fn get_provided_symbols( - &mut self, - path: &Path, - ) -> Result<&Vec> { - match self.provided_symbols.entry(path.to_path_buf()) { - Entry::Occupied(e) => Ok(e.into_mut()), - Entry::Vacant(e) => { - let content = fs::read_to_string(path).map_err(|e| { - VwError::FileSystem { - message: format!("Failed to read file {path:?}: {e}"), - } - })?; - let symbols = parse_provided_symbols(&content)?; - Ok(e.insert(symbols)) + let Some(dep_path) = + resolve_dep_source_path(workspace_dir, &ws, &name, &dep)? + else { + continue; + }; + // If the dep is itself a workspace, follow it too so + // we pick up its own deps' VHDL. Recursed workspaces + // never see their own test-deps — Cargo parity. + if dep_path.join("vw.toml").is_file() { + if let Ok(u) = Utf8PathBuf::from_path_buf(dep_path.clone()) { + queue.push((u, false)); + } } - } - } - - /// Get cached entities in file, reading and parsing if not cached. - pub fn get_entities(&mut self, path: &Path) -> Result<&Vec> { - match self.entities.entry(path.to_path_buf()) { - Entry::Occupied(e) => Ok(e.into_mut()), - Entry::Vacant(e) => { - let content = fs::read_to_string(path).map_err(|e| { - VwError::FileSystem { - message: format!("Failed to read file {path:?}: {e}"), - } - })?; - let entities = parse_entities(&content)?; - Ok(e.insert(entities)) + let files = enumerate_dep_vhdl_files(&dep_path, &dep)?; + if files.is_empty() { + continue; + } + let library = library_name_for_dep(&name); + for path in files { + out.push(VhdlDepSource { + library: library.clone(), + path, + }); } } } - - /// Get mutable access to the entities cache for functions that only need entity lookups. - pub fn entities_cache_mut(&mut self) -> &mut HashMap> { - &mut self.entities - } + out.sort_by(|a, b| a.library.cmp(&b.library).then(a.path.cmp(&b.path))); + Ok(out) } -impl Default for FileCache { - fn default() -> Self { - Self::new() +/// Resolve one dep's on-disk root. Local (`path = "..."`) deps +/// point at the user's tree; git deps resolve through the +/// workspace's lockfile. Returns `Ok(None)` when the dep is a +/// git dep the caller hasn't `vw update`-d yet — the enumeration +/// treats that as "no VHDL" rather than erroring, since a half- +/// updated workspace shouldn't gate every downstream call. +fn resolve_dep_source_path( + entry_workspace_dir: &Utf8Path, + parent_workspace_dir: &Utf8Path, + name: &str, + dep: &Dependency, +) -> Result> { + if let Some(p) = dep.local_path() { + // Relative path deps resolve against the workspace that + // DECLARES them (same rule Cargo uses). Absolute paths + // pass through unchanged. This lets a workspace ship a + // portable path-dep like `path = "test/fixtures/foo"` + // without hard-coding a machine-specific prefix. + if p.is_absolute() { + return Ok(Some(p.to_path_buf())); + } + return Ok(Some(parent_workspace_dir.as_std_path().join(p))); } + // Git dep — look up the resolved cache path in the lockfile + // of the ENTRY workspace (only the entry has a meaningful + // lockfile; transitive walks reuse the entry's pins for + // Cargo-parity). + let Ok(lock) = load_lock_file(entry_workspace_dir) else { + return Ok(None); + }; + let Some(locked) = lock.dependencies.get(name) else { + return Ok(None); + }; + Ok(Some(resolve_dep_path(&locked.path)?)) } -/// Parse dependencies from file content (extracted for use by FileCache). -fn parse_file_dependencies(content: &str) -> Result> { - let mut dependencies = Vec::new(); - let mut seen = HashSet::new(); - - // Package imports from "use work.package_name" - let imports = get_package_imports(content)?; - for pkg in imports { - let key = format!("pkg:{}", pkg.to_lowercase()); - if seen.insert(key) { - dependencies.push(VwSymbol::new(None, &pkg, SymbolKind::Package)); - } +/// Enumerate every VHDL file a dep publishes, honoring its +/// declared `src` / `recursive` / `exclude` filters. Empty when +/// `dep.src` is empty (htcl-only dep, publishes no VHDL). +/// +/// Two dep kinds diverge here: +/// - **Git deps** cache into `~/.vw/deps/-/` with the +/// `src` prefix STRIPPED at copy time. `copy_vhdl_files_glob` +/// flattens away the source repo's directory structure. So +/// applying `src` here as a subdirectory path finds nothing — +/// we just walk the whole cache dir recursively (its contents +/// were already filtered by the update step). +/// - **Path deps** point at an unmodified checkout of the dep's +/// tree, so `src` still maps to a real subdirectory. +fn enumerate_dep_vhdl_files( + dep_root: &Path, + dep: &Dependency, +) -> Result> { + if dep.src.is_empty() { + return Ok(Vec::new()); } - - // Find direct entity instantiations (instance_name: entity work.entity_name) - let entity_inst_pattern = r"(?i)\w+\s*:\s*entity\s+work\.(\w+)"; - let entity_inst_re = regex::Regex::new(entity_inst_pattern)?; - - for captures in entity_inst_re.captures_iter(content) { - if let Some(entity_name) = captures.get(1) { - let name = entity_name.as_str().to_string(); - let key = format!("ent:{}", name.to_lowercase()); - if seen.insert(key) { - dependencies.push(VwSymbol::new( - None, - &name, - SymbolKind::Entity, - )); - } + // Git-dep cache: flattened at copy time — walk everything and + // apply the exclude patterns (which are structure-relative, so + // they still work as-is over the flat tree). + if !dep.is_local() { + let mut files = + find_vhdl_files(dep_root, /*recursive=*/ true, &[])?; + if !dep.exclude.is_empty() { + let exclude_patterns: Vec = dep + .exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + files.retain(|f| { + let rel = f.strip_prefix(dep_root).unwrap_or(f); + let rel_str = rel.to_string_lossy(); + !exclude_patterns.iter().any(|p| p.matches(&rel_str)) + }); } + files.sort(); + files.dedup(); + return Ok(files); } - - // Find component declarations - let comp_decl_pattern = r"(?i)component\s+(\w+)"; - let comp_decl_re = regex::Regex::new(comp_decl_pattern)?; - - for captures in comp_decl_re.captures_iter(content) { - if let Some(comp_name) = captures.get(1) { - let name = comp_name.as_str().to_string(); - let key = format!("ent:{}", name.to_lowercase()); - if seen.insert(key) { - dependencies.push(VwSymbol::new( - None, - &name, - SymbolKind::Entity, - )); + // Path dep: honor src patterns against the real tree. + let exclude_patterns: Vec = dep + .exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + let mut out = Vec::new(); + for src_pattern in &dep.src { + let src_path = dep_root.join(src_pattern); + let candidates = if src_path.is_dir() { + let base = + src_path.to_str().ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in dep src path".to_string(), + })?; + let mut cands = Vec::new(); + let patterns = if dep.recursive { + vec![format!("{base}/**/*.vhd"), format!("{base}/**/*.vhdl")] + } else { + vec![format!("{base}/*.vhd"), format!("{base}/*.vhdl")] + }; + for p in patterns { + let entries = + glob::glob(&p).map_err(|e| VwError::FileSystem { + message: format!("Invalid glob pattern '{p}': {e}"), + })?; + for entry in entries.flatten() { + cands.push((src_path.clone(), entry)); + } + } + cands + } else if src_path.is_file() { + vec![( + src_path + .parent() + .ok_or_else(|| VwError::FileSystem { + message: "dep src file has no parent".to_string(), + })? + .to_path_buf(), + src_path.clone(), + )] + } else { + // Glob pattern rooted at the dep root — exclude + // patterns match relative to the dep root here. + let base = dep_root.to_path_buf(); + let pat = src_path + .to_str() + .ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in dep src glob".to_string(), + })? + .to_string(); + let entries = + glob::glob(&pat).map_err(|e| VwError::FileSystem { + message: format!("Invalid glob pattern '{pat}': {e}"), + })?; + entries.flatten().map(|e| (base.clone(), e)).collect() + }; + for (strip_prefix, path) in candidates { + if !path.is_file() { + continue; + } + let ext = path.extension().and_then(|e| e.to_str()); + if ext != Some("vhd") && ext != Some("vhdl") { + continue; + } + if !exclude_patterns.is_empty() { + let rel = path.strip_prefix(&strip_prefix).unwrap_or(&path); + let rel_str = rel.to_string_lossy(); + if exclude_patterns.iter().any(|p| p.matches(&rel_str)) { + continue; + } } + out.push(path); } } + out.sort(); + out.dedup(); + Ok(out) +} - Ok(dependencies) +/// Enumerate every VHDL source under `/hdl/` +/// (recursively). These are the workspace's own design sources, +/// as distinct from IP wrappers (which live under `target/ip/` and +/// are enumerated by a separate helper) and testbenches (which +/// live under `bench/`). +/// +/// Returns an empty vec when `/hdl/` doesn't exist +/// — a freshly-scaffolded workspace hasn't checked anything in +/// yet, and that's not an error. +pub fn vhdl_design_sources(workspace_dir: &Utf8Path) -> Result> { + vhdl_design_sources_for_variant(workspace_dir, None) } -/// Parse provided symbols (packages and entities) from file content. -fn parse_provided_symbols(content: &str) -> Result> { - let mut symbols = Vec::new(); +/// Enumerate every VHDL source under `/hdl/**` and +/// filter by the `exclusive` file lists on the workspace's +/// variants: +/// +/// - A file is "variant-owned" if it matches any `exclusive` +/// glob on ANY variant (across the whole list). +/// - Variant-owned files contribute ONLY when their owning +/// variant is the active one. +/// - Files not in any `exclusive` set are shared and always +/// contribute. +/// +/// `active_variant` is the CURRENTLY active variant's name; when +/// `None`, no variant is active (used by the analyzer's default +/// path and by tools that don't yet flow a variant selection +/// through). Under `None`, variant-owned files are still +/// excluded — otherwise a variant-mode workspace would drag +/// every other variant's exclusive sources into the surface. +/// +/// Empty vec when `/hdl/` doesn't exist. +pub fn vhdl_design_sources_for_variant( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result> { + let hdl_dir = workspace_dir.join("hdl"); + if !hdl_dir.exists() { + return Ok(Vec::new()); + } + let mut files = + find_vhdl_files(hdl_dir.as_std_path(), /*recursive=*/ true, &[])?; + files.sort(); + // Compile each variant's `exclusive` globs relative to the + // workspace root. Empty variants list → nothing to filter, + // early-return keeps the common no-variants path cheap. + let cfg = match load_workspace_config(workspace_dir) { + Ok(c) => c, + // No workspace config → can't know about variants, skip filter. + Err(_) => return Ok(files), + }; + if cfg.workspace.variants.is_empty() { + return Ok(files); + } + let owner = build_variant_ownership(workspace_dir, &cfg.workspace)?; + files.retain(|path| { + match owner.owner_of(path) { + // Shared file — always keep. + None => true, + // Variant-owned file — keep iff active variant owns it. + Some(name) => Some(name) == active_variant, + } + }); + Ok(files) +} - // Find package declarations - let package_pattern = r"(?i)\bpackage\s+(\w+)\s+is\b"; - let package_re = regex::Regex::new(package_pattern)?; +/// Precomputed variant-ownership index. Each entry maps a +/// canonicalized absolute path to the variant that "owns" it +/// (the first variant whose `exclusive` glob matched during the +/// build). Files not present in the map are shared. +struct VariantOwnership { + /// Absolute path → owning variant name. + owners: std::collections::HashMap, +} - for captures in package_re.captures_iter(content) { - if let Some(package_name) = captures.get(1) { - symbols.push(VwSymbol::new( - None, - package_name.as_str(), - SymbolKind::Package, - )); - } +impl VariantOwnership { + fn owner_of(&self, path: &Path) -> Option<&str> { + self.owners.get(path).map(|s| s.as_str()) } +} - // Find entity declarations - let entity_pattern = r"(?i)\bentity\s+(\w+)\s+is\b"; - let entity_re = regex::Regex::new(entity_pattern)?; - - for captures in entity_re.captures_iter(content) { - if let Some(entity_name) = captures.get(1) { - symbols.push(VwSymbol::new( - None, - entity_name.as_str(), - SymbolKind::Entity, - )); +fn build_variant_ownership( + workspace_dir: &Utf8Path, + ws: &WorkspaceInfo, +) -> Result { + let mut owners: std::collections::HashMap = + std::collections::HashMap::new(); + for variant in &ws.variants { + for pattern in &variant.exclusive { + // Absolutize relative-to-workspace patterns so the + // glob crate walks the right filesystem tree. + let abs_pattern = workspace_dir.as_std_path().join(pattern); + let pattern_str = + abs_pattern.to_str().ok_or_else(|| VwError::FileSystem { + message: format!( + "variant `{}` exclusive pattern is not valid UTF-8: {}", + variant.name, + abs_pattern.display(), + ), + })?; + let entries = + glob::glob(pattern_str).map_err(|e| VwError::FileSystem { + message: format!( + "variant `{}` invalid glob `{pattern}`: {e}", + variant.name, + ), + })?; + for entry in entries.flatten() { + if !entry.is_file() { + continue; + } + // First-writer wins: if two variants claim the + // same file exclusively, the first entry in the + // list owns it. That's a config bug the user + // should fix; we don't error to keep the surface + // predictable in the interim. + owners.entry(entry).or_insert_with(|| variant.name.clone()); + } } } - - Ok(symbols) + Ok(VariantOwnership { owners }) } -/// Parse entity declarations from file content. -fn parse_entities(content: &str) -> Result> { - let mut entities = Vec::new(); - - let entity_pattern = r"(?i)\bentity\s+(\w+)\s+is\b"; - let re = regex::Regex::new(entity_pattern)?; +/// Enumerate every Vivado design-constraint file under +/// `/constraints/**/*.{xdc,sdc}`. Handles both +/// physical (`.xdc`) and Synopsys-style (`.sdc`) constraints — +/// both are accepted by `read_xdc` in Vivado. +/// +/// Returned separately from [`vhdl_design_sources`] because +/// constraints have their own file kind and a different +/// consumption command (`read_xdc` vs. `read_vhdl`). +/// +/// Empty vec when `constraints/` doesn't exist yet. +pub fn design_constraints(workspace_dir: &Utf8Path) -> Result> { + design_constraints_in(workspace_dir, None) +} - for captures in re.captures_iter(content) { - if let Some(entity_name) = captures.get(1) { - entities.push(entity_name.as_str().to_string()); - } - } +/// Enumerate only the `synth`-scoped constraints — +/// `/constraints/synth/**/*.{xdc,sdc}`. Used to +/// hand synthesis-only constraints to `read_xdc -used_in +/// synthesis` (or the equivalent set_property USED_IN) so +/// route/place-only constraints don't spuriously apply during +/// synth. Empty vec when the subdir doesn't exist. +pub fn design_synth_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("synth")) +} - Ok(entities) +/// Enumerate only the `place`-scoped constraints under +/// `/constraints/place/**/*.{xdc,sdc}`. Mirrors +/// [`design_synth_constraints`] for the placement flow. Empty +/// vec when the subdir doesn't exist. +pub fn design_place_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("place")) } -pub async fn analyze_ext_libraries( - vhdl_ls_config: &VhdlLsConfig, - processor: &mut RecordProcessor, - vhdl_std: VhdlStandard, - cache: &mut FileCache, -) -> Result<()> { - // Collect non-defaultlib library names - let ext_lib_names: Vec = vhdl_ls_config - .libraries - .keys() - .filter(|k| k.as_str() != "defaultlib") - .cloned() - .collect(); +/// Enumerate only the `route`-scoped constraints under +/// `/constraints/route/**/*.{xdc,sdc}`. Mirrors +/// [`design_synth_constraints`] for the routing flow. Empty +/// vec when the subdir doesn't exist. +pub fn design_route_constraints( + workspace_dir: &Utf8Path, +) -> Result> { + design_constraints_in(workspace_dir, Some("route")) +} - // Build inter-library dependency graph by scanning for `library ;` - let ext_lib_set: HashSet = ext_lib_names.iter().cloned().collect(); - let mut lib_deps: HashMap> = HashMap::new(); - for lib_name in &ext_lib_names { - let mut deps = Vec::new(); - if let Some(library) = vhdl_ls_config.libraries.get(lib_name) { - for file_path in &library.files { - let expanded = if file_path.starts_with("$HOME") { - if let Some(home) = dirs::home_dir() { - home.join( - file_path - .strip_prefix("$HOME/") - .unwrap_or(file_path), - ) - } else { - PathBuf::from(file_path) - } - } else { - PathBuf::from(file_path) - }; - if let Ok(contents) = fs::read_to_string(&expanded) { - for line in contents.lines() { - let trimmed = line.trim().to_lowercase(); - if let Some(rest) = trimmed.strip_prefix("library ") { - let dep_lib = rest.trim_end_matches(';').trim(); - if ext_lib_set.contains(dep_lib) - && dep_lib != lib_name.to_lowercase() - { - deps.push(dep_lib.to_string()); - } - } - } - } - } - } - lib_deps.insert(lib_name.clone(), deps); +/// Enumeration primitive shared by [`design_constraints`] and +/// the phase-scoped variants. `subdir = None` walks +/// `/constraints/`; `subdir = Some(name)` walks +/// `/constraints//`. +fn design_constraints_in( + workspace_dir: &Utf8Path, + subdir: Option<&str>, +) -> Result> { + let mut dir = workspace_dir.join("constraints"); + if let Some(sub) = subdir { + dir = dir.join(sub); + } + if !dir.exists() { + return Ok(Vec::new()); } + let mut files = Vec::new(); + find_constraint_files_impl( + dir.as_std_path(), + &mut files, + /*recursive=*/ true, + )?; + files.sort(); + Ok(files) +} - // Topological sort of library names (Kahn's algorithm) - let mut in_degree: HashMap = - ext_lib_names.iter().map(|n| (n.clone(), 0)).collect(); - let mut adj: HashMap> = ext_lib_names - .iter() - .map(|n| (n.clone(), Vec::new())) - .collect(); - for (lib, deps) in &lib_deps { - for dep in deps { - if let Some(neighbors) = adj.get_mut(dep) { - neighbors.push(lib.clone()); - } - if let Some(deg) = in_degree.get_mut(lib) { - *deg += 1; +/// Mirror of `find_vhdl_files_impl` but for the `.xdc` / `.sdc` +/// extension set. Kept separate rather than parameterizing the +/// existing walker because the extension list is small and +/// domain-specific — a generic "find by extensions" helper would +/// obscure the intent at the call site. +fn find_constraint_files_impl( + dir: &Path, + files: &mut Vec, + recursive: bool, +) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory: {e}"), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + if path.is_dir() { + if recursive { + find_constraint_files_impl(&path, files, recursive)?; } - } - } - let mut queue: VecDeque = in_degree - .iter() - .filter(|(_, &d)| d == 0) - .map(|(n, _)| n.clone()) - .collect(); - let mut sorted_libs = Vec::new(); - while let Some(current) = queue.pop_front() { - sorted_libs.push(current.clone()); - if let Some(neighbors) = adj.get(¤t) { - for neighbor in neighbors { - if let Some(deg) = in_degree.get_mut(neighbor) { - *deg -= 1; - if *deg == 0 { - queue.push_back(neighbor.clone()); - } - } + } else if let Some(ext) = path.extension().and_then(|s| s.to_str()) { + if ext == "xdc" || ext == "sdc" { + files.push(path); } } } - // Fall back to unsorted if cycle detected - if sorted_libs.len() != ext_lib_names.len() { - sorted_libs = ext_lib_names; - } + Ok(()) +} - // Analyze libraries in dependency order - for lib_name in &sorted_libs { - if let Some(library) = vhdl_ls_config.libraries.get(lib_name) { - // Convert library name to be NVC-compatible (no hyphens) - let nvc_lib_name = lib_name.replace('-', "_"); +/// Enumerate every generated IP wrapper under +/// `/target/ip/**/*.{vhd,vhdl}`. Populated by +/// `vw::make_wrapper` — see `~/src/htcl/vw/module.htcl` — which +/// drops one `wrapper.vhd` per IP into `target/ip//`. +/// +/// Returned separately from [`vhdl_design_sources`] because IP +/// wrappers have a different lifecycle: they're TOOL-generated +/// (regen on IP config change), not human-authored, and typically +/// compile into their own VHDL library (`ip` by convention). The +/// caller decides the library assignment. +/// +/// **Excludes**: +/// - `target/ip/bd/**` and `target/ip/xci/**` — legacy caches +/// from the old custom BD/XCI save-restore path (retained as +/// an exclusion for now to survive workspaces that still have +/// those dirs on disk; the caches themselves are deleted at +/// session start — see the migration cleanup in `vw run` / +/// `vw repl`). +/// - `target/vw-project/**` — defensive. The on-disk Vivado +/// project (see `vw_project_dir`) sits SIBLING to `target/ip/`, +/// so this walker's root at `target/ip/` shouldn't ever +/// traverse it — but keep the filter as an invariant guard +/// for future walker refactors and to document intent. +/// +/// The excluded content lands in the Vivado project via +/// `read_bd`/`read_ip`/`synth_ip` — re-adding through +/// `read_vhdl` would trigger `[filemgmt 20-1440] already exists +/// in the project as a part of sub-design file` CRITICAL WARNINGs. +/// +/// Empty vec when `target/ip/` doesn't exist yet — a fresh +/// workspace hasn't run `vw::make_wrapper` for anything. +pub fn vhdl_ip_sources(workspace_dir: &Utf8Path) -> Result> { + let ip_dir = workspace_dir.join("target").join("ip"); + if !ip_dir.exists() { + return Ok(Vec::new()); + } + let bd_cache = ip_dir.join("bd"); + let xci_cache = ip_dir.join("xci"); + let vw_project = workspace_dir.join("target").join("vw-project"); + let mut files = + find_vhdl_files(ip_dir.as_std_path(), /*recursive=*/ true, &[])?; + // `starts_with` on each canonical prefix filters every + // nested path (`bd/cips/synth/cips.vhd`, `xci/primary_clock/ + // primary_clock.vhd`, and defensively any hypothetical + // `vw-project/...` VHDL that a future walker refactor might + // reach). + files.retain(|p| { + !p.starts_with(bd_cache.as_std_path()) + && !p.starts_with(xci_cache.as_std_path()) + && !p.starts_with(vw_project.as_std_path()) + }); + files.sort(); + Ok(files) +} - let mut files = Vec::new(); - for file_path in &library.files { - // Convert $HOME paths to absolute paths - let expanded_path = if file_path.starts_with("$HOME") { - let home_dir = dirs::home_dir().ok_or_else(|| { - VwError::FileSystem { - message: "Could not determine home directory" - .to_string(), - } - })?; - home_dir.join( - file_path.strip_prefix("$HOME/").unwrap_or(file_path), - ) - } else { - PathBuf::from(file_path) - }; - files.push(expanded_path); - } +/// Enumerate every VHDL source under `/bench/` — the +/// testbenches plus any shared bench code — skipping the `bench/target/` +/// build tree. These join `defaultlib` so the LSP can resolve an opened +/// testbench against the design it exercises (otherwise its `work.*` +/// instantiations are all undefined), and so `run_testbench` sees shared +/// bench code. +pub fn vhdl_bench_sources(workspace_dir: &Utf8Path) -> Result> { + let bench_dir = workspace_dir.join("bench"); + if !bench_dir.exists() { + return Ok(Vec::new()); + } + let target = bench_dir.join("target"); + let mut files = find_vhdl_files( + bench_dir.as_std_path(), + /*recursive=*/ true, + &[], + )?; + files.retain(|p| !p.starts_with(target.as_std_path())); + files.sort(); + Ok(files) +} - // Sort files in dependency order (dependencies first) - sort_files_by_dependencies(processor, &mut files, cache)?; +/// Derive the VHDL library name a dep's sources compile into. +/// Hyphens become underscores (Vivado's `xelab` and NVC both +/// reject library names containing hyphens). Same rule +/// `vhdl_ls.toml` generation uses so the analyzer and the +/// synthesizer see identical library assignments. +fn library_name_for_dep(name: &str) -> String { + name.replace('-', "_") +} - let file_strings: Vec = files - .iter() - .map(|p| p.to_string_lossy().to_string()) - .collect(); +pub fn list_htcl_tests(workspace_dir: &Utf8Path) -> Result> { + let test_dir = workspace_dir.join("test"); + if !test_dir.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + walk_htcl_tests(test_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} - run_nvc_analysis( - vhdl_std, - BUILD_DIR, - &nvc_lib_name, - &file_strings, - false, - ) - .await?; +fn walk_htcl_tests(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!( + "Failed to read test directory {}: {e}", + dir.display() + ), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with('.') || name_str == "target" { + continue; + } + if path.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + out.push(path); + } + } else if path.is_dir() { + walk_htcl_tests(&path, out)?; } } - Ok(()) } -/// Run a testbench using NVC simulator. -pub async fn run_testbench( +/// Enumerate every `.htcl` file under `` recursively, +/// excluding hidden dirs (`.git`, `.vw`), the build-artifact `target/` +/// dir, and vendored deps (`~/.vw/deps/` isn't under a workspace but +/// callers should never point us there anyway). +/// +/// Broader than [`list_htcl_tests`] (which is `test/**/*.htcl` only) +/// — this walks the whole workspace so `synth_needs_update` can +/// invalidate a checkpoint when ANY authored htcl changes (the +/// entry-file `design.htcl`, per-IP `ip/*.htcl`, whatever the user +/// writes). Sorted for determinism. +pub fn list_workspace_htcl_files( workspace_dir: &Utf8Path, - testbench_name: String, - vhdl_std: VhdlStandard, - recurse: bool, - runtime_flags: &[String], - build_rust: bool, - scaffold: bool, -) -> Result<()> { - // Check for mixed-signal test (mist.toml in bench//) - let bench_test_dir = workspace_dir.join("bench").join(&testbench_name); - let mist_toml = bench_test_dir.join("mist.toml"); - if mist_toml.exists() { - let ws_config = load_workspace_config(workspace_dir)?; - let mist_content = - fs::read_to_string(&mist_toml).map_err(|e| VwError::Config { - message: format!("Failed to read mist.toml: {e}"), - })?; - let mist_config: MistConfig = - toml::from_str(&mist_content).map_err(|e| VwError::Config { - message: format!("Failed to parse mist.toml: {e}"), - })?; - if scaffold { - return sim::scaffold( - &bench_test_dir, - &mist_config, - &ws_config.tools, - ); +) -> Result> { + let mut out = Vec::new(); + walk_workspace_htcl(workspace_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} + +fn walk_workspace_htcl(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!( + "Failed to read workspace directory {}: {e}", + dir.display() + ), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + // Skip hidden dirs (.git, .vw, .vscode, etc.) and the + // build-artifact `target/` tree. A user's checked-in + // sources never live in either. + if name_str.starts_with('.') || name_str == "target" { + continue; + } + if path.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("htcl") { + out.push(path); + } + } else if path.is_dir() { + walk_workspace_htcl(&path, out)?; } - return sim::run_analog_test( - workspace_dir, - &testbench_name, - &bench_test_dir, - &mist_config, - &ws_config.tools, - vhdl_std, - ) - .await; } + Ok(()) +} - let vhdl_ls_config = load_existing_vhdl_ls_config(workspace_dir)?; - let mut processor = RecordProcessor::new(vhdl_std); - let mut cache = FileCache::new(); - - fs::create_dir_all(BUILD_DIR)?; - - // First, analyze all non-defaultlib libraries - analyze_ext_libraries( - &vhdl_ls_config, - &mut processor, - vhdl_std, - &mut cache, - ) - .await?; - - // Get defaultlib files for later use - let defaultlib_files = vhdl_ls_config - .libraries - .get("defaultlib") - .map(|lib| lib.files.clone()) - .unwrap_or_default(); - - // Look for the testbench file in bench folder - let bench_dir = workspace_dir.join("bench"); - if !bench_dir.exists() { - return Err(VwError::Testbench { - message: format!("No 'bench' directory found in {workspace_dir}"), - }); +/// Enumerate every tracked source file for the synth checkpoint, +/// in the fixed order fingerprint computation depends on. +/// +/// Tracked source scope — matches what [`vw::synth`] actually +/// reads, plus a few coarse-grained triggers: +/// - VHDL design under `/hdl/**` (variant-filtered). +/// - IP wrappers under `/target/ip/**`. +/// - Synth-scoped XDCs under `/constraints/synth/**`. +/// - Every `.htcl` under `` (excludes hidden dirs + `target/`) +/// — captures edits to `design.htcl`, `ip/*.htcl`, and any +/// local htcl libs. +/// - Every dep-published VHDL file (from +/// `vhdl_dependency_sources_ext(exclude_sim_only=true)`, same +/// surface `vw::synth` feeds `read_vhdl`). Includes git deps +/// (materialized in `~/.vw/deps/-`, content locked +/// by commit) and path deps (files change in place). +/// - `/vw.toml` — variant / target-part / deps-list changes +/// should re-trigger synth. +/// - `/vw.lock` — captures dep-version bumps that alter +/// which cache-dir a name resolves to. +/// +/// Missing files pass through as-is; the fingerprint hasher +/// distinguishes "file present with content X" from "file +/// absent" by only folding present files into the digest and +/// including the sorted list of paths as part of the mix. +fn synth_source_paths( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result> { + let mut sources: Vec = Vec::new(); + sources.extend(vhdl_design_sources_for_variant( + workspace_dir, + active_variant, + )?); + sources.extend(vhdl_ip_sources(workspace_dir)?); + sources.extend(design_synth_constraints(workspace_dir)?); + sources.extend(list_workspace_htcl_files(workspace_dir)?); + // Dependency VHDL — same surface `vw::synth` feeds into + // `read_vhdl`. Matters for path deps (whose files change + // in place, invisible to vw.lock) and belt-and-braces for + // git deps (content is locked by commit, but hashing the + // materialized files means a torn `.vw/deps` extraction + // or a manual edit invalidates too). + if let Ok(dep_sources) = + vhdl_dependency_sources_ext(workspace_dir, false, true) + { + sources.extend(dep_sources.into_iter().map(|s| s.path)); } + sources.push(workspace_dir.join("vw.toml").into_std_path_buf()); + sources.push(workspace_dir.join("vw.lock").into_std_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} - let testbench_file = find_testbench_file( - &testbench_name, - &bench_dir, - recurse, - cache.entities_cache_mut(), - )?; +/// FNV-1a 64-bit hash. Stable across Rust versions (unlike +/// `std::hash::DefaultHasher`, which the language reserves the +/// right to change), fast, and good enough for cache-invalidation +/// use. Not cryptographic — collisions here just mean a false +/// "up-to-date" and a stale checkpoint, but the search space is +/// dozens to hundreds of files. +fn fnv1a_64(bytes: &[u8]) -> u64 { + fnv1a_64_extend(0xcbf2_9ce4_8422_2325, bytes) +} - // Filter defaultlib files to exclude OTHER testbenches but allow common bench code - let bench_dir_abs = workspace_dir.as_std_path().join("bench"); +fn fnv1a_64_extend(mut h: u64, bytes: &[u8]) -> u64 { + for &b in bytes { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} - // Pre-compute entities for bench files to avoid mutable borrow in closure - let mut bench_file_entities: HashMap> = HashMap::new(); - for file_path in &defaultlib_files { - let absolute_path = if file_path.is_relative() { - workspace_dir.as_std_path().join(file_path) - } else { - file_path.clone() - }; - if absolute_path.starts_with(&bench_dir_abs) { - if let Ok(entities) = cache.get_entities(&absolute_path) { - bench_file_entities.insert(absolute_path, entities.clone()); +/// Combined fingerprint over the contents + relative paths of a +/// list of source files. Mtime-independent, so tool-regenerated +/// files (IP wrappers rewritten identically each run) don't +/// spuriously invalidate a fresh checkpoint. Missing files are +/// folded in with a marker byte so a source appearing / +/// disappearing changes the digest. Shared engine for every +/// checkpoint kind — synth, IP configure, and future +/// checkpoints just supply their own path list. +fn fingerprint_paths(workspace_dir: &Utf8Path, paths: &[PathBuf]) -> u64 { + let mut digest: u64 = 0xcbf2_9ce4_8422_2325; + for path in paths { + // Fold the path (relative to workspace root when possible, + // else absolute) so renames register as changes even when + // content is identical. + let rel = path + .strip_prefix(workspace_dir.as_std_path()) + .unwrap_or(path); + digest = fnv1a_64_extend(digest, rel.to_string_lossy().as_bytes()); + match fs::read(path) { + Ok(content) => { + // Marker byte `0x01` for "file present"; then the + // file's own content hash mixed into the running + // digest. Hashing the per-file hash (rather than + // the full content stream) keeps this branch small. + digest = fnv1a_64_extend(digest, &[0x01]); + let file_hash = fnv1a_64(&content); + digest = fnv1a_64_extend(digest, &file_hash.to_le_bytes()); + } + Err(_) => { + // Marker byte `0x00` for "file absent". Consistent + // with the "missing = separate state from empty" + // rule so `vw.lock` present-and-empty ≠ absent. + digest = fnv1a_64_extend(digest, &[0x00]); } } } + digest +} - let filtered_defaultlib_files: Vec = defaultlib_files - .into_iter() - .filter(|file_path| { - // Convert to absolute path for comparison - let absolute_path = if file_path.is_relative() { - workspace_dir.as_std_path().join(file_path) - } else { - file_path.clone() - }; +/// Combined fingerprint over the synth source set. Backs +/// [`synth_needs_update`] and [`write_synth_checkpoint_manifest`]. +pub fn synth_source_fingerprint( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result { + let paths = synth_source_paths(workspace_dir, active_variant)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} - // If it's not in the bench directory, include it - if !absolute_path.starts_with(&bench_dir_abs) { - return true; - } +/// Manifest sidecar path — sits next to the checkpoint under +/// `.manifest`. Small plain-text file (single u64 in +/// decimal). Kept alongside the checkpoint so `rm -rf target/` +/// wipes both together and there's never a stale manifest +/// pointing at a deleted checkpoint. +fn checkpoint_manifest_path(checkpoint: &Path) -> PathBuf { + let mut name = checkpoint.file_name().unwrap_or_default().to_os_string(); + name.push(".manifest"); + checkpoint.with_file_name(name) +} - // If it's in the bench directory, check if it's a different testbench - if let Some(entities) = bench_file_entities.get(&absolute_path) { - // Exclude files that contain testbench entities other than the one we're running - for entity in entities { - if entity.to_lowercase().ends_with("_tb") - && entity != &testbench_name - { - return false; // This is a different testbench, exclude it - } - } - } +/// Write a manifest sidecar recording `fingerprint` next to +/// `checkpoint`. Shared by `write_synth_checkpoint_manifest`, +/// `write_place_checkpoint_manifest`, and `write_project_manifest` +/// so every checkpoint kind uses the same on-disk format. +fn write_checkpoint_manifest_with_fingerprint( + checkpoint: &Path, + fingerprint: u64, +) -> Result<()> { + let manifest = checkpoint_manifest_path(checkpoint); + fs::write(&manifest, format!("{fingerprint}\n")).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to write checkpoint manifest {}: {e}", + manifest.display() + ), + } + }) +} - // Include this file (it's either the current testbench or common bench code) - true - }) - .collect(); +/// Compare `current_fingerprint` against the manifest sidecar +/// next to `checkpoint`. Returns `true` when the checkpoint is +/// missing, the manifest is missing / unparseable, or the +/// fingerprints disagree. Shared by every `*_needs_update` fn. +fn checkpoint_needs_update_with_fingerprint( + checkpoint: &Path, + current_fingerprint: u64, +) -> bool { + if !checkpoint.exists() { + return true; + } + let manifest = checkpoint_manifest_path(checkpoint); + let Ok(stored) = fs::read_to_string(&manifest) else { + return true; + }; + let Ok(stored_fp) = stored.trim().parse::() else { + return true; + }; + stored_fp != current_fingerprint +} - // Find only the defaultlib files that are actually referenced by this testbench - let mut referenced_files = find_referenced_files( - &testbench_file, - &filtered_defaultlib_files, - &mut cache, - )?; +/// Write the manifest sidecar for a freshly-produced synth +/// checkpoint. Records the current source fingerprint so +/// [`synth_needs_update`] can decide freshness by content +/// comparison rather than mtime. +/// +/// Called from `vw::synth` immediately after +/// `vivado_cmd::write_checkpoint` completes. +pub fn write_synth_checkpoint_manifest( + workspace_dir: &Utf8Path, + checkpoint: &Path, + active_variant: Option<&str>, +) -> Result<()> { + let fp = synth_source_fingerprint(workspace_dir, active_variant)?; + write_checkpoint_manifest_with_fingerprint(checkpoint, fp) +} - // Sort files in dependency order (dependencies first) - sort_files_by_dependencies( - &mut processor, - &mut referenced_files, - &mut cache, - )?; +/// Returns `true` when either +/// - the checkpoint file is missing, or +/// - its manifest sidecar is missing / unreadable / stores a +/// fingerprint different from the tracked source set's current +/// fingerprint. +/// +/// Content-hash based (not mtime): identical `make_wrapper` +/// output with a shifted mtime does NOT invalidate the checkpoint, +/// which is the whole point — the wrapper is regenerated on every +/// design.htcl run but its stripped-header body is stable when +/// the source `.bd` hasn't changed. +pub fn synth_needs_update( + workspace_dir: &Utf8Path, + checkpoint: &Path, + active_variant: Option<&str>, +) -> Result { + let current_fp = synth_source_fingerprint(workspace_dir, active_variant)?; + Ok(checkpoint_needs_update_with_fingerprint( + checkpoint, current_fp, + )) +} - let mut files: Vec = referenced_files - .iter() - .map(|s| s.to_string_lossy().to_string()) - .collect(); +// --------------------------------------------------------------------- +// Place checkpoint — a per-workspace cache scoped to the place stage. +// Backs `vw::place` in the htcl vw module. +// +// Source scope is narrower than synth's: place XDCs under +// `/constraints/place/**` PLUS the upstream synth checkpoint +// file itself. The synth DCP acts as a proxy for "everything synth +// depended on" — if any synth-scope source changed, synth re-ran +// and produced a fresh DCP, invalidating the place fingerprint. +// This avoids duplicating the synth source enumeration here. +// --------------------------------------------------------------------- + +/// Path list feeding [`place_source_fingerprint`]. Sorted + +/// deduped. Missing files pass through — the fingerprint's +/// present/absent marker byte covers the "checkpoint doesn't +/// exist yet" case correctly. +fn place_source_paths( + workspace_dir: &Utf8Path, + synth_checkpoint: &Path, +) -> Result> { + let mut sources: Vec = Vec::new(); + sources.extend(design_place_constraints(workspace_dir)?); + sources.push(synth_checkpoint.to_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} - files.push(testbench_file.to_string_lossy().to_string()); +/// Combined fingerprint over the place source set (place XDCs + +/// the upstream synth checkpoint file). Backs +/// [`place_needs_update`] and [`write_place_checkpoint_manifest`]. +pub fn place_source_fingerprint( + workspace_dir: &Utf8Path, + synth_checkpoint: &Path, +) -> Result { + let paths = place_source_paths(workspace_dir, synth_checkpoint)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} - run_nvc_analysis(vhdl_std, BUILD_DIR, "work", &files, false).await?; +/// Write the manifest sidecar for a freshly-produced place +/// checkpoint. Called from `vw::place` after +/// `vivado_cmd::write_checkpoint` completes. +pub fn write_place_checkpoint_manifest( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, + synth_checkpoint: &Path, +) -> Result<()> { + let fp = place_source_fingerprint(workspace_dir, synth_checkpoint)?; + write_checkpoint_manifest_with_fingerprint(place_checkpoint, fp) +} - run_nvc_elab(vhdl_std, BUILD_DIR, "work", &testbench_name, false).await?; +/// Returns `true` when the place checkpoint OR its manifest is +/// missing, OR the fingerprint stored in the manifest differs +/// from the current one. Mirrors [`synth_needs_update`] with a +/// tighter source scope (place XDCs + the synth DCP proxy). +pub fn place_needs_update( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, + synth_checkpoint: &Path, +) -> Result { + let current_fp = place_source_fingerprint(workspace_dir, synth_checkpoint)?; + Ok(checkpoint_needs_update_with_fingerprint( + place_checkpoint, + current_fp, + )) +} - // Build Rust library if requested - let rust_lib_path = if build_rust { - Some( - build_rust_library(&bench_dir, &testbench_file) - .await? - .to_string_lossy() - .to_string(), - ) - } else { - None - }; +// --------------------------------------------------------------------- +// Route checkpoint helpers +// +// Same shape as the place helpers above but one stage down: route +// XDCs (`/constraints/route/**`) PLUS the upstream *place* +// DCP file. The place DCP acts as the "everything place depended +// on" proxy — if place re-ran, its DCP is fresh and the route +// fingerprint invalidates automatically. Same reason place folds +// in the synth DCP. +// --------------------------------------------------------------------- + +/// Path list feeding [`route_source_fingerprint`]. Sorted + deduped. +/// Missing files pass through — the fingerprint's present/absent +/// marker byte covers the "checkpoint doesn't exist yet" case. +fn route_source_paths( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, +) -> Result> { + let mut sources: Vec = Vec::new(); + sources.extend(design_route_constraints(workspace_dir)?); + sources.push(place_checkpoint.to_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} - // Run NVC simulation - run_nvc_sim( - vhdl_std, - BUILD_DIR, - "work", - &testbench_name, - rust_lib_path, - &runtime_flags.to_vec(), - false, - ) - .await?; +/// Combined fingerprint over the route source set (route XDCs + +/// the upstream place checkpoint file). Backs +/// [`route_needs_update`] and [`write_route_checkpoint_manifest`]. +pub fn route_source_fingerprint( + workspace_dir: &Utf8Path, + place_checkpoint: &Path, +) -> Result { + let paths = route_source_paths(workspace_dir, place_checkpoint)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} - Ok(()) +/// Write the manifest sidecar for a freshly-produced route +/// checkpoint. Called from `vw::route` after +/// `vivado_cmd::write_checkpoint` completes. +pub fn write_route_checkpoint_manifest( + workspace_dir: &Utf8Path, + route_checkpoint: &Path, + place_checkpoint: &Path, +) -> Result<()> { + let fp = route_source_fingerprint(workspace_dir, place_checkpoint)?; + write_checkpoint_manifest_with_fingerprint(route_checkpoint, fp) } -pub fn find_referenced_files( - testbench_file: &Path, - available_files: &[PathBuf], - cache: &mut FileCache, -) -> Result> { - let mut referenced_files = Vec::new(); - let mut processed_files = HashSet::new(); - let mut files_to_process = vec![testbench_file.to_path_buf()]; +/// Returns `true` when the route checkpoint OR its manifest is +/// missing, OR the fingerprint stored in the manifest differs +/// from the current one. Mirrors [`place_needs_update`] with the +/// route source scope (route XDCs + the place DCP proxy). +pub fn route_needs_update( + workspace_dir: &Utf8Path, + route_checkpoint: &Path, + place_checkpoint: &Path, +) -> Result { + let current_fp = route_source_fingerprint(workspace_dir, place_checkpoint)?; + Ok(checkpoint_needs_update_with_fingerprint( + route_checkpoint, + current_fp, + )) +} - while let Some(current_file) = files_to_process.pop() { - if processed_files.contains(¤t_file) { - continue; - } - processed_files.insert(current_file.clone()); +// --------------------------------------------------------------------- +// IP configuration checkpoint — a per-workspace cache scoped to the +// entry `ip/module.htcl` (and everything it srcs). Backs +// `vw::configure_ip` in the htcl vw module. +// +// Source scope: every `.htcl` under `/ip/`. This is intentionally +// tighter than the synth scope — `ip::configure` produces BDs / XCI +// IPs / wrappers whose input surface is defined entirely by the ip/ +// tree. If the user's ip/ htcl changes (adding an IP, tweaking +// a configure_* parameter), the checkpoint invalidates. If the +// user's design.htcl changes, it doesn't — that only matters for +// the downstream synth step, which has its own checkpoint. +// --------------------------------------------------------------------- + +/// Enumerate every `.htcl` file under `/ip/`, +/// recursively, sorted. Empty vec when `ip/` doesn't exist yet. +/// Skips hidden dirs (`.git`, `.vw`) and `target/` for the same +/// reason [`list_workspace_htcl_files`] does — those aren't +/// authored sources. +pub fn list_ip_htcl_files(workspace_dir: &Utf8Path) -> Result> { + let ip_dir = workspace_dir.join("ip"); + if !ip_dir.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + walk_workspace_htcl(ip_dir.as_std_path(), &mut out)?; + out.sort(); + Ok(out) +} - // Don't include the testbench file itself in the referenced files - // (it will be added separately) - if current_file != testbench_file { - referenced_files.push(current_file.clone()); - } +// --------------------------------------------------------------------- +// On-disk Vivado project — vw manages a persistent Vivado project at +// `/target/vw-project//` so BD/IP state survives across +// sessions via Vivado's native `save_project`/`open_project` +// machinery (instead of hand-serializing BD .bd files, XCI dirs, +// ipshared/, etc.). See `~/.claude/plans/abundant-petting-lecun.md`. +// +// Fingerprint scope: `/ip/**/*.htcl` + `/vw.toml`. Any +// substantive edit to those invalidates the on-disk project — +// wiped + recreated on the next spawn, then `ip::configure` re-runs +// and `vw::mark_project_configured` writes a fresh manifest. +// --------------------------------------------------------------------- + +/// Absolute path of the on-disk Vivado project directory vw +/// manages. The `.xpr` itself lives at +/// `//.xpr` (Vivado's +/// `create_project -dir` convention — it always nests one level +/// deep under the given dir). +pub fn vw_project_dir(workspace_dir: &Utf8Path) -> Utf8PathBuf { + workspace_dir.join("target").join("vw-project") +} - let dependencies = cache.get_dependencies(¤t_file)?.clone(); +/// Path list feeding [`project_source_fingerprint`]. Sorted + +/// deduped. Every `.htcl` under `/ip/` plus `/vw.toml`. +/// A missing `vw.toml` folds in as the "absent" marker via +/// [`fingerprint_paths`]'s present/absent handling, so an empty +/// or missing workspace still hashes deterministically. +fn project_source_paths(workspace_dir: &Utf8Path) -> Result> { + let mut sources = list_ip_htcl_files(workspace_dir)?; + sources.push(workspace_dir.join("vw.toml").into_std_path_buf()); + sources.sort(); + sources.dedup(); + Ok(sources) +} - // Find corresponding files for each dependency - for dep in dependencies { - for available_file in available_files { - if file_provides_symbol(available_file, &dep, cache)? { - if !processed_files.contains(available_file) { - files_to_process.push(available_file.clone()); - } - break; - } - } - } - } +/// Combined fingerprint over the on-disk-project source set. +/// Backs [`project_needs_wipe`] and [`write_project_manifest`]. +pub fn project_source_fingerprint(workspace_dir: &Utf8Path) -> Result { + let paths = project_source_paths(workspace_dir)?; + Ok(fingerprint_paths(workspace_dir, &paths)) +} - Ok(referenced_files) +/// Absolute path of the `.xpr` inside the on-disk project dir. +/// Vivado's `create_project -dir -name ` produces +/// `//.xpr`. All manifest / checkpoint plumbing uses +/// this path as the "checkpoint" argument to the shared helpers +/// (`write_checkpoint_manifest_with_fingerprint`, +/// `checkpoint_needs_update_with_fingerprint`), which append +/// `.manifest` to derive the sidecar path +/// (`//.xpr.manifest`). +pub fn vw_project_xpr(project_dir: &Path, name: &str) -> PathBuf { + project_dir.join(name).join(format!("{name}.xpr")) } -pub fn sort_files_by_dependencies( - processor: &mut RecordProcessor, - files: &mut Vec, - cache: &mut FileCache, +/// Write the manifest sidecar for a freshly-configured on-disk +/// project. Called from `vw::configure_ip` (via the +/// `mark_project_configured` RPC) after `save_project` completes. +/// +/// Invariant: manifest presence means "the project at this dir +/// was successfully configured with these sources' fingerprint." +/// `project_needs_wipe` relies on this — never write the manifest +/// before `save_project` succeeds. +pub fn write_project_manifest( + workspace_dir: &Utf8Path, + project_dir: &Path, + name: &str, ) -> Result<()> { - // Build dependency graph - let mut dependencies: HashMap> = HashMap::new(); - let mut all_symbols: HashMap = HashMap::new(); - - // First pass: collect all symbols provided by each file - for file in files.iter() { - let symbols = analyze_file(processor, file)?; - for symbol in symbols { - match &symbol.kind { - SymbolKind::Package => { - all_symbols.insert(symbol.name.clone(), file.clone()); - let entry = processor - .file_info - .entry(file.to_string_lossy().to_string()) - .or_default(); - entry.add_defined_pkg(&symbol.name); - - // Use cache to get package imports only - let deps = cache.get_dependencies(file)?; - for dep in deps { - if let SymbolKind::Package = dep.kind { - entry.add_imported_pkg(&dep.name); - } - } - } - SymbolKind::Entity => { - all_symbols.insert(symbol.name, file.clone()); - } - _ => {} - } - } - } + let fp = project_source_fingerprint(workspace_dir)?; + write_checkpoint_manifest_with_fingerprint( + &vw_project_xpr(project_dir, name), + fp, + ) +} - // Second pass: find dependencies for each file - for file in files.iter() { - let deps = cache.get_dependencies(file)?.clone(); - let mut file_deps = Vec::new(); +/// Returns `true` when the on-disk project at `//` +/// should be wiped + recreated: +/// - the `.xpr` is missing, OR +/// - the manifest sidecar is missing / unreadable, OR +/// - the stored fingerprint differs from the current one. +/// +/// Called by `vw run` / `vw repl` before spawning Vivado (see +/// `vw-cli/src/main.rs` / `vw-repl/src/app.rs`); when it returns +/// true, the caller `remove_dir_all(project_dir)` before passing +/// `persist_dir: Some(project_dir)` to `AutoProject`. +pub fn project_needs_wipe( + workspace_dir: &Utf8Path, + project_dir: &Path, + name: &str, +) -> Result { + let current_fp = project_source_fingerprint(workspace_dir)?; + Ok(checkpoint_needs_update_with_fingerprint( + &vw_project_xpr(project_dir, name), + current_fp, + )) +} - for dep in deps { - let dep_name = match &dep.kind { - SymbolKind::Package | SymbolKind::Entity => &dep.name, - _ => continue, - }; - if let Some(provider_file) = all_symbols.get(dep_name) { - if provider_file != file { - file_deps.push(provider_file.clone()); - } - } +/// Idempotent one-shot cleanup of the legacy IP-cache artifacts +/// that lived under `/target/ip/` before the on-disk Vivado +/// project migration: +/// `/target/ip/bd/` +/// `/target/ip/xci/` +/// `/target/ip/.ip-cache` +/// `/target/ip/.ip-cache.manifest` +/// +/// Called by `vw run` / `vw repl` at bootstrap; on first +/// on-disk-mode session it wipes stale bytes and returns the +/// count. On subsequent sessions it's a no-op (returns 0). +/// +/// Deliberately does NOT touch `/target/ip//wrapper.vhd` +/// — those are `vw::make_wrapper` outputs still consumed by +/// `vw::synth`, and their `` sibling dirs may still be +/// populated by the on-disk project's `generate_target` outputs. +pub fn cleanup_legacy_ip_cache(workspace_dir: &Utf8Path) -> usize { + let ip_dir = workspace_dir.join("target").join("ip"); + let targets = [ + ip_dir.join("bd"), + ip_dir.join("xci"), + ip_dir.join(".ip-cache"), + ip_dir.join(".ip-cache.manifest"), + ]; + let mut removed = 0; + for t in targets { + let p = t.as_std_path(); + if !p.exists() { + continue; + } + let ok = if p.is_dir() { + fs::remove_dir_all(p).is_ok() + } else { + fs::remove_file(p).is_ok() + }; + if ok { + removed += 1; } - - dependencies.insert(file.clone(), file_deps); } + removed +} - // Topological sort using Kahn's algorithm - let sorted = topological_sort_files(files.clone(), dependencies)?; - *files = sorted; - - Ok(()) +/// Outcome of [`prepare_vw_project_dir`]. The caller decides how +/// to surface the messages (`tracing::info!`, REPL banner, etc.); +/// vw-lib itself intentionally doesn't do user-facing IO. +#[derive(Debug, Clone)] +pub struct PreparedProjectDir { + /// Absolute path of `/target/vw-project/`, ready to pass + /// as `AutoProject::persist_dir = Some(...)`. + pub project_dir: Utf8PathBuf, + /// Number of legacy `/target/ip/{bd,xci,.ip-cache*}` + /// entries removed by the one-shot Phase 6 migration cleanup. + pub legacy_cache_removed: usize, + /// `Some(path)` iff the persist dir existed AND + /// [`project_needs_wipe`] returned true, so we wiped it before + /// returning. The caller can log this as the reason for the + /// fresh recreate that Vivado is about to do. + /// + /// We wipe `/target/vw-project/` in its entirety (not + /// just the per-`name` subdir): historically bugs have + /// caused Vivado to scatter flat siblings like + /// `/.xpr`, `/.srcs` alongside + /// the nested `//` — a full-dir wipe reliably + /// sweeps any layout drift instead of leaving cross-version + /// clutter behind. + pub wiped_project: Option, } -pub fn load_existing_vhdl_ls_config( +/// Bootstrap the on-disk Vivado project dir for a workspace and +/// return its absolute path, ready to pass as +/// `AutoProject::persist_dir = Some(...)`. +/// +/// Does three things in order (all idempotent): +/// +/// 1. Silently clean up legacy `target/ip/{bd,xci,.ip-cache*}` +/// artifacts from the pre-migration IP cache +/// ([`cleanup_legacy_ip_cache`]). +/// 2. Consult [`project_needs_wipe`]; if the `.xpr` is missing, +/// the manifest sidecar is missing, or the fingerprint is +/// stale, `remove_dir_all(/)` so the worker +/// takes the fresh-create branch instead of `open_project` on +/// stale bytes. +/// 3. `create_dir_all()` so Vivado's later +/// `create_project -dir` has a real parent. +/// +/// Everything worth reporting to the user (legacy-cleanup counts, +/// wipe reason) lands in the returned [`PreparedProjectDir`] for +/// the caller to log. +pub fn prepare_vw_project_dir( workspace_dir: &Utf8Path, -) -> Result { - let config_path = workspace_dir.join("vhdl_ls.toml"); - if config_path.exists() { - let config_content = fs::read_to_string(&config_path).map_err(|e| { - VwError::FileSystem { - message: format!("Failed to read existing vhdl_ls.toml: {e}"), - } - })?; - - let config: VhdlLsConfig = toml::from_str(&config_content)?; - - Ok(config) - } else { - Ok(VhdlLsConfig { - standard: None, - libraries: HashMap::new(), - lint: None, - }) + name: &str, +) -> Result { + let project_dir = vw_project_dir(workspace_dir); + let legacy_cache_removed = cleanup_legacy_ip_cache(workspace_dir); + let mut wiped_project = None; + if project_needs_wipe(workspace_dir, project_dir.as_std_path(), name)? + && project_dir.exists() + { + fs::remove_dir_all(project_dir.as_std_path())?; + wiped_project = Some(project_dir.clone()); } + fs::create_dir_all(project_dir.as_std_path())?; + Ok(PreparedProjectDir { + project_dir, + legacy_cache_removed, + wiped_project, + }) } -// ============================================================================ -// Internal Helper Functions -// ============================================================================ +/// Per-bench output directory under the workspace `target/`, holding a +/// testbench run's artifacts (waveform, Xyce `.prn`, generated plots). +pub fn bench_output_dir(workspace_dir: &Utf8Path, name: &str) -> Utf8PathBuf { + workspace_dir.join("target").join("bench").join(name) +} -fn get_package_imports(content: &str) -> Result> { - // Find 'use work.package_name' statements - let use_work_pattern = r"(?i)use\s+work\.(\w+)"; - let use_work_re = regex::Regex::new(use_work_pattern)?; - let mut imports = Vec::new(); +/// Workspace-relative locations for anodizer artifacts. +const ANODIZER_BUILD_SUBDIR: &str = "target/anodizer/build"; +const ANODIZER_GEN_SUBDIR: &str = "target/anodizer/gen"; +const ANODIZER_FINGERPRINT_FILE: &str = "target/anodizer/.fingerprint"; - for captures in use_work_re.captures_iter(content) { - if let Some(package_name) = captures.get(1) { - imports.push(package_name.as_str().to_string()); - } +/// Generate anodizer Rust structs for the workspace's `serialize_rust`-tagged +/// VHDL records when they are missing or stale, so the testbench Rust build can +/// consume them. +/// +/// Detection is two-stage: (1) skip entirely when no design source carries the +/// `serialize_rust` attribute; (2) otherwise regenerate only when the design +/// sources' content fingerprint differs from the last successful run. The nvc +/// scratch build lands in `target/anodizer/build` and the generated Rust in +/// `target/anodizer/gen`, both under the workspace root. +pub async fn ensure_anodized( + workspace_dir: &Utf8Path, + vhdl_std: VhdlStandard, + active_variant: Option<&str>, +) -> Result<()> { + let config = render_vhdl_ls_config(workspace_dir, active_variant, false)?; + + // Tagged records live in the design sources, i.e. `defaultlib`. + let defaultlib_files = config + .libraries + .get("defaultlib") + .map(|lib| lib.files.clone()) + .unwrap_or_default(); + if defaultlib_files.is_empty() { + return Ok(()); } - Ok(imports) -} -fn file_provides_symbol( - file_path: &Path, - needed: &VwSymbol, - cache: &mut FileCache, -) -> Result { - let provided = cache.get_provided_symbols(file_path)?; - Ok(provided.iter().any(|s| match (&needed.kind, &s.kind) { - // Package dependency matches package declaration - (SymbolKind::Package, SymbolKind::Package) => { - needed.name.eq_ignore_ascii_case(&s.name) - } - // Entity dependency matches entity declaration - (SymbolKind::Entity, SymbolKind::Entity) => { - needed.name.eq_ignore_ascii_case(&s.name) - } - _ => false, - })) -} - -fn analyze_file( - processor: &mut RecordProcessor, - file: &Path, -) -> Result> { - let parser = VHDLParser::new(processor.vhdl_std.into()); - let mut diagnostics = Vec::new(); - let (_, design_file) = parser.parse_design_file(file, &mut diagnostics)?; - - let mut file_finder = VwSymbolFinder::new(&processor.target_attr); - walk_design_file(&mut file_finder, &design_file); - - let file_str = file.to_string_lossy().to_string(); - - // Add symbols to the map - - for symbol in file_finder.get_symbols() { - match symbol.kind { - SymbolKind::Enum(_) - | SymbolKind::Record(_) - | SymbolKind::Constant(_) => { - let name = symbol.get_name().to_string(); - processor.symbols.insert(name.clone(), symbol.clone()); - processor.symbol_to_file.insert(name, file_str.clone()); - } - _ => {} - } + // Stage 1: cheap gate — is anything tagged for serialization at all? + let any_tagged = defaultlib_files.iter().any(|f| { + fs::read_to_string(f) + .map(|c| c.contains("serialize_rust")) + .unwrap_or(false) + }); + if !any_tagged { + return Ok(()); } - for tagged_type in file_finder.get_tagged_types() { - processor.tagged_names.insert(tagged_type.clone()); + // Stage 2: regenerate only when the design sources changed. + let gen_dir = workspace_dir.join(ANODIZER_GEN_SUBDIR); + let generated = gen_dir.join("generated_structs.rs"); + let fingerprint_file = workspace_dir.join(ANODIZER_FINGERPRINT_FILE); + let fingerprint = fingerprint_paths(workspace_dir, &defaultlib_files); + + let stored = fs::read_to_string(&fingerprint_file) + .ok() + .and_then(|s| s.trim().parse::().ok()); + if generated.exists() && stored == Some(fingerprint) { + return Ok(()); } - Ok(file_finder.get_symbols().clone()) -} + let build_dir = workspace_dir.join(ANODIZER_BUILD_SUBDIR); + fs::create_dir_all(&gen_dir)?; + anodizer::anodize(&config, &build_dir, &gen_dir, vhdl_std).await?; -fn topological_sort_files( - files: Vec, - dependencies: HashMap>, -) -> Result> { - let mut dep_graph: DiGraph = DiGraph::default(); - let mut index_map: HashMap = HashMap::new(); + fs::write(&fingerprint_file, fingerprint.to_string())?; + Ok(()) +} - // initialize the nodes - for file in &files { - let index = dep_graph.add_node(file.clone()); - index_map.insert(file.clone(), index); - } +/// Run a testbench using NVC simulator. +#[allow(clippy::too_many_arguments)] +/// True when `dir` holds a Cargo.toml that declares a `[package]` — i.e. a +/// buildable crate (a cosim testbench driver), as opposed to a +/// `[workspace]`-only manifest like the top-level `bench/Cargo.toml`. +fn dir_is_rust_crate(dir: &Path) -> bool { + let Ok(contents) = fs::read_to_string(dir.join("Cargo.toml")) else { + return false; + }; + toml::from_str::(&contents) + .map(|v| v.get("package").is_some()) + .unwrap_or(false) +} - // now add edges from files to their dependencies - for (file, deps) in &dependencies { - let source_node = index_map.get(file).ok_or(VwError::Dependency { - message: format!( - "Index map somehow didn't contain file {:?}", - file - ), - })?; - // file depends on every dep in deps - for dep in deps { - let dst_node = index_map.get(dep).ok_or(VwError::Dependency { +/// Regenerate the cosim bridge scaffold (`Cargo.toml`, `build.rs`, +/// generated sources) for every `bench//mist.toml` in the +/// workspace, up front. +/// +/// `bench/` is a single cargo workspace whose members include each +/// mixed-signal bench crate, so a missing `bench//Cargo.toml` +/// (e.g. after `git clean -fdx`, which wipes the generated scaffold) +/// makes cargo fail to LOAD the workspace manifest — breaking +/// `cargo build` for EVERY bench, not just the cosim ones. Scaffolding +/// them all before any bench builds keeps the workspace valid; the +/// mixed-signal benches themselves don't have to be run (or even +/// discovered) for their crate to need to exist. `write_file` is +/// content-aware, so unchanged scaffolds don't touch the tree. +pub fn ensure_bench_scaffolds(workspace_dir: &Utf8Path) -> Result<()> { + let bench_dir = workspace_dir.join("bench"); + let Ok(entries) = fs::read_dir(bench_dir.as_std_path()) else { + return Ok(()); // no bench dir → nothing to scaffold + }; + let mut ws_config: Option = None; + for entry in entries.flatten() { + let dir = entry.path(); + let mist_toml = dir.join("mist.toml"); + if !dir.is_dir() || !mist_toml.exists() { + continue; + } + let mist_content = + fs::read_to_string(&mist_toml).map_err(|e| VwError::Config { + message: format!("Failed to read {}: {e}", mist_toml.display()), + })?; + let mist_config: MistConfig = + toml::from_str(&mist_content).map_err(|e| VwError::Config { message: format!( - "Index map somehow didn't contain dep {:?}", - dep + "Failed to parse {}: {e}", + mist_toml.display() ), })?; - dep_graph.add_edge(*source_node, *dst_node, ()); + // Load the workspace config lazily and once — only needed when + // there's at least one mixed-signal bench. + if ws_config.is_none() { + ws_config = Some(load_workspace_config(workspace_dir)?); } + let bench_test_dir = + Utf8PathBuf::from_path_buf(dir.clone()).map_err(|p| { + VwError::FileSystem { + message: format!( + "bench path is not UTF-8: {}", + p.display() + ), + } + })?; + sim::scaffold( + &bench_test_dir, + &mist_config, + &ws_config.as_ref().unwrap().tools, + )?; } - - // ok now topological sort - let ordered_files = - toposort(&dep_graph, None).map_err(|_| VwError::Dependency { - message: "Got circular dependency".to_string(), - })?; - - let result: Vec = ordered_files - .iter() - .map(|&idx| dep_graph[idx].clone()) - .rev() - .collect(); - Ok(result) + Ok(()) } -fn find_testbench_file_recurse( - testbench_name: &str, - bench_dir: &Utf8Path, +#[allow(clippy::too_many_arguments)] +pub async fn run_testbench( + workspace_dir: &Utf8Path, + testbench_name: String, + vhdl_std: VhdlStandard, recurse: bool, - entities_cache: &mut HashMap>, -) -> Result> { - let mut found_files = Vec::new(); - - for entry in fs::read_dir(bench_dir).map_err(|e| VwError::FileSystem { - message: format!("Failed to read bench directory: {e}"), - })? { - let entry = entry.map_err(|e| VwError::FileSystem { - message: format!("Failed to read directory entry: {e}"), - })?; - let path = entry.path(); - - if path.is_file() { - if let Some(extension) = path.extension() { - if extension == "vhd" || extension == "vhdl" { - // Check if this file contains the entity we're looking for - if file_contains_entity( - &path, - testbench_name, - entities_cache, - )? { - found_files.push(path); - } - } - } - } else if recurse { - let dir_path: Utf8PathBuf = - path.try_into().map_err(|e| VwError::FileSystem { - message: format!("Failed to get dir path: {e}"), - })?; - let mut lower_testbenches = find_testbench_file_recurse( - testbench_name, - &dir_path, - recurse, - entities_cache, - )?; - found_files.append(&mut lower_testbenches); + runtime_flags: &[String], + build_rust: bool, + scaffold: bool, + build_dir: &str, +) -> Result<()> { + // Check for mixed-signal test (mist.toml in bench//) + let bench_test_dir = workspace_dir.join("bench").join(&testbench_name); + let mist_toml = bench_test_dir.join("mist.toml"); + if mist_toml.exists() { + let ws_config = load_workspace_config(workspace_dir)?; + let mist_content = + fs::read_to_string(&mist_toml).map_err(|e| VwError::Config { + message: format!("Failed to read mist.toml: {e}"), + })?; + let mist_config: MistConfig = + toml::from_str(&mist_content).map_err(|e| VwError::Config { + message: format!("Failed to parse mist.toml: {e}"), + })?; + if scaffold { + return sim::scaffold( + &bench_test_dir, + &mist_config, + &ws_config.tools, + ); } + // Auto-scaffold before simulating so `vw bench` works straight + // from a clean checkout (`git clean -fdx` wipes the generated + // bridge crate — `Cargo.toml`, `build.rs`, generated sources — + // that `run_analog_test`'s `build_bridge_library` needs) without + // a manual `vw bench --scaffold ` pre-step. `scaffold` + // regenerates only the boilerplate (the user-owned `src/lib.rs` + // is left alone) and `write_file` is content-aware, so this is a + // cheap no-op when nothing changed. + sim::scaffold(&bench_test_dir, &mist_config, &ws_config.tools)?; + return sim::run_analog_test( + workspace_dir, + &testbench_name, + &bench_test_dir, + &mist_config, + &ws_config.tools, + vhdl_std, + build_dir, + ) + .await; } - Ok(found_files) -} -fn find_testbench_file( - testbench_name: &str, - bench_dir: &Utf8Path, - recurse: bool, - entities_cache: &mut HashMap>, -) -> Result { - let found_files = find_testbench_file_recurse( - testbench_name, - bench_dir, - recurse, - entities_cache, - )?; + let vhdl_ls_config = render_vhdl_ls_config(workspace_dir, None, false)?; + let mut processor = RecordProcessor::new(vhdl_std); + let mut cache = FileCache::new(); - match found_files.len() { - 0 => Err(VwError::Testbench { - message: format!("Testbench entity '{testbench_name}' not found in bench directory") - }), - 1 => Ok(found_files.into_iter().next().unwrap()), - _ => Err(VwError::Testbench { - message: format!("Multiple files contain entity '{testbench_name}': {found_files:?}") - }), - } -} + fs::create_dir_all(build_dir)?; -fn file_contains_entity( - file_path: &Path, - entity_name: &str, - entities_cache: &mut HashMap>, -) -> Result { - let entities = get_cached_entities(file_path, entities_cache)?; - Ok(entities.iter().any(|e| e.eq_ignore_ascii_case(entity_name))) -} + // First, analyze all non-defaultlib libraries + analyze_ext_libraries( + &vhdl_ls_config, + &mut processor, + vhdl_std, + build_dir, + &mut cache, + ) + .await?; -/// Get entities from cache, parsing and caching if not present. -fn get_cached_entities<'a>( - path: &Path, - entities_cache: &'a mut HashMap>, -) -> Result<&'a Vec> { - match entities_cache.entry(path.to_path_buf()) { - Entry::Occupied(e) => Ok(e.into_mut()), - Entry::Vacant(e) => { - let content = - fs::read_to_string(path).map_err(|e| VwError::FileSystem { - message: format!("Failed to read file {path:?}: {e}"), - })?; - let entities = parse_entities(&content)?; - Ok(e.insert(entities)) - } + // Get defaultlib files for later use + let defaultlib_files = vhdl_ls_config + .libraries + .get("defaultlib") + .map(|lib| lib.files.clone()) + .unwrap_or_default(); + + // Look for the testbench file in bench folder + let bench_dir = workspace_dir.join("bench"); + if !bench_dir.exists() { + return Err(VwError::Testbench { + message: format!("No 'bench' directory found in {workspace_dir}"), + }); } -} -fn make_path_portable(path: PathBuf) -> PathBuf { - if let Some(home_dir) = dirs::home_dir() { - if let Ok(relative_path) = path.strip_prefix(&home_dir) { - return PathBuf::from("$HOME").join(relative_path); + let testbench_file = find_testbench_file( + &testbench_name, + &bench_dir, + recurse, + cache.entities_cache_mut(), + )?; + + // Filter defaultlib files to exclude OTHER testbenches but allow common bench code + let bench_dir_abs = workspace_dir.as_std_path().join("bench"); + + // Pre-compute entities for bench files to avoid mutable borrow in closure + let mut bench_file_entities: HashMap> = HashMap::new(); + for file_path in &defaultlib_files { + let absolute_path = if file_path.is_relative() { + workspace_dir.as_std_path().join(file_path) + } else { + file_path.clone() + }; + if absolute_path.starts_with(&bench_dir_abs) { + if let Ok(entities) = cache.get_entities(&absolute_path) { + bench_file_entities.insert(absolute_path, entities.clone()); + } } } - path -} -fn extract_repo_name(repo_url: &str) -> String { - repo_url - .trim_end_matches(".git") - .split('/') - .next_back() - .unwrap_or("dependency") - .to_string() -} + let filtered_defaultlib_files: Vec = defaultlib_files + .into_iter() + .filter(|file_path| { + // Convert to absolute path for comparison + let absolute_path = if file_path.is_relative() { + workspace_dir.as_std_path().join(file_path) + } else { + file_path.clone() + }; -fn save_workspace_config( - workspace_dir: &Utf8Path, - config: &WorkspaceConfig, -) -> Result<()> { - let toml_content = toml::to_string_pretty(config)?; - let config_path = workspace_dir.join("vw.toml"); + // If it's not in the bench directory, include it + if !absolute_path.starts_with(&bench_dir_abs) { + return true; + } - fs::write(&config_path, toml_content).map_err(|e| VwError::FileSystem { - message: format!("Failed to write vw.toml file: {e}"), - })?; + // If it's in the bench directory, check if it's a different testbench + if let Some(entities) = bench_file_entities.get(&absolute_path) { + // Exclude files that contain testbench entities other than the one we're running + for entity in entities { + if entity.to_lowercase().ends_with("_tb") + && entity != &testbench_name + { + return false; // This is a different testbench, exclude it + } + } + } - Ok(()) -} + // Include this file (it's either the current testbench or common bench code) + true + }) + .collect(); -pub fn load_workspace_config( - workspace_dir: &Utf8Path, -) -> Result { - let config_path = workspace_dir.join("vw.toml"); - if !config_path.exists() { - return Err(VwError::Config { - message: format!("No vw.toml file found in {workspace_dir}"), - }); - } + // Find only the defaultlib files that are actually referenced by this testbench + let mut referenced_files = find_referenced_files( + &testbench_file, + &filtered_defaultlib_files, + &mut cache, + )?; - let config_content = - fs::read_to_string(&config_path).map_err(|e| VwError::FileSystem { - message: format!("Failed to read vw.toml: {e}"), - })?; + // Sort files in dependency order (dependencies first) + sort_files_by_dependencies( + &mut processor, + &mut referenced_files, + &mut cache, + )?; - let config: WorkspaceConfig = toml::from_str(&config_content)?; + let mut files: Vec = referenced_files + .iter() + .map(|s| s.to_string_lossy().to_string()) + .collect(); - Ok(config) + files.push(testbench_file.to_string_lossy().to_string()); + + run_nvc_analysis(vhdl_std, build_dir, "work", &files, false).await?; + + run_nvc_elab(vhdl_std, build_dir, "work", &testbench_name, false).await?; + + // A testbench whose directory is a Rust *crate* is a cosim bench: its DUT + // inputs are driven by that Rust driver, so it must be loaded or the + // inputs float and numeric_std floods with metavalue warnings. Build and + // load it automatically (even without an explicit `--build-rust`). A + // pure-VHDL testbench sitting directly in `bench/` is *not* — its parent + // `Cargo.toml` is the bench `[workspace]` manifest (no `[package]`), so we + // must check for a real crate rather than any `Cargo.toml`. + let is_cosim_bench = testbench_file + .parent() + .map(dir_is_rust_crate) + .unwrap_or(false); + let rust_lib_path = if build_rust || is_cosim_bench { + Some( + build_rust_library(&bench_dir, &testbench_file) + .await? + .to_string_lossy() + .to_string(), + ) + } else { + None + }; + + // Run NVC simulation, writing the waveform into the per-bench output dir. + let bench_out = bench_output_dir(workspace_dir, &testbench_name); + fs::create_dir_all(&bench_out)?; + run_nvc_sim( + vhdl_std, + build_dir, + "work", + &testbench_name, + bench_out.as_str(), + rust_lib_path, + &runtime_flags.to_vec(), + false, + ) + .await?; + + Ok(()) } -fn load_lock_file(workspace_dir: &Utf8Path) -> Result { - let lock_path = workspace_dir.join("vw.lock"); - if !lock_path.exists() { - return Err(VwError::Config { - message: format!("No vw.lock file found in {workspace_dir}"), - }); +/// Build a `VhdlLsConfig` in memory from the workspace's live +/// enumeration — no disk I/O against `/vhdl_ls.toml`. +/// +/// Populated libraries: +/// - `defaultlib` = design VHDL under `/hdl/**` (variant-filtered). +/// - `ip` = IP wrappers under `/target/ip//wrapper.vhd` +/// (Vivado cache subtrees are excluded by `vhdl_ip_sources`). +/// - `xil_defaultlib` = Vivado-generated BD RTL under +/// `/target/vw-project/**/*.gen/sources_1/bd/**/*.vhd`, +/// present only when the on-disk Vivado project exists. +/// - one library per dep name (hyphens→underscores via +/// `library_name_for_dep`), sourced from +/// `vhdl_dependency_sources_ext`. +/// +/// The sim and LSP layers both consume this instead of parsing +/// `vhdl_ls.toml`; the file is no longer authoritative. +pub fn render_vhdl_ls_config( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, + include_bench: bool, +) -> Result { + let mut libraries: HashMap = HashMap::new(); + + // `active_variant = None` in a variant-mode workspace means + // "the workspace's default variant". Unioning across every + // variant instead would put both `top-vpk120.vhd` AND + // `top-metro.vhd` in `defaultlib`, and vhdl_lang would report + // duplicate-entity / cross-variant unresolved references as + // workspace-wide diagnostics — polluting the LSP outline with + // the inactive variant's broken references. + let resolved_variant = + resolve_active_variant(workspace_dir, active_variant); + let mut design_files = vhdl_design_sources_for_variant( + workspace_dir, + resolved_variant.as_deref(), + )?; + // The LSP puts testbenches in `defaultlib` so an opened tb resolves + // against `work.*`. The sim / anodize paths must NOT — they compile only + // the referenced design set and would choke on bench files that pull in + // external libs (e.g. VUnit) or on unrelated broken testbenches. + if include_bench { + design_files.extend(vhdl_bench_sources(workspace_dir)?); + } + if !design_files.is_empty() { + libraries.insert( + "defaultlib".to_string(), + VhdlLsLibrary { + files: design_files, + exclude: None, + is_third_party: None, + }, + ); } - let lock_content = - fs::read_to_string(&lock_path).map_err(|e| VwError::FileSystem { - message: format!("Failed to read vw.lock: {e}"), - })?; + let ip_files = vhdl_ip_sources(workspace_dir)?; + if !ip_files.is_empty() { + libraries.insert( + "ip".to_string(), + VhdlLsLibrary { + files: ip_files, + exclude: None, + is_third_party: None, + }, + ); + } - let lock_file: LockFile = toml::from_str(&lock_content)?; + let bd_rtl = vivado_generated_sources(workspace_dir)?; + if !bd_rtl.is_empty() { + libraries.insert( + "xil_defaultlib".to_string(), + VhdlLsLibrary { + files: bd_rtl, + exclude: None, + // Suppresses lint noise on Xilinx-generated RTL, + // matching how vhdl_ls treats vendor deps. + is_third_party: Some(true), + }, + ); + } - Ok(lock_file) + let dep_sources = vhdl_dependency_sources_ext(workspace_dir, true, false)?; + for src in dep_sources { + libraries + .entry(src.library) + .or_insert_with(|| VhdlLsLibrary { + files: Vec::new(), + exclude: None, + is_third_party: Some(true), + }) + .files + .push(src.path); + } + + Ok(VhdlLsConfig { + // VHDL 2019 is vw's baseline (mode views etc. show up in + // every non-legacy workspace we support). Without this + // vhdl_lang defaults to 2008 and bails out on view syntax + // partway through the source, which drops entire packages + // from library visibility and shows up as `No primary + // unit '' within library 'defaultlib'` in the LSP. + standard: Some("2019".to_string()), + libraries, + lint: None, + }) } -/// Return the per-user dependency cache directory used by vw. -/// -/// Resolved from `$VW_DEPS_DIR` if set, otherwise `$HOME/.vw/deps`. -/// The directory is created if it does not exist. Callers holding -/// relative paths from [`resolve_deps`] or `vw.lock` should join against -/// the value returned here to obtain absolute paths. -pub fn deps_directory() -> Result { - let deps_dir = if let Some(override_dir) = - std::env::var_os("VW_DEPS_DIR").filter(|v| !v.is_empty()) - { - PathBuf::from(override_dir) - } else { - let home_dir = dirs::home_dir().ok_or_else(|| VwError::FileSystem { - message: "Could not determine home directory".to_string(), - })?; - home_dir.join(".vw").join("deps") - }; +/// Same as [`render_vhdl_ls_config`] but converts to the +/// `vhdl_lang::Config` shape the LSP-embed path wants. Round-trips +/// through TOML because `vhdl_lang::LibraryConfig` fields are +/// private and no builder API is exposed. +pub fn render_vhdl_lang_config( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> Result { + // The LSP wants testbenches resolvable, so it includes bench sources. + let ls_config = render_vhdl_ls_config(workspace_dir, active_variant, true)?; + let toml_str = toml::to_string(&ls_config)?; + vhdl_lang::Config::from_str(&toml_str, workspace_dir.as_std_path()).map_err( + |e| VwError::Config { + message: format!("vhdl_lang config parse failed: {e}"), + }, + ) +} - fs::create_dir_all(&deps_dir).map_err(|e| VwError::FileSystem { - message: format!("Failed to create dependencies directory: {e}"), - })?; +/// Severity of a [`VhdlDiagnostic`] — a `vw`-local mirror of +/// `vhdl_lang::Severity` so callers don't need to depend on +/// `vhdl_lang` themselves. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VhdlSeverity { + Error, + Warning, + Info, + Hint, +} - Ok(deps_dir) +/// A single vhdl_lang static-analysis finding, flattened for display. +/// Line/column are 1-based and ready to print. +#[derive(Debug, Clone)] +pub struct VhdlDiagnostic { + pub file: PathBuf, + pub line: u32, + pub column: u32, + pub severity: VhdlSeverity, + pub message: String, } -/// Resolve a path stored in `vw.lock` against the local dependency cache. +/// Git source of the VHDL standard library (`std` / `ieee`). +/// `vhdl_lang` needs these VHDL sources to build its type graph but +/// doesn't bundle them — on a machine with no rust_hdl install, its +/// `load_external_config` search comes up empty and analysis panics on +/// the universal-integer lookup. Rather than vendoring the library, vw +/// materializes it through the normal dependency cache +/// (`~/.vw/deps/rust_hdl-/vhdl_libraries`) and hands vhdl_lang that +/// path. /// -/// Lock-file dep paths are stored as `-` (relative to the -/// per-user `$HOME/.vw/deps` directory) so the file is identical across -/// machines. Absolute paths are returned unchanged to remain compatible -/// with lock files written by older versions of vw. -fn resolve_dep_path(path: &Path) -> Result { - if path.is_absolute() { - return Ok(path.to_path_buf()); +/// Tracks the default branch (`master`) — the only ref +/// `download_dependency`'s shallow clone can fetch — resolved once at +/// first download. The stdlib is the fixed VHDL standard, so tracking +/// HEAD is safe and independent of the analyzer version; a cached copy +/// is then reused forever with no further network. `vw clear` drops it +/// and the next check re-fetches. +const VHDL_STDLIB_REPO: &str = "https://github.com/vhdl-ls/rust_hdl"; +const VHDL_STDLIB_BRANCH: &str = "master"; + +/// Path to an already-materialized rust_hdl `vhdl_libraries` dir in the +/// dependency cache, if any. The stdlib is stable, so any cached copy +/// is usable — checked with no network so it works offline. +fn find_cached_vhdl_stdlib(deps_dir: &Path) -> Option { + for entry in fs::read_dir(deps_dir).ok()?.flatten() { + if !entry.file_name().to_string_lossy().starts_with("rust_hdl-") { + continue; + } + let libs = entry.path().join("vhdl_libraries"); + let present = libs.exists() + && fs::read_dir(&libs) + .map(|mut d| d.next().is_some()) + .unwrap_or(false); + if present { + if let Ok(u) = Utf8PathBuf::from_path_buf(libs) { + return Some(u); + } + } } - let deps_dir = deps_directory()?; - Ok(deps_dir.join(path)) + None } -async fn resolve_dependency_commit( - repo_url: &str, - branch: &Option, - commit: &Option, - credentials: Option<(&str, &str)>, // (username, password) -) -> Result { - match (branch, commit) { - (Some(_), Some(_)) => Err(VwError::Config { - message: "Cannot specify both branch and commit for dependency" +/// Ensure the VHDL standard library is present in the dependency cache +/// and return the path to its `vhdl_libraries` dir — ready to hand to +/// [`check_vhdl`] (and, through `load_external_config`, to `vhdl_lang`). +/// +/// Reuses any cached copy without touching the network; otherwise +/// downloads [`VHDL_STDLIB_REPO`] on first use through the same +/// `download_dependency` machinery as any other git dependency. +pub async fn ensure_vhdl_stdlib() -> Result { + let deps_dir = deps_directory()?; + if let Some(libs) = find_cached_vhdl_stdlib(&deps_dir) { + return Ok(libs); + } + let sha = resolve_dependency_commit( + VHDL_STDLIB_REPO, + &Some(VHDL_STDLIB_BRANCH.to_string()), + &None, + None, + ) + .await?; + let dep_path = deps_dir.join(format!("rust_hdl-{sha}")); + let libs = dep_path.join("vhdl_libraries"); + let present = libs.exists() + && fs::read_dir(&libs) + .map(|mut d| d.next().is_some()) + .unwrap_or(false); + if !present { + if dep_path.exists() { + let _ = fs::remove_dir_all(&dep_path); + } + download_dependency( + VHDL_STDLIB_REPO, + &sha, + &[], + &dep_path, + false, + &[], + false, + None, + Some("vhdl_libraries"), + ) + .await?; + } + Utf8PathBuf::from_path_buf(libs).map_err(|p| VwError::FileSystem { + message: format!("VHDL stdlib path is not UTF-8: {}", p.display()), + }) +} + +/// Cheap check for whether a workspace renders any VHDL. Gates the +/// (stdlib-fetching) VHDL check so pure-htcl workspaces skip it — and +/// its one-time network download — entirely. +pub fn workspace_has_vhdl( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, +) -> bool { + render_vhdl_ls_config(workspace_dir, active_variant, true) + .map(|c| c.libraries.values().any(|lib| !lib.files.is_empty())) + .unwrap_or(false) +} + +/// Run vhdl_lang static analysis over the workspace's VHDL — the same +/// analysis `vw-analyzer` runs live in the editor, but in one batch — +/// and return the non-suppressed findings that land in the workspace's +/// OWN tree. Dependency libraries and the bundled VHDL standard library +/// are analyzed for name resolution but their internal diagnostics are +/// filtered out: the user can't fix those and they'd only be noise. +/// Results are sorted by file, then position. +/// +/// Cheap no-op returning an empty vec when the workspace renders no +/// VHDL libraries — a pure-htcl workspace has no HDL to check, so we +/// skip building the project (and parsing the standard library) too. +pub fn check_vhdl( + workspace_dir: &Utf8Path, + active_variant: Option<&str>, + stdlib_libraries_path: Option<&Utf8Path>, +) -> Result> { + let ls_config = render_vhdl_ls_config(workspace_dir, active_variant, true)?; + if ls_config.libraries.values().all(|lib| lib.files.is_empty()) { + return Ok(Vec::new()); // no VHDL to analyze + } + let toml_str = toml::to_string(&ls_config)?; + let user_config = + vhdl_lang::Config::from_str(&toml_str, workspace_dir.as_std_path()) + .map_err(|e| VwError::Config { + message: format!("vhdl_lang config parse failed: {e}"), + })?; + + // Load the VHDL standard library (`std` / `ieee`) via + // `load_external_config`, then append the workspace's own libraries + // on top. `stdlib_libraries_path` points at a `vhdl_libraries` dir + // (vw fetches one into the dep cache — see `ensure_vhdl_stdlib`); + // `None` falls back to vhdl_lang's built-in search of installed + // locations. Without `std`, vhdl_lang's type-graph construction + // panics on the universal-integer lookup — so if it's still missing + // we skip the VHDL check rather than crash. `vw check` still runs + // the htcl half. + let mut messages = vhdl_lang::NullMessages; + let mut config = vhdl_lang::Config::default(); + config.load_external_config( + &mut messages, + stdlib_libraries_path.map(|p| p.to_string()), + ); + if !config.iter_libraries().any(|lib| lib.name() == "std") { + return Ok(Vec::new()); + } + config.append(&user_config, &mut messages); + + let severities = *config.severities(); + let mut project = vhdl_lang::Project::from_config(config, &mut messages); + + // Restrict findings to the workspace's own *source* tree: not deps + // under `~/.vw/deps`, not the bundled standard library, and not the + // `target/` build output. Generated RTL (Vivado IP wrappers, BD + // netlists) is still analyzed so the design's references resolve, + // but diagnostics INSIDE it are the tool's output, not the user's + // HDL — reporting them (e.g. an undeclared `UNISIM` in a generated + // wrapper) is just noise here. + let ws_root = workspace_dir + .as_std_path() + .canonicalize() + .unwrap_or_else(|_| workspace_dir.as_std_path().to_path_buf()); + let ws_target = ws_root.join("target"); + + let mut out: Vec = project + .analyse() + .into_iter() + .filter_map(|d| { + let severity = severities[d.code]?; // `None` = suppressed + let file = d.pos.file_name(); + let canon = file.canonicalize(); + let path = canon.as_deref().unwrap_or(file); + if !path.starts_with(&ws_root) || path.starts_with(&ws_target) { + return None; + } + let start = d.pos.start(); + Some(VhdlDiagnostic { + file: file.to_path_buf(), + line: start.line + 1, + column: start.character + 1, + severity: match severity { + vhdl_lang::Severity::Error => VhdlSeverity::Error, + vhdl_lang::Severity::Warning => VhdlSeverity::Warning, + vhdl_lang::Severity::Info => VhdlSeverity::Info, + vhdl_lang::Severity::Hint => VhdlSeverity::Hint, + }, + message: d.message, + }) + }) + .collect(); + out.sort_by(|a, b| { + a.file + .cmp(&b.file) + .then(a.line.cmp(&b.line)) + .then(a.column.cmp(&b.column)) + }); + Ok(out) +} + +/// Resolve which variant the LSP renderer should filter to. +/// +/// Precedence: +/// 1. Caller-supplied `active_variant` (e.g. sim invocation). +/// 2. `VW_ACTIVE_VARIANT` env var — the LSP-side selector: users +/// launch `helix` (or a shell) with the var set to switch +/// which board's tree is analyzed. Empty / whitespace-only +/// values are ignored so `unset` and `export FOO=""` behave +/// identically. +/// 3. Workspace's `default = true` variant — the natural fallback +/// (matches `vw run` semantics). +/// 4. `None` — workspace has no variants; caller does no +/// filtering. +fn resolve_active_variant( + workspace_dir: &Utf8Path, + caller_supplied: Option<&str>, +) -> Option { + if let Some(v) = caller_supplied { + return Some(v.to_string()); + } + if let Ok(env) = std::env::var("VW_ACTIVE_VARIANT") { + let trimmed = env.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + let cfg = load_workspace_config(workspace_dir).ok()?; + if cfg.workspace.variants.is_empty() { + return None; + } + cfg.workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.name.clone()) +} + +/// Walk `/target/vw-project/*/*.gen/sources_1/**/*.{vhd,vhdl}`, +/// keeping only the files vhdl_lang can meaningfully analyze from +/// Vivado's output tree. +/// +/// The unfiltered tree has three inclusion hazards: +/// - **Sim/synth duplicates.** Every BD gets a `bd//synth/.vhd` +/// AND a `bd//sim/.vhd`, both declaring `entity is`. +/// Including both fires vhdl_lang's duplicate-declaration path plus +/// thousands of Xilinx-specific attribute errors from the sim +/// variant (`entity txr0` alone contributed 614 errors on metroid). +/// - **`ipshared/` shared IP bundles.** Xilinx-provided reusable +/// function sets (`*_rfs.vhd`) use vendor extensions and error out +/// under vhdl_lang. +/// - **`*_sim_netlist.vhdl`.** Post-synth flattened netlists that +/// redeclare the same entity name as the corresponding `_stub`. +/// +/// Kept: `bd/*/hdl/*_wrapper.vhd`, `bd/*/synth/**/*.{vhd,vhdl}`, +/// `bd/*/ip/**/synth/**/*.{vhd,vhdl}`, `ip/*/*_stub.{vhd,vhdl}`. +fn vivado_generated_sources(workspace_dir: &Utf8Path) -> Result> { + let project_root = workspace_dir.join("target/vw-project"); + if !project_root.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + let Ok(entries) = fs::read_dir(project_root.as_std_path()) else { + return Ok(Vec::new()); + }; + for entry in entries.flatten() { + let dir_path = entry.path(); + let Some(name) = dir_path.file_name() else { + continue; + }; + let gen_dir = format!("{}.gen", name.to_string_lossy()); + let sources_1 = dir_path.join(gen_dir).join("sources_1"); + if !sources_1.is_dir() { + continue; + } + let mut all = Vec::new(); + find_vhdl_files_impl(&sources_1, &mut all, true)?; + for path in all { + if keep_vivado_generated_path(&path) { + out.push(path); + } + } + } + out.sort(); + Ok(out) +} + +/// Filter predicate for [`vivado_generated_sources`]. Only files +/// the user's own code can name are retained; the deeper +/// synthesis-time entities user code never references directly are +/// dropped so vhdl_lang doesn't parse them at all. +/// +/// User code references BD components as `_wrapper` (the +/// entity in `bd//hdl/_wrapper.vhd`) — that wrapper +/// wraps `bd//synth/.vhd` internally through a +/// component declaration, and the component isn't part of the LSP +/// resolution surface. Include the wrapper, skip the synth entity. +/// Same reasoning drops `bd//ip/**` sub-IP wrappers and every +/// `ipshared/` bundle: they're compiled by Vivado but never +/// mentioned by user-authored VHDL. XCI IPs like `primary_clock` +/// have the same shape one level up — `ip//_stub.vhdl` +/// declares the entity user code names. +fn keep_vivado_generated_path(path: &Path) -> bool { + let s = path.to_string_lossy().replace('\\', "/"); + let name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + // BD subtree: only the top-level `hdl/_wrapper.vhd`. + if let Some(bd_rel) = s + .split("/sources_1/bd/") + .nth(1) + .and_then(|rel| rel.split_once('/')) + { + let sub_path = bd_rel.1; + return sub_path.starts_with("hdl/") && name.ends_with("_wrapper.vhd"); + } + // XCI IP subtree: only `_stub.{vhd,vhdl}`. + if s.contains("/sources_1/ip/") { + return name.ends_with("_stub.vhd") || name.ends_with("_stub.vhdl"); + } + // Unknown subtree under `sources_1/` — keep by default so + // future Vivado output shapes aren't silently dropped. + true +} + +/// Strip VHDL line comments (`-- …` to end of line) so a structural +/// scan doesn't trip over the `.vho` template's banner lines (e.g. +/// `------ Begin Cut here for COMPONENT Declaration`, which would +/// otherwise look like a `component` token). +fn strip_vhdl_line_comments(src: &str) -> String { + src.lines() + .map(|line| match line.find("--") { + Some(i) => &line[..i], + None => line, + }) + .collect::>() + .join("\n") +} + +/// Rewrite the VHDL *component* declaration Vivado emits in an IP's +/// `.vho` instantiation template into a standalone black-box *entity* +/// plus an empty architecture. A component and an entity share +/// generic/port syntax verbatim, so this is a mechanical splice — no +/// port parsing — which keeps it robust to whatever types the IP's +/// ports use. Returns `None` when no component declaration is found. +/// +/// The result declares only the interface; the empty architecture is +/// a black box. It's compiled into `xil_defaultlib` (per +/// `vhdl_ls.toml`) so `entity xil_defaultlib.` resolves for the +/// static check. It is NOT for synthesis — `vw::synth` uses the real +/// IP netlist. +fn vho_component_to_entity(vho: &str) -> Option { + let clean = strip_vhdl_line_comments(vho); + let lower = clean.to_ascii_lowercase(); + + // Find the `component` keyword that OPENS the declaration — not the + // `end component` terminator. Scan for a `component` token whose + // preceding word isn't `end` and which sits on word boundaries. + let is_ident_char = |b: u8| b.is_ascii_alphanumeric() || b == b'_'; + let decl = { + let bytes = lower.as_bytes(); + let mut from = 0; + loop { + let rel = lower[from..].find("component")?; + let idx = from + rel; + let after = idx + "component".len(); + let left_ok = idx == 0 || !is_ident_char(bytes[idx - 1]); + let right_ok = + bytes.get(after).map(|b| !is_ident_char(*b)).unwrap_or(true); + let prev_word_end = + lower[..idx].split_whitespace().last() != Some("end"); + if left_ok && right_ok && prev_word_end { + break idx; + } + from = after; + } + }; + + // The IP/entity name is the first identifier after `component`. + let after_kw = decl + "component".len(); + let rest = &clean[after_kw..]; + let name_off = rest.find(|c: char| !c.is_whitespace())?; + let name_rest = &rest[name_off..]; + let name_len = name_rest + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .unwrap_or(name_rest.len()); + let name = &name_rest[..name_len]; + if name.is_empty() { + return None; + } + + // Body = generics/ports between the name and `end component`. + let body_start = after_kw + name_off + name_len; + let term_rel = lower[body_start..].find("end component")?; + let mut body = clean[body_start..body_start + term_rel].trim(); + // VHDL-2008 allows `component NAME is`; drop a leading `is` so we + // don't emit `entity NAME is is …`. + if body.len() >= 2 + && body[..2].eq_ignore_ascii_case("is") + && body[2..].starts_with(char::is_whitespace) + { + body = body[2..].trim_start(); + } + + Some(format!( + "-- Auto-generated black-box stub for IP `{name}`, derived from\n\ + -- Vivado's VHDL instantiation template (`{name}.vho`) so the\n\ + -- static VHDL check can resolve `entity xil_defaultlib.{name}`.\n\ + -- NOT for synthesis — vw::synth uses the real IP netlist.\n\ + library ieee;\n\ + use ieee.std_logic_1164.all;\n\ + use ieee.numeric_std.all;\n\ + \n\ + entity {name} is\n\ + {body}\n\ + end entity;\n\ + \n\ + architecture stub of {name} is\n\ + begin\n\ + end architecture;\n" + )) +} + +/// Turn each standalone XCI IP's Vivado instantiation template +/// (`.vho`) into a black-box `_stub.vhdl` alongside it, so the +/// static VHDL check can resolve `entity xil_defaultlib.` without +/// the IP ever being synthesized. Scans +/// `target/vw-project/*/*.gen/sources_1/ip/*/` for a top-level `.vho` +/// (the top IP; sub-IP templates in nested dirs are skipped). Writes +/// only when content changed, and is a no-op when there are no +/// templates. Returns how many stubs were (re)written. +pub fn write_ip_stubs_from_templates( + workspace_dir: &Utf8Path, +) -> Result { + let project_root = workspace_dir.join("target/vw-project"); + let Ok(projects) = fs::read_dir(project_root.as_std_path()) else { + return Ok(0); + }; + let mut written = 0usize; + for project in projects.flatten() { + let name = project.file_name(); + let ip_root = project + .path() + .join(format!("{}.gen", name.to_string_lossy())) + .join("sources_1") + .join("ip"); + let Ok(ip_dirs) = fs::read_dir(&ip_root) else { + continue; + }; + for ip_dir in ip_dirs.flatten() { + if !ip_dir.path().is_dir() { + continue; + } + let Ok(entries) = fs::read_dir(ip_dir.path()) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("vho") { + continue; + } + let Ok(vho) = fs::read_to_string(&path) else { + continue; + }; + let Some(stub) = vho_component_to_entity(&vho) else { + continue; + }; + let Some(stem) = path.file_stem().and_then(|s| s.to_str()) + else { + continue; + }; + let out = ip_dir.path().join(format!("{stem}_stub.vhdl")); + let unchanged = fs::read_to_string(&out) + .map(|c| c == stub) + .unwrap_or(false); + if !unchanged && fs::write(&out, &stub).is_ok() { + written += 1; + } + } + } + } + Ok(written) +} + +fn find_testbench_file_recurse( + testbench_name: &str, + bench_dir: &Utf8Path, + recurse: bool, + entities_cache: &mut HashMap>, +) -> Result> { + let mut found_files = Vec::new(); + + for entry in fs::read_dir(bench_dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to read bench directory: {e}"), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + + if path.is_file() { + if let Some(extension) = path.extension() { + if extension == "vhd" || extension == "vhdl" { + // Check if this file contains the entity we're looking for + if file_contains_entity( + &path, + testbench_name, + entities_cache, + )? { + found_files.push(path); + } + } + } + } else if recurse { + let dir_path: Utf8PathBuf = + path.try_into().map_err(|e| VwError::FileSystem { + message: format!("Failed to get dir path: {e}"), + })?; + let mut lower_testbenches = find_testbench_file_recurse( + testbench_name, + &dir_path, + recurse, + entities_cache, + )?; + found_files.append(&mut lower_testbenches); + } + } + Ok(found_files) +} + +fn find_testbench_file( + testbench_name: &str, + bench_dir: &Utf8Path, + recurse: bool, + entities_cache: &mut HashMap>, +) -> Result { + let found_files = find_testbench_file_recurse( + testbench_name, + bench_dir, + recurse, + entities_cache, + )?; + + match found_files.len() { + 0 => Err(VwError::Testbench { + message: format!("Testbench entity '{testbench_name}' not found in bench directory") + }), + 1 => Ok(found_files.into_iter().next().unwrap()), + _ => Err(VwError::Testbench { + message: format!("Multiple files contain entity '{testbench_name}': {found_files:?}") + }), + } +} + +fn file_contains_entity( + file_path: &Path, + entity_name: &str, + entities_cache: &mut HashMap>, +) -> Result { + let entities = get_cached_entities(file_path, entities_cache)?; + Ok(entities.iter().any(|e| e.eq_ignore_ascii_case(entity_name))) +} + +/// Get entities from cache, parsing and caching if not present. +fn get_cached_entities<'a>( + path: &Path, + entities_cache: &'a mut HashMap>, +) -> Result<&'a Vec> { + match entities_cache.entry(path.to_path_buf()) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let content = + fs::read_to_string(path).map_err(|e| VwError::FileSystem { + message: format!("Failed to read file {path:?}: {e}"), + })?; + let entities = parse_entities(&content)?; + Ok(e.insert(entities)) + } + } +} + +fn extract_repo_name(repo_url: &str) -> String { + repo_url + .trim_end_matches(".git") + .split('/') + .next_back() + .unwrap_or("dependency") + .to_string() +} + +fn save_workspace_config( + workspace_dir: &Utf8Path, + config: &WorkspaceConfig, +) -> Result<()> { + let toml_content = toml::to_string_pretty(config)?; + let config_path = workspace_dir.join("vw.toml"); + + fs::write(&config_path, toml_content).map_err(|e| VwError::FileSystem { + message: format!("Failed to write vw.toml file: {e}"), + })?; + + Ok(()) +} + +/// Walk up from `start` (typically a directory) looking for the +/// first `vw.toml` file in the ancestor chain. Returns the +/// containing directory, or `None` if none is found. +/// +/// Callers that hold a FILE path — say `entry.htcl` — should pass +/// `entry.parent()` since a file itself cannot contain a +/// `vw.toml`. The check uses `is_file()` so a stray directory +/// named `vw.toml` doesn't trip a false positive. +/// +/// Consolidates three near-duplicate helpers that previously +/// lived in `vw-cli::find_workspace_dir`, `vw-repl::app:: +/// find_vw_toml_ancestor`, and `vw-repl::lower:: +/// find_workspace_dir`. The `vw-analyzer` multi-root discovery +/// (LSP `initialize`-supplied roots) is a different concept and +/// stays in the analyzer. +/// Return `/design.htcl` when it exists on disk, +/// else `None`. Mirrors the `module.htcl` convention for library +/// workspaces — `design.htcl` is the project workspace's entry +/// script, auto-discovered by `vw run` / `vw repl` / `vw check` +/// when the user invokes them with no file argument. +pub fn find_design_file(workspace_dir: &Utf8Path) -> Option { + let p = workspace_dir.join("design.htcl"); + p.is_file().then_some(p) +} + +pub fn find_workspace_dir(start: &Path) -> Option { + // Canonicalize UP FRONT so the walk-up starts from an + // absolute path. Callers hand us relative or empty paths in + // real workflows: `vw run prime.htcl` derives + // `Path("prime.htcl").parent() == ""`, which used to yield an + // empty-string workspace root — served over RPC to htcl, + // that caused `[file join $root target ip $ip]` to compute a + // RELATIVE path (`target/ip/cips`), which Vivado created + // inside its auto-cleaned tempdir cwd. The wrapper was + // silently written and immediately deleted on process exit. + // Canonicalizing here fixes every downstream consumer + // (workspace_root RPC, LSP compat check, htcl-test) in one + // place. Fallback to the raw start when canonicalize fails + // (e.g. path doesn't exist yet) so we don't regress the "no + // workspace" branch for freshly-created files. + // Empty path is a special case — `Path("").canonicalize()` + // errors with ENOENT, and `parent()` on it yields None + // immediately, so we'd terminate the walk before ever + // checking the cwd. Fold empty → cwd first, then canonicalize. + let start_pb = if start.as_os_str().is_empty() { + std::env::current_dir().ok()? + } else { + start.to_path_buf() + }; + let canon = start_pb.canonicalize().unwrap_or(start_pb); + let mut cur = Utf8PathBuf::from_path_buf(canon).ok()?; + loop { + if cur.join("vw.toml").is_file() { + return Some(cur); + } + // `Utf8Path::parent()` returns `None` at the filesystem + // root; the loop naturally terminates without needing a + // manual `parent == cur` guard. + cur = cur.parent()?.to_path_buf(); + } +} + +pub fn load_workspace_config( + workspace_dir: &Utf8Path, +) -> Result { + let config_path = workspace_dir.join("vw.toml"); + if !config_path.exists() { + return Err(VwError::Config { + message: format!("No vw.toml file found in {workspace_dir}"), + }); + } + + let config_content = + fs::read_to_string(&config_path).map_err(|e| VwError::FileSystem { + message: format!("Failed to read vw.toml: {e}"), + })?; + + let config: WorkspaceConfig = toml::from_str(&config_content)?; + validate_variant_shape(&config.workspace)?; + Ok(config) +} + +/// Post-deserialize validation for the `[[target-parts]]` / +/// `[[workspace.variants]]` mutual exclusion + variant-name +/// uniqueness. Returns [`VwError::Config`] with the same +/// user-facing message the [`VariantSelectError`] carries so +/// the loader surfaces the specific failure verbatim. +fn validate_variant_shape(ws: &WorkspaceInfo) -> Result<()> { + if !ws.variants.is_empty() && !ws.target_parts.is_empty() { + return Err(VwError::Config { + message: VariantSelectError::BothPartsAndVariants.to_string(), + }); + } + let mut seen: std::collections::HashSet<&str> = + std::collections::HashSet::new(); + for v in &ws.variants { + if !seen.insert(v.name.as_str()) { + return Err(VwError::Config { + message: VariantSelectError::DuplicateName { + name: v.name.clone(), + } + .to_string(), + }); + } + } + Ok(()) +} + +fn load_lock_file(workspace_dir: &Utf8Path) -> Result { + let lock_path = workspace_dir.join("vw.lock"); + if !lock_path.exists() { + return Err(VwError::Config { + message: format!("No vw.lock file found in {workspace_dir}"), + }); + } + + let lock_content = + fs::read_to_string(&lock_path).map_err(|e| VwError::FileSystem { + message: format!("Failed to read vw.lock: {e}"), + })?; + + let lock_file: LockFile = toml::from_str(&lock_content)?; + + Ok(lock_file) +} + +/// Return the per-user dependency cache directory used by vw. +/// +/// Resolved from `$VW_DEPS_DIR` if set, otherwise `$HOME/.vw/deps`. +/// The directory is created if it does not exist. Callers holding +/// relative paths from [`resolve_deps`] or `vw.lock` should join against +/// the value returned here to obtain absolute paths. +pub fn deps_directory() -> Result { + let deps_dir = if let Some(override_dir) = + std::env::var_os("VW_DEPS_DIR").filter(|v| !v.is_empty()) + { + PathBuf::from(override_dir) + } else { + let home_dir = dirs::home_dir().ok_or_else(|| VwError::FileSystem { + message: "Could not determine home directory".to_string(), + })?; + home_dir.join(".vw").join("deps") + }; + + fs::create_dir_all(&deps_dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to create dependencies directory: {e}"), + })?; + + Ok(deps_dir) +} + +/// Resolve a path stored in `vw.lock` against the local dependency cache. +/// +/// Lock-file dep paths are stored as `-` (relative to the +/// per-user `$HOME/.vw/deps` directory) so the file is identical across +/// machines. Absolute paths are returned unchanged to remain compatible +/// with lock files written by older versions of vw. +/// Build a `name → absolute cache path` map for every dependency in +/// the workspace's `vw.lock`. Used by htcl's `src @name/...` resolver +/// in `vw-htcl::src_path::Resolver` so the language-layer crate stays +/// free of workspace / lockfile concerns. +/// +/// Returns an empty map (not an error) if the workspace has no +/// `vw.lock` yet — relative and absolute `src` imports still work +/// against an empty resolver, only `@name/` lookups fail. +pub fn dep_cache_paths( + workspace_dir: &Utf8Path, +) -> Result> { + dep_cache_paths_with_test(workspace_dir, false) +} + +/// Same as [`dep_cache_paths`] but optionally includes +/// `[test-dependencies]`. Only `vw test` should pass +/// `include_test = true`; other callers see the same map they +/// always did, so `vw run`/`vw check`/`vw update`'s dep behavior +/// is unchanged. +pub fn dep_cache_paths_with_test( + workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + let mut out = HashMap::new(); + + // Local deps live wherever `vw.toml` says; they don't need a + // lockfile (nothing to pin). Read them straight from the workspace + // config so they work before — or without — a `vw update`. + if let Ok(config) = load_workspace_config(workspace_dir) { + for (name, dep) in config.dependencies { + if let Some(path) = dep.local_path() { + out.insert(name, resolve_local_dep_path(workspace_dir, path)); + } + } + if include_test { + for (name, dep) in config.test_dependencies { + if let Some(path) = dep.local_path() { + out.insert( + name, + resolve_local_dep_path(workspace_dir, path), + ); + } + } + } + } + + // Git deps are resolved through the lockfile and the per-user + // cache. A missing lock isn't an error here — just skip git entries. + // The lockfile stores test-dep and normal-dep entries in the same + // `dependencies` section (no separate section for locks — see the + // rationale on `update_workspace_with_token`). Non-test callers + // want to filter out entries that came exclusively from + // `[test-dependencies]`; simplest safe rule for now: everything in + // the lockfile is exposed regardless of section. A subsequent PR + // can add a `test = true` marker per lock entry if this becomes a + // real concern. + match load_lock_file(workspace_dir) { + Ok(lock) => { + for (name, locked) in lock.dependencies { + // A manifest path dep already claimed this name. The + // manifest is authoritative for a dependency's KIND, so + // a stale git lock entry left over from a `repo → path` + // switch must not override it (the reason the `out` + // insert order matters). The lock is rewritten to drop + // the entry on the next re-resolve — see + // `lock_is_stale_against_manifest` — but resolution has + // to be correct even before that runs. + if out.contains_key(&name) { + continue; + } + let abs = resolve_dep_path(&locked.path)?; + out.insert(name, abs); + } + } + Err(VwError::Config { .. }) => {} + Err(e) => return Err(e), + } + + Ok(out) +} + +/// Drop `vw.lock` entries the manifest now declares as **path** deps — +/// the stale git pin left behind by a `repo → path` switch. Path deps +/// are never locked (nothing to pin), so the correct lock simply omits +/// them. Rewrites the lock in place when it changed; returns whether it +/// did (so the caller can report it). +/// +/// Deliberately surgical: it removes only the now-path entries and +/// leaves every other (git) pin untouched. A full re-resolve would hit +/// the network and re-pin every branch-tracking git dep to its current +/// HEAD — a `repo → path` edit must not silently bump unrelated +/// dependencies. The reverse switch (`path → repo`) needs no handling +/// here: it leaves the manifest dep git-shaped and unlocked, which +/// [`dependencies_present`] already treats as "fetch me". +pub fn prune_stale_path_deps_from_lock( + workspace_dir: &Utf8Path, +) -> Result { + let Ok(mut lock) = load_lock_file(workspace_dir) else { + return Ok(false); + }; + let Ok(config) = load_workspace_config(workspace_dir) else { + return Ok(false); + }; + let path_dep_names: std::collections::HashSet = config + .dependencies + .iter() + .chain(config.test_dependencies.iter()) + .filter(|(_, dep)| dep.local_path().is_some()) + .map(|(name, _)| name.clone()) + .collect(); + let before = lock.dependencies.len(); + lock.dependencies + .retain(|name, _| !path_dep_names.contains(name)); + if lock.dependencies.len() == before { + return Ok(false); + } + write_lock_file(workspace_dir, &lock)?; + Ok(true) +} + +fn resolve_dep_path(path: &Path) -> Result { + if path.is_absolute() { + return Ok(path.to_path_buf()); + } + let deps_dir = deps_directory()?; + Ok(deps_dir.join(path)) +} + +/// Resolve a `path = "..."` dependency to an absolute, canonicalized +/// root. Relative paths resolve against the workspace that DECLARES +/// them (the same Cargo rule [`resolve_dep_source_path`] uses), so a +/// dep buried in a transitive workspace — e.g. an in-tree `testlib` +/// that declares `vw = ".."` — points at the real directory instead of +/// leaking its literal `..` into the import resolver. Canonicalizing +/// also lets the transitive walk dedup a circular dep (vw → testlib → +/// vw) by real path rather than looping on `/testlib/..`. +fn resolve_local_dep_path(workspace_dir: &Utf8Path, path: &Path) -> PathBuf { + let abs = if path.is_absolute() { + path.to_path_buf() + } else { + workspace_dir.as_std_path().join(path) + }; + abs.canonicalize().unwrap_or(abs) +} + +/// Like [`dep_cache_paths`], but walks the dependency graph +/// transitively: for every dep whose cached root is itself a +/// workspace (i.e. has its own `vw.toml`), pull in *its* deps too, +/// and so on. The result is a flat `name → root` map covering every +/// dep any file in this workspace's transitive closure might +/// `src @/...`-import. +/// +/// First-seen-wins on name conflicts so the entry workspace's +/// declarations take precedence over a dep's choice of the same +/// name (matching Cargo's resolution: the top-level `Cargo.toml` +/// pins the version for the whole graph). +/// +/// Returns an empty map (not an error) if the entry workspace has +/// no deps. Per-dep failures (missing `vw.toml`, malformed config) +/// are skipped: a dep may not be its own htcl workspace, and that's +/// fine — we just won't see *its* deps. +pub fn transitive_dep_cache_paths( + entry_workspace_dir: &Utf8Path, +) -> Result> { + transitive_dep_cache_paths_with_test(entry_workspace_dir, false) +} + +/// Like [`transitive_dep_cache_paths`] but optionally includes +/// the ENTRY workspace's `[test-dependencies]`. Cargo-parity +/// semantic for `dev-dependencies`: test-deps are private to the +/// workspace that declares them. Recursed-into workspaces are +/// walked with `include_test = false` so a dep's own test-deps +/// aren't pulled into your consumer. +pub fn transitive_dep_cache_paths_with_test( + entry_workspace_dir: &Utf8Path, + include_test: bool, +) -> Result> { + let mut out: HashMap = HashMap::new(); + let mut visited: std::collections::HashSet = + std::collections::HashSet::new(); + // Only the entry workspace's paths are gathered with + // `include_test`; everything queued after gets the normal + // treatment. + let mut first_iter = true; + // Canonicalize the entry so it dedups against the canonicalized + // dep roots `resolve_local_dep_path` produces — otherwise a + // circular local dep (vw → testlib → vw) reappears as + // `/testlib/..` and the walk never converges. + let entry = entry_workspace_dir.as_std_path().to_path_buf(); + let entry = entry.canonicalize().unwrap_or(entry); + let mut queue: Vec = vec![entry]; + + while let Some(ws) = queue.pop() { + if !visited.insert(ws.clone()) { + continue; + } + let Ok(ws_utf8) = Utf8PathBuf::from_path_buf(ws) else { + continue; + }; + let want_test = include_test && first_iter; + first_iter = false; + let Ok(paths) = dep_cache_paths_with_test(&ws_utf8, want_test) else { + continue; + }; + for (name, dep_path) in paths { + // First-seen wins — don't let a transitive dep override + // the entry workspace's choice. + out.entry(name).or_insert_with(|| dep_path.clone()); + // If the dep is itself a workspace, recurse into it. A + // dep without a `vw.toml` is a leaf (just files). + if dep_path.join("vw.toml").exists() { + queue.push(dep_path); + } + } + } + Ok(out) +} + +async fn resolve_dependency_commit( + repo_url: &str, + branch: &Option, + commit: &Option, + credentials: Option<(&str, &str)>, // (username, password) +) -> Result { + match (branch, commit) { + (Some(_), Some(_)) => Err(VwError::Config { + message: "Cannot specify both branch and commit for dependency" + .to_string(), + }), + (None, None) => Err(VwError::Config { + message: "Must specify either branch or commit for dependency" + .to_string(), + }), + (None, Some(commit)) => Ok(commit.clone()), + (Some(branch), None) => { + get_branch_head_commit(repo_url, branch, credentials).await + } + } +} + +async fn get_branch_head_commit( + repo_url: &str, + branch: &str, + credentials: Option<(&str, &str)>, // (username, password) +) -> Result { + // Normalize repository URL to ensure it ends with .git for GitHub + let normalized_repo_url = + if repo_url.contains("github.com") && !repo_url.ends_with(".git") { + format!("{repo_url}.git") + } else { + repo_url.to_string() + }; + + let branch = branch.to_string(); + let credentials = credentials.map(|(u, p)| (u.to_string(), p.to_string())); + + tokio::time::timeout( + std::time::Duration::from_secs(30), + tokio::task::spawn_blocking(move || { + // Create a temporary directory for the operation + let temp_dir = + tempfile::tempdir().map_err(|e| VwError::FileSystem { + message: format!( + "Failed to create temporary directory: {e}" + ), + })?; + + // Create an empty repository to work with remotes + let repo = + git2::Repository::init_bare(temp_dir.path()).map_err(|e| { + VwError::Git { + message: format!( + "Failed to initialize temporary repository: {e}" + ), + } + })?; + + // Create a remote + let mut remote = repo + .remote_anonymous(&normalized_repo_url) + .map_err(|e| VwError::Git { + message: format!("Failed to create remote: {e}"), + })?; + + // Connect and list references + // Always set a credentials callback so git2 doesn't fail with "no callback set". + // The callback will try explicit credentials first, then fall back to git's + // credential helper system (which includes .netrc support). + let mut callbacks = git2::RemoteCallbacks::new(); + let attempt_count = RefCell::new(0); + + callbacks.credentials( + move |url, username_from_url, allowed_types| { + let mut attempts = attempt_count.borrow_mut(); + *attempts += 1; + + // Limit attempts to prevent infinite loops + if *attempts > 1 { + return git2::Cred::default(); + } + + // First, try explicit credentials from netrc if available + if allowed_types + .contains(git2::CredentialType::USER_PASS_PLAINTEXT) + { + if let Some((ref username, ref password)) = credentials + { + // Use both username and password from netrc + return git2::Cred::userpass_plaintext( + username, password, + ); + } + } + + // Try SSH key if available + if allowed_types.contains(git2::CredentialType::SSH_KEY) { + if let Some(username) = username_from_url { + if let Ok(cred) = + git2::Cred::ssh_key_from_agent(username) + { + return Ok(cred); + } + } + } + + // Fall back to git's credential helper system (includes .netrc) + if let Ok(config) = git2::Config::open_default() { + if let Ok(cred) = git2::Cred::credential_helper( + &config, + url, + username_from_url, + ) { + return Ok(cred); + } + } + + git2::Cred::default() + }, + ); + + remote + .connect_auth(git2::Direction::Fetch, Some(callbacks), None) + .map_err(|e| VwError::Git { + message: format!("Failed to connect to remote: {e}"), + })?; + + let refs = remote.list().map_err(|e| VwError::Git { + message: format!("Failed to list remote references: {e}"), + })?; + + // Look for the specific branch reference + let ref_name = format!("refs/heads/{branch}"); + for remote_head in refs { + if remote_head.name() == ref_name { + return Ok(remote_head.oid().to_string()); + } + } + + Err(VwError::Git { + message: format!( + "Branch '{branch}' not found in remote repository" + ), + }) + }), + ) + .await + .map_err(|_| VwError::Git { + message: "Git ls-remote timed out after 30 seconds".to_string(), + })? + .map_err(|e| VwError::Git { + message: format!("Failed to execute git ls-remote task: {e}"), + })? +} + +#[allow(clippy::too_many_arguments)] +async fn download_dependency( + repo_url: &str, + commit: &str, + src_paths: &[String], + dest_path: &Path, + recursive: bool, + exclude: &[String], + submodules: bool, + credentials: Option<(&str, &str)>, // (username, password) + // When `Some`, materialize ONLY this subdirectory of the checkout + // (structure-preserving, all file types) instead of the whole tree + // — lets us pull just `vhdl_libraries/` out of the large rust_hdl + // repo rather than caching its entire source. + subdir: Option<&str>, +) -> Result<()> { + let temp_dir = tempfile::tempdir().map_err(|e| VwError::FileSystem { + message: format!("Failed to create temporary directory: {e}"), + })?; + + // Normalize repository URL to ensure it ends with .git for GitHub + let normalized_repo_url = + if repo_url.contains("github.com") && !repo_url.ends_with(".git") { + format!("{repo_url}.git") + } else { + repo_url.to_string() + }; + + let commit = commit.to_string(); + let temp_path = temp_dir.path().to_path_buf(); + let src_paths = src_paths.to_vec(); + let credentials = credentials.map(|(u, p)| (u.to_string(), p.to_string())); + + tokio::time::timeout( + std::time::Duration::from_secs(120), + tokio::task::spawn_blocking(move || { + // Set up clone options with authentication + let mut builder = git2::build::RepoBuilder::new(); + + // Always set a credentials callback so git2 doesn't fail with "no callback set". + // The callback will try explicit credentials first, then fall back to git's + // credential helper system (which includes .netrc support). + let mut callbacks = git2::RemoteCallbacks::new(); + let attempt_count = RefCell::new(0); + + callbacks.credentials( + move |url, username_from_url, allowed_types| { + let mut attempts = attempt_count.borrow_mut(); + *attempts += 1; + + // Limit attempts to prevent infinite loops + if *attempts > 1 { + return git2::Cred::default(); + } + + // First, try explicit credentials from netrc if available + if allowed_types + .contains(git2::CredentialType::USER_PASS_PLAINTEXT) + { + if let Some((ref username, ref password)) = credentials + { + // Use both username and password from netrc + return git2::Cred::userpass_plaintext( + username, password, + ); + } + } + + // Try SSH key if available + if allowed_types.contains(git2::CredentialType::SSH_KEY) { + if let Some(username) = username_from_url { + if let Ok(cred) = + git2::Cred::ssh_key_from_agent(username) + { + return Ok(cred); + } + } + } + + // Fall back to git's credential helper system (includes .netrc) + if let Ok(config) = git2::Config::open_default() { + if let Ok(cred) = git2::Cred::credential_helper( + &config, + url, + username_from_url, + ) { + return Ok(cred); + } + } + + git2::Cred::default() + }, + ); + + let mut fetch_options = git2::FetchOptions::new(); + fetch_options.depth(1); // shallow clone — only need one commit + fetch_options.remote_callbacks(callbacks); + builder.fetch_options(fetch_options); + + // Clone the repository + let repo = builder + .clone(&normalized_repo_url, &temp_path) + .map_err(|e| VwError::Git { + message: format!("Failed to clone repository: {e}"), + })?; + + // Parse the commit SHA + let commit_oid = + git2::Oid::from_str(&commit).map_err(|e| VwError::Git { + message: format!("Invalid commit SHA '{commit}': {e}"), + })?; + + // Find the commit object + let commit_obj = + repo.find_commit(commit_oid).map_err(|e| VwError::Git { + message: format!("Commit '{commit}' not found: {e}"), + })?; + + // Checkout the specific commit + repo.checkout_tree(commit_obj.as_object(), None) + .map_err(|e| VwError::Git { + message: format!( + "Failed to checkout commit '{commit}': {e}" + ), + })?; + + // Set HEAD to the commit + repo.set_head_detached(commit_oid) + .map_err(|e| VwError::Git { + message: format!( + "Failed to set HEAD to commit '{commit}': {e}" + ), + })?; + + // Initialize and update submodules if requested + if submodules { + for mut submodule in + repo.submodules().map_err(|e| VwError::Git { + message: format!("Failed to list submodules: {e}"), + })? + { + submodule.init(false).map_err(|e| VwError::Git { + message: format!( + "Failed to init submodule '{}': {e}", + submodule.name().unwrap_or("unknown") + ), + })?; + submodule.update(true, None).map_err(|e| VwError::Git { + message: format!( + "Failed to update submodule '{}': {e}", + submodule.name().unwrap_or("unknown") + ), + })?; + } + } + + Ok::<(), VwError>(()) + }), + ) + .await + .map_err(|_| VwError::Git { + message: "Git clone timed out after 120 seconds".to_string(), + })? + .map_err(|e| VwError::Git { + message: format!("Failed to execute git operations: {e}"), + })??; + + fs::create_dir_all(dest_path).map_err(|e| VwError::FileSystem { + message: format!("Failed to create destination directory: {e}"), + })?; + + if let Some(subdir) = subdir { + // Subtree dependency: copy just `/` into + // `/`, preserving structure and every file type. + copy_module_tree( + &temp_dir.path().join(subdir), + &dest_path.join(subdir), + exclude, + )?; + } else if src_paths.is_empty() { + // Htcl module dependency: no VHDL `src` globs are declared, so + // the dep publishes its WHOLE module tree — `module.htcl` plus + // everything it `src`s and any shipped assets (e.g. a + // `vivado-shim.tcl`). Materialize the full checkout into the + // cache, preserving directory structure. Without this the + // VHDL-only copy below matches nothing and leaves the cache + // dir empty, so `src @` can't find `/module.htcl`. + copy_module_tree(temp_dir.path(), dest_path, exclude)?; + } else { + // VHDL source dependency: copy the declared `src` globs, + // filtered to VHDL and flattened relative to each prefix. + for src_path in &src_paths { + copy_vhdl_files_glob( + temp_dir.path(), + src_path, + dest_path, + recursive, + exclude, + )?; + } + } + + Ok(()) +} + +/// Copy an entire checked-out repo tree from `src_root` into `dest`, +/// preserving directory structure. Used for htcl module dependencies +/// (empty `src`), which publish their whole tree rather than a +/// filtered set of VHDL sources — see the call site in +/// [`download_dependency`]. Skips the `.git` metadata dir and honors +/// structure-relative `exclude` globs. +fn copy_module_tree( + src_root: &Path, + dest: &Path, + exclude: &[String], +) -> Result<()> { + let exclude_patterns: Vec = exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + copy_module_tree_impl(src_root, src_root, dest, &exclude_patterns) +} + +fn copy_module_tree_impl( + root: &Path, + dir: &Path, + dest: &Path, + exclude: &[glob::Pattern], +) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory {dir:?}: {e}"), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + // Never materialize VCS metadata (at any depth — submodules + // carry their own `.git` file/dir). + if entry.file_name() == std::ffi::OsStr::new(".git") { + continue; + } + let path = entry.path(); + let rel = path.strip_prefix(root).unwrap_or(&path); + let rel_str = rel.to_string_lossy(); + if exclude.iter().any(|p| p.matches(&rel_str)) { + continue; + } + if path.is_dir() { + copy_module_tree_impl(root, &path, dest, exclude)?; + } else if path.is_file() { + let dest_file = dest.join(rel); + if let Some(parent) = dest_file.parent() { + fs::create_dir_all(parent).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to create directory {parent:?}: {e}" + ), + } + })?; + } + fs::copy(&path, &dest_file).map_err(|e| VwError::FileSystem { + message: format!( + "Failed to copy {path:?} to {dest_file:?}: {e}" + ), + })?; + } + } + Ok(()) +} + +fn copy_vhdl_files_glob( + repo_root: &Path, + src_pattern: &str, + dest: &Path, + recursive: bool, + exclude: &[String], +) -> Result<()> { + // Build patterns to match + let src_path = repo_root.join(src_pattern); + let mut patterns = Vec::new(); + let strip_prefix: PathBuf; + + // Compile exclude patterns + let exclude_patterns: Vec = exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + + // Check if src_pattern points to a directory + if src_path.is_dir() { + // It's a directory - create appropriate glob patterns + let base_pattern = + src_path.to_str().ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in path".to_string(), + })?; + + if recursive { + // Recursively find all VHDL files + patterns.push(format!("{base_pattern}/**/*.vhd")); + patterns.push(format!("{base_pattern}/**/*.vhdl")); + } else { + // Only files directly in the directory + patterns.push(format!("{base_pattern}/*.vhd")); + patterns.push(format!("{base_pattern}/*.vhdl")); + } + // For directories, strip the src directory from paths + strip_prefix = src_path; + } else if src_path.is_file() { + // It's a single file - use as-is + patterns.push( + src_path + .to_str() + .ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in path".to_string(), + })? .to_string(), - }), - (None, None) => Err(VwError::Config { - message: "Must specify either branch or commit for dependency" + ); + // For single files, strip the parent directory + strip_prefix = src_path + .parent() + .ok_or_else(|| VwError::FileSystem { + message: "File has no parent directory".to_string(), + })? + .to_path_buf(); + } else { + // It's a glob pattern or doesn't exist yet - use as-is + patterns.push( + src_path + .to_str() + .ok_or_else(|| VwError::FileSystem { + message: "Invalid UTF-8 in glob pattern path".to_string(), + })? .to_string(), - }), - (None, Some(commit)) => Ok(commit.clone()), - (Some(branch), None) => { - get_branch_head_commit(repo_url, branch, credentials).await + ); + // For glob patterns, strip the repo root to preserve relative structure + strip_prefix = repo_root.to_path_buf(); + } + + let mut copied_count = 0; + for pattern_str in &patterns { + // Use glob to find matching files + let entries = + glob::glob(pattern_str).map_err(|e| VwError::FileSystem { + message: format!("Invalid glob pattern '{pattern_str}': {e}"), + })?; + + for entry in entries { + let path = entry.map_err(|e| VwError::FileSystem { + message: format!("Error reading glob entry: {e}"), + })?; + + // Only copy VHDL files + if path.is_file() { + if let Some(ext) = path.extension() { + if ext == "vhd" || ext == "vhdl" { + // Compute relative path based on strip_prefix + let relative_path = + path.strip_prefix(&strip_prefix).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to compute relative path for {path:?}: {e}" + ), + } + })?; + + // Check if file matches any exclude pattern + let path_str = relative_path.to_string_lossy(); + if exclude_patterns.iter().any(|p| p.matches(&path_str)) + { + continue; // Skip excluded files + } + + let dest_file = dest.join(relative_path); + + // Create parent directories if needed + if let Some(parent) = dest_file.parent() { + fs::create_dir_all(parent).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to create directory {parent:?}: {e}" + ), + } + })?; + } + + fs::copy(&path, &dest_file).map_err(|e| { + VwError::FileSystem { + message: format!( + "Failed to copy file {path:?}: {e}" + ), + } + })?; + copied_count += 1; + } + } + } + } + } + + if copied_count == 0 { + return Err(VwError::Dependency { + message: format!("No VHDL files matched pattern '{src_pattern}'"), + }); + } + + Ok(()) +} + +fn find_vhdl_files( + dir: &Path, + recursive: bool, + exclude: &[String], +) -> Result> { + let mut vhdl_files = Vec::new(); + find_vhdl_files_impl(dir, &mut vhdl_files, recursive)?; + + // Filter out excluded files + if !exclude.is_empty() { + let exclude_patterns: Vec = exclude + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + + vhdl_files.retain(|file| { + // Match against path relative to the base directory + let relative = file.strip_prefix(dir).unwrap_or(file); + let path_str = relative.to_string_lossy(); + !exclude_patterns + .iter() + .any(|pattern| pattern.matches(&path_str)) + }); + } + + Ok(vhdl_files) +} + +fn find_vhdl_files_impl( + dir: &Path, + vhdl_files: &mut Vec, + recursive: bool, +) -> Result<()> { + for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory: {e}"), + })? { + let entry = entry.map_err(|e| VwError::FileSystem { + message: format!("Failed to read directory entry: {e}"), + })?; + let path = entry.path(); + + if path.is_dir() { + if recursive { + find_vhdl_files_impl(&path, vhdl_files, recursive)?; + } + } else if let Some(extension) = + path.extension().and_then(|ext| ext.to_str()) + { + if extension == "vhd" || extension == "vhdl" { + vhdl_files.push(path); + } + } + } + Ok(()) +} + +fn write_lock_file( + workspace_dir: &Utf8Path, + lock_file: &LockFile, +) -> Result<()> { + let toml_content = toml::to_string_pretty(lock_file)?; + let lock_path = workspace_dir.join("vw.lock"); + + fs::write(&lock_path, toml_content).map_err(|e| VwError::FileSystem { + message: format!("Failed to write vw.lock file: {e}"), + })?; + + Ok(()) +} + +/// Build a Rust library for a testbench. +/// Looks for Cargo.toml in the testbench directory, builds it, and returns the path to the .so file. +async fn build_rust_library( + bench_dir: &Utf8Path, + testbench_file: &Path, +) -> Result { + // Get the testbench directory + let testbench_dir = + testbench_file.parent().ok_or_else(|| VwError::Testbench { + message: format!( + "Testbench file {:?} has no parent directory???", + testbench_file + ), + })?; + + // Look for Cargo.toml in the testbench directory + let cargo_toml_path = testbench_dir.join("Cargo.toml"); + if !cargo_toml_path.exists() { + return Err(VwError::Testbench { + message: format!( + "Cargo.toml not found in testbench directory: {:?}", + testbench_dir + ), + }); + } + + // Parse Cargo.toml to get the package name + let cargo_toml_content = + fs::read_to_string(&cargo_toml_path).map_err(|e| { + VwError::FileSystem { + message: format!("Failed to read Cargo.toml: {e}"), + } + })?; + + let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?; + let package_name = cargo_toml.package.name; + + // Run cargo build in the testbench directory + let testbench_dir_owned = testbench_dir.to_path_buf(); + tokio::task::spawn_blocking(move || { + let output = std::process::Command::new("cargo") + .arg("build") + .current_dir(&testbench_dir_owned) + .output() + .map_err(|e| VwError::Testbench { + message: format!("Failed to execute cargo build: {e}"), + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(VwError::Testbench { + message: format!("cargo build failed:\n{stderr}"), + }); } + + Ok::<(), VwError>(()) + }) + .await + .map_err(|e| VwError::Testbench { + message: format!("Failed to execute cargo build task: {e}"), + })??; + + // Find the .so file in the workspace target directory (parent of testbench dir) + let ext = if cfg!(target_os = "macos") { + "dylib" + } else { + "so" + }; + let lib_name = format!("lib{}.{ext}", package_name.replace('-', "_")); + let workspace_target = bench_dir.join("target").join("debug"); + + let lib_path = workspace_target.join(&lib_name); + + if !lib_path.exists() { + return Err(VwError::Testbench { + message: format!( + "Built Rust library not found at expected path: {:?}", + lib_path + ), + }); } + + Ok(lib_path.into()) } -async fn get_branch_head_commit( - repo_url: &str, - branch: &str, - credentials: Option<(&str, &str)>, // (username, password) -) -> Result { - // Normalize repository URL to ensure it ends with .git for GitHub - let normalized_repo_url = - if repo_url.contains("github.com") && !repo_url.ends_with(".git") { - format!("{repo_url}.git") - } else { - repo_url.to_string() +#[cfg(test)] +mod dependency_source_tests { + use super::*; + + #[test] + fn manifest_path_dep_wins_over_stale_git_lock_entry() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8Path::from_path(tmp.path()).unwrap(); + fs::create_dir_all(ws.join("local-foo")).unwrap(); + fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"t\"\nversion = \"0.1.0\"\n\ + [dependencies.foo]\npath = \"local-foo\"\n", + ) + .unwrap(); + // Stale git pin for `foo`, left over from before the + // `repo → path` switch. + fs::write( + ws.join("vw.lock"), + "[dependencies.foo]\n\ + repo = \"https://example.com/foo.git\"\n\ + commit = \"deadbeef\"\n\ + path = \"foo-deadbeef\"\n", + ) + .unwrap(); + let paths = dep_cache_paths_with_test(ws, false).unwrap(); + let foo = paths.get("foo").expect("foo resolves"); + assert!( + foo.ends_with("local-foo"), + "manifest path dep must win over a stale git lock entry, \ + got {foo:?}" + ); + } + + #[test] + fn prune_drops_now_path_dep_but_keeps_git_deps() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8Path::from_path(tmp.path()).unwrap(); + fs::create_dir_all(ws.join("local-foo")).unwrap(); + fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"t\"\nversion = \"0.1.0\"\n\ + [dependencies.foo]\npath = \"local-foo\"\n\ + [dependencies.bar]\n\ + repo = \"https://example.com/bar.git\"\nbranch = \"main\"\n", + ) + .unwrap(); + fs::write( + ws.join("vw.lock"), + "[dependencies.foo]\n\ + repo = \"https://example.com/foo.git\"\n\ + commit = \"dead\"\npath = \"foo-dead\"\n\ + [dependencies.bar]\n\ + repo = \"https://example.com/bar.git\"\n\ + commit = \"beef\"\npath = \"bar-beef\"\n", + ) + .unwrap(); + // `foo` is now a path dep → its stale git entry is pruned; + // `bar` (still git) is left untouched. + assert!(prune_stale_path_deps_from_lock(ws).unwrap()); + let lock = load_lock_file(ws).unwrap(); + assert!(!lock.dependencies.contains_key("foo"), "foo pruned"); + assert!(lock.dependencies.contains_key("bar"), "bar kept"); + // Idempotent: nothing left to prune on a second pass. + assert!(!prune_stale_path_deps_from_lock(ws).unwrap()); + } + + #[test] + fn vho_template_rewrites_to_black_box_entity() { + // A realistic Vivado VHDL instantiation template: banner + // comments (one of which literally says "COMPONENT"), the + // component declaration, then the instantiation example. + let vho = "\ +-- (c) Copyright 1995-2024 AMD, Inc. All rights reserved.\n\ +-- The following code must appear in the VHDL architecture header:\n\ +------------- Begin Cut here for COMPONENT Declaration ------ COMP_TAG\n\ +component primary_clock\n\ +port (\n\ + clk_out1 : out std_logic;\n\ + locked : out std_logic;\n\ + clk_in1 : in std_logic\n\ +);\n\ +end component;\n\ +-- COMP_TAG_END ------ End COMPONENT Declaration ------------\n\ +-- The following code must appear in the VHDL architecture body:\n\ +-------------- Begin Cut here for INSTANTIATION Template ----- INST_TAG\n\ +your_instance_name : primary_clock\n\ + port map (\n\ + clk_out1 => clk_out1,\n\ + locked => locked,\n\ + clk_in1 => clk_in1\n\ + );\n\ +-- INST_TAG_END ------ End INSTANTIATION Template ---------\n"; + let stub = vho_component_to_entity(vho).expect("component found"); + assert!(stub.contains("entity primary_clock is"), "stub:\n{stub}"); + assert!(stub.contains("clk_out1 : out std_logic"), "ports copied"); + assert!(stub.contains("clk_in1 : in std_logic"), "ports copied"); + assert!(stub.contains("end entity;")); + assert!(stub.contains("architecture stub of primary_clock is")); + // The instantiation example (a `component`-free section) must + // not leak in, and we must not have matched the banner comment. + assert!( + !stub.contains("your_instance_name"), + "instantiation template leaked into the stub" + ); + assert!(!stub.contains("component"), "no component syntax remains"); + } + + #[test] + fn vho_without_component_returns_none() { + assert!(vho_component_to_entity("-- just a comment\nfoo\n").is_none()); + } + + /// A `vw.toml` entry with `repo = "..."` parses as a git source — + /// the historical behaviour that pre-dates path deps. + #[test] + fn git_dep_parses_from_repo_key() { + let toml = r#" + [workspace] + name = "demo" + version = "0.1.0" + + [dependencies.quartz] + repo = "https://github.com/oxidecomputer/quartz" + branch = "main" + src = ["hdl/ip/vhd"] + recursive = true + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let dep = &config.dependencies["quartz"]; + assert!(!dep.is_local()); + assert_eq!(dep.repo(), Some("https://github.com/oxidecomputer/quartz")); + assert_eq!(dep.branch(), Some("main")); + assert!(dep.recursive); + assert_eq!(dep.src, vec!["hdl/ip/vhd".to_string()]); + } + + /// The metroid layout: `path = "..."` and nothing else. + #[test] + fn path_dep_parses_from_path_key() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [dependencies.amd-htcl] + path = "/home/ry/src/amd-htcl" + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let dep = &config.dependencies["amd-htcl"]; + assert!(dep.is_local()); + assert_eq!(dep.local_path(), Some(Path::new("/home/ry/src/amd-htcl"))); + assert_eq!(dep.repo(), None); + assert_eq!(dep.branch(), None); + } + + #[test] + fn transitive_dep_resolution_pulls_in_lib_of_lib() { + // metroid → cips → vivado-cmd. Asking for metroid's deps + // transitively should return cips AND vivado-cmd, even though + // metroid only declares cips. + let dir = tempfile::tempdir().unwrap(); + let metroid = dir.path().join("metroid"); + let cips = dir.path().join("cips"); + let vivado_cmd = dir.path().join("vivado-cmd"); + std::fs::create_dir_all(&metroid).unwrap(); + std::fs::create_dir_all(&cips).unwrap(); + std::fs::create_dir_all(&vivado_cmd).unwrap(); + std::fs::write( + metroid.join("vw.toml"), + format!( + "[workspace]\nname=\"metroid\"\nversion=\"0.1.0\"\n\n\ + [dependencies.cips]\npath = \"{}\"\n", + cips.display() + ), + ) + .unwrap(); + std::fs::write( + cips.join("vw.toml"), + format!( + "[workspace]\nname=\"cips\"\nversion=\"0.1.0\"\n\n\ + [dependencies.vivado-cmd]\npath = \"{}\"\n", + vivado_cmd.display() + ), + ) + .unwrap(); + // vivado-cmd is a leaf — has a vw.toml but no deps of its own. + std::fs::write( + vivado_cmd.join("vw.toml"), + "[workspace]\nname=\"vivado-cmd\"\nversion=\"0.1.0\"\n", + ) + .unwrap(); + + let metroid_utf8 = Utf8PathBuf::from_path_buf(metroid.clone()).unwrap(); + let resolved = transitive_dep_cache_paths(&metroid_utf8).unwrap(); + assert_eq!(resolved.get("cips"), Some(&cips)); + assert_eq!(resolved.get("vivado-cmd"), Some(&vivado_cmd)); + assert_eq!(resolved.len(), 2, "{resolved:?}"); + } + + #[test] + fn transitive_dep_resolution_first_seen_wins() { + // entry → A and entry → B, both A and B declare a dep + // `shared` pointing at different paths. Entry's view of + // `shared` is whichever was inserted first; entry itself + // doesn't declare `shared`, so the test just asserts we got + // *one* deterministic answer rather than a panic / duplicate. + let dir = tempfile::tempdir().unwrap(); + let entry = dir.path().join("entry"); + let a = dir.path().join("a"); + let b = dir.path().join("b"); + let shared_v1 = dir.path().join("shared-v1"); + let shared_v2 = dir.path().join("shared-v2"); + for d in [&entry, &a, &b, &shared_v1, &shared_v2] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write( + entry.join("vw.toml"), + format!( + "[workspace]\nname=\"entry\"\nversion=\"0.1.0\"\n\n\ + [dependencies.a]\npath = \"{}\"\n\ + [dependencies.b]\npath = \"{}\"\n", + a.display(), + b.display() + ), + ) + .unwrap(); + std::fs::write( + a.join("vw.toml"), + format!( + "[workspace]\nname=\"a\"\nversion=\"0.1.0\"\n\n\ + [dependencies.shared]\npath = \"{}\"\n", + shared_v1.display() + ), + ) + .unwrap(); + std::fs::write( + b.join("vw.toml"), + format!( + "[workspace]\nname=\"b\"\nversion=\"0.1.0\"\n\n\ + [dependencies.shared]\npath = \"{}\"\n", + shared_v2.display() + ), + ) + .unwrap(); + + let entry_utf8 = Utf8PathBuf::from_path_buf(entry).unwrap(); + let resolved = transitive_dep_cache_paths(&entry_utf8).unwrap(); + // `shared` is present exactly once and points at one of the + // two candidates; we don't pin which (HashMap iter order). + let shared = resolved.get("shared").unwrap(); + assert!(*shared == shared_v1 || *shared == shared_v2, "{shared:?}"); + } + + /// Local deps round-trip through serialize/deserialize. + #[test] + fn path_dep_roundtrips() { + let dep = Dependency { + source: DependencySource::Path { + path: PathBuf::from("/some/where"), + }, + src: Vec::new(), + recursive: false, + sim_only: false, + exclude: Vec::new(), }; + let serialized = toml::to_string(&dep).unwrap(); + let deserialized: Dependency = toml::from_str(&serialized).unwrap(); + assert!(deserialized.is_local()); + assert_eq!(deserialized.local_path(), Some(Path::new("/some/where"))); + } - let branch = branch.to_string(); - let credentials = credentials.map(|(u, p)| (u.to_string(), p.to_string())); + #[test] + fn test_dependencies_parse_from_test_dependencies_section() { + let toml = r#" + [workspace] + name = "demo" + version = "0.1.0" + + [dependencies.vivado-cmd] + path = "/home/x/vivado-cmd" + + [test-dependencies.test] + path = "/home/x/test" + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(config.dependencies.len(), 1); + assert_eq!(config.test_dependencies.len(), 1); + assert!(config.test_dependencies["test"].is_local()); + } - tokio::time::timeout( - std::time::Duration::from_secs(30), - tokio::task::spawn_blocking(move || { - // Create a temporary directory for the operation - let temp_dir = - tempfile::tempdir().map_err(|e| VwError::FileSystem { - message: format!( - "Failed to create temporary directory: {e}" - ), - })?; + #[test] + fn list_htcl_tests_walks_test_directory() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + std::fs::write(ws.join("vw.toml"), "").unwrap(); + std::fs::create_dir_all(ws.join("test/nested")).unwrap(); + std::fs::create_dir_all(ws.join("test/.hidden")).unwrap(); + std::fs::create_dir_all(ws.join("test/target")).unwrap(); + std::fs::write(ws.join("test/a.htcl"), "").unwrap(); + std::fs::write(ws.join("test/b.htcl"), "").unwrap(); + std::fs::write(ws.join("test/skip.vhd"), "").unwrap(); + std::fs::write(ws.join("test/nested/c.htcl"), "").unwrap(); + std::fs::write(ws.join("test/.hidden/z.htcl"), "").unwrap(); + std::fs::write(ws.join("test/target/z.htcl"), "").unwrap(); + let tests = list_htcl_tests(&ws).unwrap(); + assert_eq!(tests.len(), 3, "{:?}", tests); + assert!(tests[0].ends_with("test/a.htcl")); + assert!(tests[1].ends_with("test/b.htcl")); + assert!(tests[2].ends_with("test/nested/c.htcl")); + } - // Create an empty repository to work with remotes - let repo = - git2::Repository::init_bare(temp_dir.path()).map_err(|e| { - VwError::Git { - message: format!( - "Failed to initialize temporary repository: {e}" - ), - } - })?; + #[test] + fn list_htcl_tests_returns_empty_when_test_dir_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + std::fs::write(ws.join("vw.toml"), "").unwrap(); + let tests = list_htcl_tests(&ws).unwrap(); + assert!(tests.is_empty()); + } - // Create a remote - let mut remote = repo - .remote_anonymous(&normalized_repo_url) - .map_err(|e| VwError::Git { - message: format!("Failed to create remote: {e}"), - })?; + #[test] + fn target_pattern_parses_brace_form() { + let p = parse_target_pattern("versal{xcvm3(.*)}").unwrap(); + assert_eq!(p.family, "versal"); + assert!(p.regex.is_match("xcvm3358-vsvh1747-2M-e-S")); + assert!(!p.regex.is_match("xc7z020clg484-1")); + } - // Connect and list references - // Always set a credentials callback so git2 doesn't fail with "no callback set". - // The callback will try explicit credentials first, then fall back to git's - // credential helper system (which includes .netrc support). - let mut callbacks = git2::RemoteCallbacks::new(); - let attempt_count = RefCell::new(0); + #[test] + fn target_pattern_rejects_bare_family() { + // Bare `artix7` (no braces) shouldn't reach downstream vw. + // toml; `vw ip generate` normalizes into brace form. + let e = parse_target_pattern("artix7").unwrap_err(); + assert!(matches!(e, TargetParseError::MissingBraces { .. })); + } - callbacks.credentials( - move |url, username_from_url, allowed_types| { - let mut attempts = attempt_count.borrow_mut(); - *attempts += 1; + #[test] + fn target_pattern_anchors_at_start() { + // `xcvm3(.*)` should NOT match "xxx-xcvm3358" (regex would + // otherwise be "contains xcvm3"). Anchoring at start + // prevents that. + let p = parse_target_pattern("versal{xcvm3(.*)}").unwrap(); + assert!(p.regex.is_match("xcvm3358")); + assert!(!p.regex.is_match("blah-xcvm3358")); + } - // Limit attempts to prevent infinite loops - if *attempts > 1 { - return git2::Cred::default(); - } + #[test] + fn workspace_config_parses_target_part_and_targets() { + let toml = r#" + [workspace] + name = "clk-wizard" + version = "0.1.0" + + [targets] + supported = [ + "versal{xcvm3(.*)}", + "versal{xc2ve3(.*)}", + ] + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.name, "clk-wizard"); + assert!(cfg.workspace.target_parts.is_empty()); + let t = cfg.targets.expect("expected [targets]"); + assert_eq!(t.supported.len(), 2); + } - // First, try explicit credentials from netrc if available - if allowed_types - .contains(git2::CredentialType::USER_PASS_PLAINTEXT) - { - if let Some((ref username, ref password)) = credentials - { - // Use both username and password from netrc - return git2::Cred::userpass_plaintext( - username, password, - ); - } - } + #[test] + fn workspace_config_parses_multi_target_parts() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.target_parts.len(), 2); + assert_eq!( + cfg.workspace.default_target_part().unwrap(), + Some("xcvp1202-vsva2785-2MHP-e-S"), + ); + // Substring selector picks the non-default. + assert_eq!( + cfg.workspace.select_target_part(Some("3HP")).unwrap(), + Some("xcvp1202-vsva2785-3HP-e-S"), + ); + } - // Try SSH key if available - if allowed_types.contains(git2::CredentialType::SSH_KEY) { - if let Some(username) = username_from_url { - if let Ok(cred) = - git2::Cred::ssh_key_from_agent(username) - { - return Ok(cred); - } - } - } + #[test] + fn multi_parts_without_default_flag_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(matches!( + cfg.workspace.default_target_part(), + Err(TargetSelectError::NoDefault { count: 2 }), + )); + } - // Fall back to git's credential helper system (includes .netrc) - if let Ok(config) = git2::Config::open_default() { - if let Ok(cred) = git2::Cred::credential_helper( - &config, - url, - username_from_url, - ) { - return Ok(cred); - } - } + #[test] + fn ambiguous_substring_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + // "xcvp1202" matches both entries. + assert!(matches!( + cfg.workspace.select_target_part(Some("xcvp1202")), + Err(TargetSelectError::Ambiguous { .. }), + )); + } - git2::Cred::default() - }, - ); + #[test] + fn single_part_is_implicit_default() { + let toml = r#" + [workspace] + name = "vw" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!( + cfg.workspace.default_target_part().unwrap(), + Some("xcvp1202-vsva2785-3HP-e-S"), + ); + } - remote - .connect_auth(git2::Direction::Fetch, Some(callbacks), None) - .map_err(|e| VwError::Git { - message: format!("Failed to connect to remote: {e}"), - })?; + #[test] + fn workspace_config_parses_variants_block() { + let toml = r#" + [workspace] + name = "metroid" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + exclusive = ["hdl/ethernet-vpk120.vhd"] + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + exclusive = ["hdl/ethernet-metro.vhd"] + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!(cfg.workspace.variants.len(), 2); + assert_eq!(cfg.workspace.variants[0].name, "vpk120"); + assert_eq!( + cfg.workspace.variants[0].part, + "xcvp1202-vsva2785-2MHP-e-S" + ); + assert!(cfg.workspace.variants[0].default); + assert_eq!( + cfg.workspace.variants[0].exclusive, + vec!["hdl/ethernet-vpk120.vhd"], + ); + assert_eq!(cfg.workspace.variants[1].name, "metro"); + assert!(!cfg.workspace.variants[1].default); + // Empty target_parts — variants own their parts inline. + assert!(cfg.workspace.target_parts.is_empty()); + } - let refs = remote.list().map_err(|e| VwError::Git { - message: format!("Failed to list remote references: {e}"), - })?; + #[test] + fn variants_and_target_parts_are_mutually_exclusive() { + // Deserialization allows both (unknown-field serde is + // lenient), but `load_workspace_config` refuses to + // return a config that has both — variants own parts. + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.target-parts]] + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "v" + part = "xcvp1202-vsva2785-2MHP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let err = validate_variant_shape(&cfg.workspace).unwrap_err(); + assert!( + err.to_string().contains("mutually exclusive"), + "expected mutual-exclusion error: {err}", + ); + } - // Look for the specific branch reference - let ref_name = format!("refs/heads/{branch}"); - for remote_head in refs { - if remote_head.name() == ref_name { - return Ok(remote_head.oid().to_string()); - } - } + #[test] + fn duplicate_variant_names_error() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let err = validate_variant_shape(&cfg.workspace).unwrap_err(); + assert!( + err.to_string().contains("duplicate variant name"), + "expected duplicate-name error: {err}", + ); + } - Err(VwError::Git { - message: format!( - "Branch '{branch}' not found in remote repository" - ), - }) - }), - ) - .await - .map_err(|_| VwError::Git { - message: "Git ls-remote timed out after 30 seconds".to_string(), - })? - .map_err(|e| VwError::Git { - message: format!("Failed to execute git ls-remote task: {e}"), - })? -} + #[test] + fn default_variant_no_variants_yields_none() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(cfg.workspace.default_variant().unwrap().is_none()); + } -#[allow(clippy::too_many_arguments)] -async fn download_dependency( - repo_url: &str, - commit: &str, - src_paths: &[String], - dest_path: &Path, - recursive: bool, - exclude: &[String], - submodules: bool, - credentials: Option<(&str, &str)>, // (username, password) -) -> Result<()> { - let temp_dir = tempfile::tempdir().map_err(|e| VwError::FileSystem { - message: format!("Failed to create temporary directory: {e}"), - })?; + #[test] + fn default_variant_multi_with_default_flag() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + let v = cfg.workspace.default_variant().unwrap().unwrap(); + assert_eq!(v.name, "vpk120"); + } - // Normalize repository URL to ensure it ends with .git for GitHub - let normalized_repo_url = - if repo_url.contains("github.com") && !repo_url.ends_with(".git") { - format!("{repo_url}.git") - } else { - repo_url.to_string() - }; + #[test] + fn default_variant_multi_without_default_errors() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(matches!( + cfg.workspace.default_variant(), + Err(VariantSelectError::NoDefault { count: 2 }), + )); + } - let commit = commit.to_string(); - let temp_path = temp_dir.path().to_path_buf(); - let src_paths = src_paths.to_vec(); - let credentials = credentials.map(|(u, p)| (u.to_string(), p.to_string())); + #[test] + fn select_variant_exact_match_only() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + // Exact match returns the entry. + assert_eq!( + cfg.workspace + .select_variant(Some("metro")) + .unwrap() + .unwrap() + .name, + "metro", + ); + // Substring is NOT accepted — variant names are the + // whole selector. + assert!(matches!( + cfg.workspace.select_variant(Some("vpk")), + Err(VariantSelectError::NoMatch { .. }), + )); + } - tokio::time::timeout( - std::time::Duration::from_secs(120), - tokio::task::spawn_blocking(move || { - // Set up clone options with authentication - let mut builder = git2::build::RepoBuilder::new(); + #[test] + fn resolve_top_variant_overrides_workspace() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + top = "workspace_default_top" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + default = true + top = "top_vpk120" + + [[workspace.variants]] + name = "metro" + part = "xcvp1202-vsva2785-3HP-e-S" + # no per-variant top → falls back to workspace top + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert_eq!( + cfg.workspace.resolve_top(Some("vpk120")).as_deref(), + Some("top_vpk120"), + ); + assert_eq!( + cfg.workspace.resolve_top(Some("metro")).as_deref(), + Some("workspace_default_top"), + ); + assert_eq!( + cfg.workspace.resolve_top(None).as_deref(), + Some("workspace_default_top"), + ); + } - // Always set a credentials callback so git2 doesn't fail with "no callback set". - // The callback will try explicit credentials first, then fall back to git's - // credential helper system (which includes .netrc support). - let mut callbacks = git2::RemoteCallbacks::new(); - let attempt_count = RefCell::new(0); + #[test] + fn resolve_top_none_when_unset_everywhere() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(cfg.workspace.resolve_top(Some("vpk120")).is_none()); + assert!(cfg.workspace.resolve_top(None).is_none()); + } - callbacks.credentials( - move |url, username_from_url, allowed_types| { - let mut attempts = attempt_count.borrow_mut(); - *attempts += 1; + #[test] + fn resolve_top_unknown_variant_falls_back_to_workspace() { + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + top = "workspace_top" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + top = "top_vpk120" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + // A variant name not in the list falls through to workspace-level. + // (In practice select_variant would error before this, but the + // resolver must not panic on unknown names.) + assert_eq!( + cfg.workspace.resolve_top(Some("ghost")).as_deref(), + Some("workspace_top"), + ); + } + + #[test] + fn single_variant_no_default_flag_still_parses() { + // A single-variant list is legal without `default = true` + // (the default becomes implicit — same rule as + // `[[target-parts]]` single-entry). + let toml = r#" + [workspace] + name = "x" + version = "0.1.0" + + [[workspace.variants]] + name = "vpk120" + part = "xcvp1202-vsva2785-2MHP-e-S" + "#; + let cfg: WorkspaceConfig = toml::from_str(toml).unwrap(); + assert!(validate_variant_shape(&cfg.workspace).is_ok()); + assert_eq!(cfg.workspace.variants.len(), 1); + } + + fn make_dep(name: &str, patterns: &[&str]) -> (String, Vec) { + let compiled: Vec = patterns + .iter() + .map(|s| parse_target_pattern(s).unwrap()) + .collect(); + (name.to_string(), compiled) + } + + #[test] + fn target_compat_matches_when_pattern_covers_part() { + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = + check_target_compatibility(Some("xcvm3358-vsvh1747-2M-e-S"), &dt); + assert!(mismatches.is_empty(), "{mismatches:?}"); + } - // Limit attempts to prevent infinite loops - if *attempts > 1 { - return git2::Cred::default(); - } + #[test] + fn target_compat_reports_unblessed_when_no_pattern_matches() { + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = + check_target_compatibility(Some("xc7z020clg484-1"), &dt); + assert_eq!(mismatches.len(), 1); + assert_eq!(mismatches[0].dep, "clk-wizard"); + assert_eq!(mismatches[0].supported_families, vec!["versal"]); + assert_eq!(mismatches[0].kind, TargetMismatchKind::Unblessed); + } - // First, try explicit credentials from netrc if available - if allowed_types - .contains(git2::CredentialType::USER_PASS_PLAINTEXT) - { - if let Some((ref username, ref password)) = credentials - { - // Use both username and password from netrc - return git2::Cred::userpass_plaintext( - username, password, - ); - } - } + #[test] + fn target_compat_reports_not_supported_wins_over_supported() { + // If a target matches both a supported and a not-supported + // pattern, the explicit ban must win — Xilinx has attested + // the combination doesn't work. + let mut dt = DepTargets::default(); + let (name, sup) = make_dep("gadget", &["versal{xcv(.*)}"]); + dt.per_dep.insert(name.clone(), sup); + let (_, ns) = make_dep("gadget", &["versal{xcvp1202.*}"]); + dt.per_dep_not_supported.insert(name, ns); + let mismatches = + check_target_compatibility(Some("xcvp1202-vsva2785-3HP-e-S"), &dt); + assert_eq!(mismatches.len(), 1); + assert_eq!(mismatches[0].kind, TargetMismatchKind::NotSupported); + } - // Try SSH key if available - if allowed_types.contains(git2::CredentialType::SSH_KEY) { - if let Some(username) = username_from_url { - if let Ok(cred) = - git2::Cred::ssh_key_from_agent(username) - { - return Ok(cred); - } - } - } + #[test] + fn target_compat_supported_match_clears_when_no_ban() { + let mut dt = DepTargets::default(); + let (name, sup) = make_dep("gadget", &["versal{xcvp1202.*}"]); + dt.per_dep.insert(name, sup); + let mismatches = + check_target_compatibility(Some("xcvp1202-vsva2785-3HP-e-S"), &dt); + assert!(mismatches.is_empty(), "{mismatches:?}"); + } - // Fall back to git's credential helper system (includes .netrc) - if let Ok(config) = git2::Config::open_default() { - if let Ok(cred) = git2::Cred::credential_helper( - &config, - url, - username_from_url, - ) { - return Ok(cred); - } - } + #[test] + fn target_compat_treats_empty_patterns_as_universal() { + // @vw / @test — no [targets] means "we support anything." + let mut dt = DepTargets::default(); + dt.per_dep.insert("vw".into(), Vec::new()); + let mismatches = + check_target_compatibility(Some("xc7z020clg484-1"), &dt); + assert!(mismatches.is_empty()); + } - git2::Cred::default() - }, - ); + #[test] + fn target_compat_no_op_when_no_target_declared() { + // Library workspaces have no target-part — check should + // silently accept. + let mut dt = DepTargets::default(); + let (name, patterns) = make_dep("clk-wizard", &["versal{xcvm3(.*)}"]); + dt.per_dep.insert(name, patterns); + let mismatches = check_target_compatibility(None, &dt); + assert!(mismatches.is_empty()); + } - let mut fetch_options = git2::FetchOptions::new(); - fetch_options.depth(1); // shallow clone — only need one commit - fetch_options.remote_callbacks(callbacks); - builder.fetch_options(fetch_options); + #[test] + fn library_name_hyphens_become_underscores() { + assert_eq!(library_name_for_dep("clk-wizard"), "clk_wizard"); + assert_eq!(library_name_for_dep("gtwiz-versal"), "gtwiz_versal"); + assert_eq!(library_name_for_dep("cpm5"), "cpm5"); + } - // Clone the repository - let repo = builder - .clone(&normalized_repo_url, &temp_path) - .map_err(|e| VwError::Git { - message: format!("Failed to clone repository: {e}"), - })?; + #[test] + fn vhdl_design_sources_empty_when_no_hdl_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + // No `hdl/` yet — return empty, not error. + let sources = vhdl_design_sources(&ws).unwrap(); + assert!(sources.is_empty()); + } - // Parse the commit SHA - let commit_oid = - git2::Oid::from_str(&commit).map_err(|e| VwError::Git { - message: format!("Invalid commit SHA '{commit}': {e}"), - })?; + #[test] + fn vhdl_dependency_sources_skips_deps_without_src() { + // Regression guard: a path dep with no `src` field is + // htcl-only and shouldn't contribute VHDL. Previously the + // enumeration walked each dep's whole tree recursively, + // scooping up e.g. `target/ip/*/wrapper.vhd` from an htcl + // library's own generated artifacts. + let tmp = tempfile::tempdir().unwrap(); + // Fake htcl-only dep: has a stray .vhd (like a generated + // wrapper) but declares no `src`. + let htcl_dep = tmp.path().join("htcl-only-dep"); + std::fs::create_dir_all(htcl_dep.join("target/ip/foo")).unwrap(); + std::fs::write( + htcl_dep.join("target/ip/foo/wrapper.vhd"), + "-- generated", + ) + .unwrap(); + // Fake VHDL dep: declares `src = ["hdl"]`. + let vhdl_dep = tmp.path().join("vhdl-dep"); + std::fs::create_dir_all(vhdl_dep.join("hdl")).unwrap(); + std::fs::write(vhdl_dep.join("hdl/mod.vhd"), "-- source").unwrap(); + + // Entry workspace vw.toml referencing both. + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.htcl-only-dep] +path = "{}" + +[dependencies.vhdl-dep] +path = "{}" +src = ["hdl"] +recursive = true +"#, + htcl_dep.display(), + vhdl_dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + // Only the vhdl-dep contributes. htcl-only-dep is skipped + // even though its tree contains a .vhd file. + assert_eq!(sources.len(), 1, "{sources:?}"); + assert_eq!(sources[0].library, "vhdl_dep"); + assert!( + sources[0].path.ends_with("hdl/mod.vhd"), + "unexpected path {}", + sources[0].path.display(), + ); + } - // Find the commit object - let commit_obj = - repo.find_commit(commit_oid).map_err(|e| VwError::Git { - message: format!("Commit '{commit}' not found: {e}"), - })?; + #[test] + fn vhdl_dependency_sources_git_cache_is_flattened() { + // Regression: git-dep caches under `~/.vw/deps/-/` + // are FLATTENED at copy time — `copy_vhdl_files_glob` strips + // the source repo's `hdl/ip/vhd/synchronizers/` prefix off, + // so files land directly at the cache root. Enumeration + // must NOT re-apply the `src` pattern as a subdir join + // (which would find nothing) — instead it walks the cache + // root recursively. + let tmp = tempfile::tempdir().unwrap(); + // Simulated cache — flat file layout. + let cache = tmp.path().join("cache/quartz_sync-abc"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(cache.join("meta_sync.vhd"), "").unwrap(); + std::fs::write(cache.join("bacd.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.quartz_sync] +repo = "https://example.invalid/quartz" +branch = "main" +src = ["hdl/ip/vhd/synchronizers"] +"#, + ) + .unwrap(); + std::fs::write( + ws.join("vw.lock"), + format!( + r#" +[dependencies.quartz_sync] +repo = "https://example.invalid/quartz" +commit = "abc" +path = "{}" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + cache.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + // Both files show up despite `src` pointing at a + // subdirectory that doesn't exist in the flat cache. + assert_eq!(sources.len(), 2, "{sources:?}"); + assert!(sources.iter().all(|s| s.library == "quartz_sync")); + } - // Checkout the specific commit - repo.checkout_tree(commit_obj.as_object(), None) - .map_err(|e| VwError::Git { - message: format!( - "Failed to checkout commit '{commit}': {e}" - ), - })?; + #[test] + fn vhdl_dependency_sources_finds_git_dep_via_lockfile() { + // Simulates the real workflow: a git dep declared in + // vw.toml, resolved to a cache dir via vw.lock. The + // lockfile's `path` is absolute so we don't need to + // override `VW_DEPS_DIR`. + let tmp = tempfile::tempdir().unwrap(); + let cache = tmp.path().join("cache/quartz_sync-abc123"); + std::fs::create_dir_all(cache.join("hdl/ip/vhd/synchronizers")) + .unwrap(); + std::fs::write( + cache.join("hdl/ip/vhd/synchronizers/sync.vhd"), + "-- synced", + ) + .unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.quartz-sync] +repo = "https://example.invalid/quartz" +branch = "main" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +"#, + ) + .unwrap(); + std::fs::write( + ws.join("vw.lock"), + format!( + r#" +[dependencies.quartz-sync] +repo = "https://example.invalid/quartz" +commit = "abc123" +path = "{}" +src = ["hdl/ip/vhd/synchronizers"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + cache.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert_eq!(sources[0].library, "quartz_sync"); + assert!(sources[0].path.ends_with("sync.vhd")); + } - // Set HEAD to the commit - repo.set_head_detached(commit_oid) - .map_err(|e| VwError::Git { - message: format!( - "Failed to set HEAD to commit '{commit}': {e}" - ), - })?; + #[test] + fn vhdl_dependency_sources_resolves_relative_path_dep() { + // Portable fixture pattern: a path dep whose `path` + // is relative to the declaring workspace's vw.toml — + // Cargo-parity. Same fixture works from any machine. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(ws.join("fixtures/lib/hdl")).unwrap(); + std::fs::write(ws.join("fixtures/lib/hdl/a.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.lib] +path = "fixtures/lib" +src = ["hdl"] +recursive = true +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert!(sources[0].path.ends_with("hdl/a.vhd")); + } - // Initialize and update submodules if requested - if submodules { - for mut submodule in - repo.submodules().map_err(|e| VwError::Git { - message: format!("Failed to list submodules: {e}"), - })? - { - submodule.init(false).map_err(|e| VwError::Git { - message: format!( - "Failed to init submodule '{}': {e}", - submodule.name().unwrap_or("unknown") - ), - })?; - submodule.update(true, None).map_err(|e| VwError::Git { - message: format!( - "Failed to update submodule '{}': {e}", - submodule.name().unwrap_or("unknown") - ), - })?; - } - } + #[test] + fn get_access_credentials_for_workspace_only_scans_test_deps_when_asked() { + // No netrc → both variants return None regardless of + // dep-set — regression guard for the `include_test` + // dispatch path. The bigger scenario (netrc HIT for a + // git URL) is covered by the underlying + // `get_access_credentials_from_netrc` test. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" + +[test-dependencies.gt] +repo = "https://example.invalid/y" +branch = "main" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + assert!( + get_access_credentials_for_workspace(&ws_utf8, false).is_none(), + ); + assert!(get_access_credentials_for_workspace(&ws_utf8, true).is_none(),); + } - Ok::<(), VwError>(()) - }), - ) - .await - .map_err(|_| VwError::Git { - message: "Git clone timed out after 120 seconds".to_string(), - })? - .map_err(|e| VwError::Git { - message: format!("Failed to execute git operations: {e}"), - })??; + #[test] + fn unlocked_git_deps_detection() { + // No git deps → never unlocked, regardless of lockfile. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.local] +path = "/tmp/somewhere" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws.clone()).unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, true).unwrap()); + + // Git dep + missing lockfile → unlocked. + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" +src = ["hdl"] +"#, + ) + .unwrap(); + assert!(workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + + // Git dep + lockfile that has an entry for it → locked. + std::fs::write( + ws.join("vw.lock"), + r#" +[dependencies.g] +repo = "https://example.invalid/x" +commit = "abc" +path = "/tmp/g" +src = ["hdl"] +recursive = false +sim_only = false +submodules = false +exclude = [] +"#, + ) + .unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + + // Git test-dep, lockfile only has the regular dep → unlocked + // for the with-test caller, locked for the plain caller. + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.g] +repo = "https://example.invalid/x" +branch = "main" + +[test-dependencies.gt] +repo = "https://example.invalid/y" +branch = "main" +"#, + ) + .unwrap(); + assert!(!workspace_has_unlocked_git_deps(&ws_utf8, false).unwrap()); + assert!(workspace_has_unlocked_git_deps(&ws_utf8, true).unwrap()); + } - fs::create_dir_all(dest_path).map_err(|e| VwError::FileSystem { - message: format!("Failed to create destination directory: {e}"), - })?; + #[test] + fn vhdl_dependency_sources_exclude_sim_only_flag() { + // Two path deps: one flagged `sim_only = true` (mirrors + // real deps like `unisim` / `xpm`) and one regular. + // With the flag off both contribute; with it on only + // the non-sim dep does. + let tmp = tempfile::tempdir().unwrap(); + let sim_dep = tmp.path().join("sim"); + std::fs::create_dir_all(sim_dep.join("hdl")).unwrap(); + std::fs::write(sim_dep.join("hdl/sim.vhd"), "").unwrap(); + let real_dep = tmp.path().join("real"); + std::fs::create_dir_all(real_dep.join("hdl")).unwrap(); + std::fs::write(real_dep.join("hdl/real.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.sim] +path = "{}" +src = ["hdl"] +recursive = true +sim_only = true + +[dependencies.real] +path = "{}" +src = ["hdl"] +recursive = true +"#, + sim_dep.display(), + real_dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + + // Default (flag = false): both deps contribute. + let all = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(all.len(), 2, "{all:?}"); + + // Flag on: only the non-sim dep survives. + let synth_clean = + vhdl_dependency_sources_ext(&ws_utf8, false, true).unwrap(); + assert_eq!(synth_clean.len(), 1, "{synth_clean:?}"); + assert_eq!(synth_clean[0].library, "real"); + } - // Treat all src values as globs (handles files, directories, and patterns) - for src_path in &src_paths { - copy_vhdl_files_glob( - temp_dir.path(), - src_path, - dest_path, - recursive, - exclude, - )?; + #[test] + fn vhdl_dependency_sources_include_test_flag() { + // A test-only dep contributes iff include_test is set. + let tmp = tempfile::tempdir().unwrap(); + let dep = tmp.path().join("dep"); + std::fs::create_dir_all(dep.join("hdl")).unwrap(); + std::fs::write(dep.join("hdl/tbutil.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[test-dependencies.tbutil] +path = "{}" +src = ["hdl"] +recursive = true +"#, + dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + // Production mode: test-dep hidden. + assert!(vhdl_dependency_sources(&ws_utf8).unwrap().is_empty()); + // Test mode: test-dep visible. + let with_test = + vhdl_dependency_sources_with_test(&ws_utf8, true).unwrap(); + assert_eq!(with_test.len(), 1); + assert_eq!(with_test[0].library, "tbutil"); } - Ok(()) -} + #[test] + fn vhdl_dependency_sources_honors_exclude() { + let tmp = tempfile::tempdir().unwrap(); + let dep = tmp.path().join("dep"); + std::fs::create_dir_all(dep.join("hdl/sims")).unwrap(); + std::fs::write(dep.join("hdl/a.vhd"), "").unwrap(); + std::fs::write(dep.join("hdl/b_tb.vhd"), "").unwrap(); + std::fs::write(dep.join("hdl/sims/x.vhd"), "").unwrap(); + + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::write( + ws.join("vw.toml"), + format!( + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[dependencies.dep] +path = "{}" +src = ["hdl"] +recursive = true +exclude = ["**/sims/**", "**/*_tb.vhd"] +"#, + dep.display(), + ), + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_dependency_sources(&ws_utf8).unwrap(); + assert_eq!(sources.len(), 1, "{sources:?}"); + assert!(sources[0].path.ends_with("a.vhd")); + } -fn copy_vhdl_files_glob( - repo_root: &Path, - src_pattern: &str, - dest: &Path, - recursive: bool, - exclude: &[String], -) -> Result<()> { - // Build patterns to match - let src_path = repo_root.join(src_pattern); - let mut patterns = Vec::new(); - let strip_prefix: PathBuf; + #[test] + fn design_constraints_empty_when_no_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + assert!(design_constraints(&ws).unwrap().is_empty()); + } - // Compile exclude patterns - let exclude_patterns: Vec = exclude - .iter() - .filter_map(|p| glob::Pattern::new(p).ok()) - .collect(); + #[test] + fn phase_scoped_constraints_isolate_per_subdir() { + // Regression guard: `synth/` files must not leak into + // `place/` or `route/`, and vice versa. Also verifies + // the whole-tree `design_constraints` still returns + // everything. + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("synth")).unwrap(); + std::fs::create_dir_all(c.join("place")).unwrap(); + std::fs::create_dir_all(c.join("route")).unwrap(); + std::fs::write(c.join("global.xdc"), "").unwrap(); + std::fs::write(c.join("synth/only.xdc"), "").unwrap(); + std::fs::write(c.join("place/only.xdc"), "").unwrap(); + std::fs::write(c.join("route/only.xdc"), "").unwrap(); + + let synth = design_synth_constraints(&ws).unwrap(); + assert_eq!(synth.len(), 1); + assert!(synth[0].ends_with("synth/only.xdc")); + + let place = design_place_constraints(&ws).unwrap(); + assert_eq!(place.len(), 1); + assert!(place[0].ends_with("place/only.xdc")); + + let route = design_route_constraints(&ws).unwrap(); + assert_eq!(route.len(), 1); + assert!(route[0].ends_with("route/only.xdc")); + + // Aggregate walk returns everything under constraints/ + // regardless of subdir. + let all = design_constraints(&ws).unwrap(); + assert_eq!(all.len(), 4); + } - // Check if src_pattern points to a directory - if src_path.is_dir() { - // It's a directory - create appropriate glob patterns - let base_pattern = - src_path.to_str().ok_or_else(|| VwError::FileSystem { - message: "Invalid UTF-8 in path".to_string(), - })?; + #[test] + fn phase_scoped_constraints_empty_when_subdir_missing() { + // `constraints/` exists (some other subdir) but + // `constraints/synth/` does not — expect empty, not error. + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("place")).unwrap(); + std::fs::write(c.join("place/only.xdc"), "").unwrap(); + + assert!(design_synth_constraints(&ws).unwrap().is_empty()); + assert!(design_route_constraints(&ws).unwrap().is_empty()); + assert_eq!(design_place_constraints(&ws).unwrap().len(), 1); + } - if recursive { - // Recursively find all VHDL files - patterns.push(format!("{base_pattern}/**/*.vhd")); - patterns.push(format!("{base_pattern}/**/*.vhdl")); - } else { - // Only files directly in the directory - patterns.push(format!("{base_pattern}/*.vhd")); - patterns.push(format!("{base_pattern}/*.vhdl")); - } - // For directories, strip the src directory from paths - strip_prefix = src_path; - } else if src_path.is_file() { - // It's a single file - use as-is - patterns.push( - src_path - .to_str() - .ok_or_else(|| VwError::FileSystem { - message: "Invalid UTF-8 in path".to_string(), - })? - .to_string(), - ); - // For single files, strip the parent directory - strip_prefix = src_path - .parent() - .ok_or_else(|| VwError::FileSystem { - message: "File has no parent directory".to_string(), - })? - .to_path_buf(); - } else { - // It's a glob pattern or doesn't exist yet - use as-is - patterns.push( - src_path - .to_str() - .ok_or_else(|| VwError::FileSystem { - message: "Invalid UTF-8 in glob pattern path".to_string(), - })? - .to_string(), - ); - // For glob patterns, strip the repo root to preserve relative structure - strip_prefix = repo_root.to_path_buf(); + #[test] + fn design_constraints_finds_xdc_and_sdc_recursively() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let c = ws.join("constraints"); + std::fs::create_dir_all(c.join("sub")).unwrap(); + std::fs::write(c.join("timing.xdc"), "").unwrap(); + std::fs::write(c.join("sub/pins.xdc"), "").unwrap(); + std::fs::write(c.join("sub/synopsys.sdc"), "").unwrap(); + // Non-constraint sibling — should be skipped. + std::fs::write(c.join("readme.md"), "").unwrap(); + + let files = design_constraints(&ws).unwrap(); + let names: Vec = files + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(files.len(), 3, "{files:?}"); + assert!(names.contains(&"timing.xdc".to_string())); + assert!(names.contains(&"pins.xdc".to_string())); + assert!(names.contains(&"synopsys.sdc".to_string())); + assert!(!names.contains(&"readme.md".to_string())); } - let mut copied_count = 0; - for pattern_str in &patterns { - // Use glob to find matching files - let entries = - glob::glob(pattern_str).map_err(|e| VwError::FileSystem { - message: format!("Invalid glob pattern '{pattern_str}': {e}"), - })?; + #[test] + fn vhdl_ip_sources_empty_when_no_target_ip_dir() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + assert!(vhdl_ip_sources(&ws).unwrap().is_empty()); + } - for entry in entries { - let path = entry.map_err(|e| VwError::FileSystem { - message: format!("Error reading glob entry: {e}"), - })?; + #[test] + fn vhdl_ip_sources_walks_target_ip_recursively() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let ip = ws.join("target/ip"); + std::fs::create_dir_all(ip.join("clocky")).unwrap(); + std::fs::create_dir_all(ip.join("cips")).unwrap(); + std::fs::write(ip.join("clocky/wrapper.vhd"), "").unwrap(); + std::fs::write(ip.join("cips/wrapper.vhd"), "").unwrap(); + // Non-VHDL siblings shouldn't get pulled in. + std::fs::write(ip.join("clocky/notes.md"), "").unwrap(); + + let sources = vhdl_ip_sources(&ws).unwrap(); + let names: Vec = sources + .iter() + .map(|p| { + let ip_name = p + .parent() + .and_then(|d| d.file_name()) + .and_then(|s| s.to_str()) + .unwrap_or(""); + ip_name.to_string() + }) + .collect(); + assert_eq!(sources.len(), 2, "{sources:?}"); + assert!(names.contains(&"clocky".to_string())); + assert!(names.contains(&"cips".to_string())); + } - // Only copy VHDL files - if path.is_file() { - if let Some(ext) = path.extension() { - if ext == "vhd" || ext == "vhdl" { - // Compute relative path based on strip_prefix - let relative_path = - path.strip_prefix(&strip_prefix).map_err(|e| { - VwError::FileSystem { - message: format!( - "Failed to compute relative path for {path:?}: {e}" - ), - } - })?; + /// Regression: `target/ip/bd/**`, `target/ip/xci/**`, and + /// `target/vw-project/**` are the (legacy + on-disk) Vivado + /// cache trees. The `.vhd` files under them are registered + /// with the Vivado project via `read_bd` / `read_ip` / + /// `synth_ip`, so `vhdl_ip_sources` must NOT list them + /// (otherwise `synth`'s `read_vhdl` conflicts with the + /// sub-design registration and Vivado emits `[filemgmt + /// 20-1440]` CRITICAL WARNINGs). + /// + /// `target/vw-project/` is a sibling of `target/ip/` so the + /// current walker rooted at `target/ip/` doesn't actually + /// descend into it — the assertion here documents the + /// invariant so a future walker refactor (rooting higher up, + /// at `target/` for example) doesn't silently regress. + #[test] + fn vhdl_ip_sources_excludes_vivado_cache_subtrees() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let ip = ws.join("target/ip"); + // Legit wrappers. + std::fs::create_dir_all(ip.join("cips")).unwrap(); + std::fs::create_dir_all(ip.join("dcmac")).unwrap(); + std::fs::write(ip.join("cips/wrapper.vhd"), "").unwrap(); + std::fs::write(ip.join("dcmac/wrapper.vhd"), "").unwrap(); + // Legacy BD cache outputs — filter out. + std::fs::create_dir_all(ip.join("bd/cips/synth")).unwrap(); + std::fs::create_dir_all(ip.join("bd/cips/sim")).unwrap(); + std::fs::create_dir_all(ip.join("bd/dcmac/synth")).unwrap(); + std::fs::write(ip.join("bd/cips/synth/cips.vhd"), "").unwrap(); + std::fs::write(ip.join("bd/cips/sim/cips.vhd"), "").unwrap(); + std::fs::write(ip.join("bd/dcmac/synth/dcmac.vhd"), "").unwrap(); + // Legacy XCI cache outputs — filter out. + std::fs::create_dir_all(ip.join("xci/primary_clock")).unwrap(); + std::fs::write(ip.join("xci/primary_clock/primary_clock.vhd"), "") + .unwrap(); + // On-disk Vivado project sibling — invariant guard. + let vw_proj_gen = ws.join( + "target/vw-project/metroid/metroid.gen/sources_1/bd/cips/synth", + ); + std::fs::create_dir_all(&vw_proj_gen).unwrap(); + std::fs::write(vw_proj_gen.join("cips.vhd"), "").unwrap(); + + let sources = vhdl_ip_sources(&ws).unwrap(); + assert_eq!(sources.len(), 2, "{sources:?}"); + for p in &sources { + let s = p.to_string_lossy(); + assert!( + !s.contains("/target/ip/bd/"), + "bd cache file leaked into ip_sources: {s}" + ); + assert!( + !s.contains("/target/ip/xci/"), + "xci cache file leaked into ip_sources: {s}" + ); + assert!( + !s.contains("/target/vw-project/"), + "vw-project file leaked into ip_sources: {s}" + ); + assert!(s.ends_with("wrapper.vhd"), "unexpected file: {s}"); + } + } + + // `render_vhdl_ls_config` composes design + wrappers + BD RTL + // + deps into a single VhdlLsConfig. Confirms every source + // lands in the right library. + #[test] + fn render_vhdl_ls_config_populates_expected_libraries() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + // Design source. + std::fs::create_dir_all(ws.join("hdl")).unwrap(); + std::fs::write(ws.join("hdl/top.vhd"), "").unwrap(); + // Empty vw.toml so workspace enumeration succeeds + // without complaining about missing config. + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n[dependencies]\n\ + [test-dependencies]\n", + ) + .unwrap(); + // IP wrapper. + std::fs::create_dir_all(ws.join("target/ip/dcmac")).unwrap(); + std::fs::write(ws.join("target/ip/dcmac/wrapper.vhd"), "").unwrap(); + // BD-generated RTL — only top-level wrappers survive the + // walker's filter (see `keep_vivado_generated_path`). + let bd_root = ws.join( + "target/vw-project/scratch/scratch.gen/sources_1/bd/dcmac/hdl", + ); + std::fs::create_dir_all(&bd_root).unwrap(); + std::fs::write(bd_root.join("dcmac_wrapper.vhd"), "").unwrap(); - // Check if file matches any exclude pattern - let path_str = relative_path.to_string_lossy(); - if exclude_patterns.iter().any(|p| p.matches(&path_str)) - { - continue; // Skip excluded files - } + let cfg = render_vhdl_ls_config(&ws, None, false).unwrap(); - let dest_file = dest.join(relative_path); + assert!( + cfg.libraries.contains_key("defaultlib"), + "missing defaultlib: {:?}", + cfg.libraries.keys().collect::>() + ); + assert!( + cfg.libraries.contains_key("ip"), + "missing ip: {:?}", + cfg.libraries.keys().collect::>() + ); + assert!( + cfg.libraries.contains_key("xil_defaultlib"), + "missing xil_defaultlib: {:?}", + cfg.libraries.keys().collect::>() + ); - // Create parent directories if needed - if let Some(parent) = dest_file.parent() { - fs::create_dir_all(parent).map_err(|e| { - VwError::FileSystem { - message: format!( - "Failed to create directory {parent:?}: {e}" - ), - } - })?; - } + let xil = &cfg.libraries["xil_defaultlib"]; + assert_eq!(xil.files.len(), 1); + assert!( + xil.files[0] + .to_string_lossy() + .ends_with("dcmac_wrapper.vhd"), + "unexpected xil_defaultlib path: {:?}", + xil.files[0] + ); + assert_eq!(xil.is_third_party, Some(true)); - fs::copy(&path, &dest_file).map_err(|e| { - VwError::FileSystem { - message: format!( - "Failed to copy file {path:?}: {e}" - ), - } - })?; - copied_count += 1; - } - } - } - } + let ip = &cfg.libraries["ip"]; + assert_eq!(ip.files.len(), 1); + assert!(ip.files[0].to_string_lossy().ends_with("wrapper.vhd")); + + let design = &cfg.libraries["defaultlib"]; + assert_eq!(design.files.len(), 1); + assert!(design.files[0].to_string_lossy().ends_with("top.vhd")); } - if copied_count == 0 { - return Err(VwError::Dependency { - message: format!("No VHDL files matched pattern '{src_pattern}'"), - }); + /// The renderer must produce output that vhdl_lang's TOML + /// parser can consume — this pins the contract that a + /// round-trip through `Config::from_str` succeeds. + #[test] + fn render_vhdl_lang_config_round_trips_through_toml() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n[dependencies]\n\ + [test-dependencies]\n", + ) + .unwrap(); + std::fs::create_dir_all(ws.join("hdl")).unwrap(); + std::fs::write(ws.join("hdl/top.vhd"), "").unwrap(); + let cfg = render_vhdl_lang_config(&ws, None).unwrap(); + // The lang config should carry the same library we + // populated in the LS config; iterate files to confirm. + // vhdl_lang's Config has no public library iterator, so + // the round-trip succeeding is itself the check. + drop(cfg); } - Ok(()) -} + #[test] + fn vhdl_design_sources_walks_recursive_and_sorts() { + let tmp = tempfile::tempdir().unwrap(); + let ws = Utf8PathBuf::from_path_buf(tmp.path().to_path_buf()).unwrap(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(hdl.join("sub")).unwrap(); + // Files under both root and a subdir; one non-VHDL to + // prove the extension filter kicks in. + std::fs::write(hdl.join("b.vhd"), "").unwrap(); + std::fs::write(hdl.join("a.vhd"), "").unwrap(); + std::fs::write(hdl.join("sub").join("c.vhdl"), "").unwrap(); + std::fs::write(hdl.join("readme.md"), "").unwrap(); + + let sources = vhdl_design_sources(&ws).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + // Sorted absolute paths → `a.vhd` before `b.vhd`, and + // `sub/c.vhdl` lands where its full path sorts. Not + // asserting exact order across subdirs — just checking + // both extensions and the recursion picked up the sub. + assert!(names.contains(&"a.vhd".to_string())); + assert!(names.contains(&"b.vhd".to_string())); + assert!(names.contains(&"c.vhdl".to_string())); + assert!(!names.contains(&"readme.md".to_string())); + } -fn find_vhdl_files( - dir: &Path, - recursive: bool, - exclude: &[String], -) -> Result> { - let mut vhdl_files = Vec::new(); - find_vhdl_files_impl(dir, &mut vhdl_files, recursive)?; + fn make_variant_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + // Layout: + // hdl/shared.vhd (in no variant's exclusive → always included) + // hdl/ethernet-vpk120.vhd (owned by vpk120) + // hdl/ethernet-metro.vhd (owned by metro) + let ws = tmp.path().to_path_buf(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(&hdl).unwrap(); + std::fs::write(hdl.join("shared.vhd"), "").unwrap(); + std::fs::write(hdl.join("ethernet-vpk120.vhd"), "").unwrap(); + std::fs::write(hdl.join("ethernet-metro.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[[workspace.variants]] +name = "vpk120" +part = "xcvp1202-vsva2785-2MHP-e-S" +default = true +exclusive = ["hdl/ethernet-vpk120.vhd"] + +[[workspace.variants]] +name = "metro" +part = "xcvp1202-vsva2785-3HP-e-S" +exclusive = ["hdl/ethernet-metro.vhd"] +"#, + ) + .unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() + } - // Filter out excluded files - if !exclude.is_empty() { - let exclude_patterns: Vec = exclude + #[test] + fn design_sources_filter_by_active_variant() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_variant_ws(&tmp); + + // Active variant vpk120 → keeps shared + vpk120 file, + // excludes metro's. + let sources = + vhdl_design_sources_for_variant(&ws, Some("vpk120")).unwrap(); + let names: Vec = sources .iter() - .filter_map(|p| glob::Pattern::new(p).ok()) + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) .collect(); + assert!(names.contains(&"shared.vhd".to_string())); + assert!(names.contains(&"ethernet-vpk120.vhd".to_string())); + assert!(!names.contains(&"ethernet-metro.vhd".to_string())); + + // Flip to metro. + let sources = + vhdl_design_sources_for_variant(&ws, Some("metro")).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert!(names.contains(&"shared.vhd".to_string())); + assert!(names.contains(&"ethernet-metro.vhd".to_string())); + assert!(!names.contains(&"ethernet-vpk120.vhd".to_string())); + } - vhdl_files.retain(|file| { - // Match against path relative to the base directory - let relative = file.strip_prefix(dir).unwrap_or(file); - let path_str = relative.to_string_lossy(); - !exclude_patterns - .iter() - .any(|pattern| pattern.matches(&path_str)) - }); + #[test] + fn design_sources_no_active_variant_still_filters_out_owned_files() { + // When active_variant is None but the workspace declares + // variants, ALL exclusive files are dropped — otherwise + // we'd spuriously pull every variant's owned files into + // one giant surface (which is the exact bug variants + // exist to solve). + let tmp = tempfile::tempdir().unwrap(); + let ws = make_variant_ws(&tmp); + let sources = vhdl_design_sources_for_variant(&ws, None).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names, vec!["shared.vhd"]); } - Ok(vhdl_files) -} + #[test] + fn design_sources_no_variants_declared_returns_all() { + // A workspace without any variants keeps the pre-variants + // behavior — every `.vhd` under `hdl/` shows up. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().to_path_buf(); + std::fs::create_dir_all(ws.join("hdl")).unwrap(); + std::fs::write(ws.join("hdl/a.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = vhdl_design_sources_for_variant(&ws_utf8, None).unwrap(); + assert_eq!(sources.len(), 1); + } -fn find_vhdl_files_impl( - dir: &Path, - vhdl_files: &mut Vec, - recursive: bool, -) -> Result<()> { - for entry in fs::read_dir(dir).map_err(|e| VwError::FileSystem { - message: format!("Failed to read directory: {e}"), - })? { - let entry = entry.map_err(|e| VwError::FileSystem { - message: format!("Failed to read directory entry: {e}"), - })?; - let path = entry.path(); + #[test] + fn design_sources_variant_exclusive_supports_globs() { + // `exclusive = ["hdl/board-vpk120/**/*.vhd"]` scopes an + // entire subtree to a variant. + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().to_path_buf(); + std::fs::create_dir_all(ws.join("hdl/board-vpk120/sub")).unwrap(); + std::fs::create_dir_all(ws.join("hdl/board-metro")).unwrap(); + std::fs::write(ws.join("hdl/shared.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-vpk120/top.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-vpk120/sub/x.vhd"), "").unwrap(); + std::fs::write(ws.join("hdl/board-metro/top.vhd"), "").unwrap(); + std::fs::write( + ws.join("vw.toml"), + r#" +[workspace] +name = "ws" +version = "0.1.0" + +[[workspace.variants]] +name = "vpk120" +part = "xcvp1202-vsva2785-2MHP-e-S" +default = true +exclusive = ["hdl/board-vpk120/**/*.vhd"] + +[[workspace.variants]] +name = "metro" +part = "xcvp1202-vsva2785-3HP-e-S" +exclusive = ["hdl/board-metro/**/*.vhd"] +"#, + ) + .unwrap(); + let ws_utf8 = Utf8PathBuf::from_path_buf(ws).unwrap(); + let sources = + vhdl_design_sources_for_variant(&ws_utf8, Some("vpk120")).unwrap(); + let names: Vec = sources + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + // shared + both vpk120 files, no metro files. + assert_eq!(sources.len(), 3, "{names:?}"); + assert!(names.iter().any(|n| n == "shared.vhd")); + assert!( + names.iter().filter(|n| n.as_str() == "top.vhd").count() == 1, + "expected exactly one top.vhd (vpk120's), got {names:?}", + ); + } - if path.is_dir() { - if recursive { - find_vhdl_files_impl(&path, vhdl_files, recursive)?; - } - } else if let Some(extension) = - path.extension().and_then(|ext| ext.to_str()) - { - if extension == "vhd" || extension == "vhdl" { - vhdl_files.push(path); - } - } + /// Minimal workspace scaffold for `synth_needs_update` tests: + /// a `vw.toml`, empty `vw.lock`, one hdl file, one synth XDC, + /// and one workspace htcl file. Returned as `Utf8PathBuf` so + /// the caller can hand it to the enumerator directly. + fn make_synth_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"snws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + std::fs::write(ws.join("vw.lock"), "{}\n").unwrap(); + let hdl = ws.join("hdl"); + std::fs::create_dir_all(&hdl).unwrap(); + std::fs::write(hdl.join("top.vhd"), "-- vhdl\n").unwrap(); + let xdc = ws.join("constraints").join("synth"); + std::fs::create_dir_all(&xdc).unwrap(); + std::fs::write(xdc.join("timing.xdc"), "# xdc\n").unwrap(); + std::fs::write(ws.join("design.htcl"), "# htcl\n").unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() } - Ok(()) -} -fn write_lock_file( - workspace_dir: &Utf8Path, - lock_file: &LockFile, -) -> Result<()> { - let toml_content = toml::to_string_pretty(lock_file)?; - let lock_path = workspace_dir.join("vw.lock"); + /// A missing checkpoint file is always stale — the whole point + /// of the cache-check is to gate a first-time synth on this + /// condition. + #[test] + fn synth_needs_update_true_when_checkpoint_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } - fs::write(&lock_path, toml_content).map_err(|e| VwError::FileSystem { - message: format!("Failed to write vw.lock file: {e}"), - })?; + /// Checkpoint present but no manifest → stale (either + /// pre-manifest era or a manually-copied checkpoint from + /// another workspace). Forces the next synth to write both, + /// which is the safe recovery path. + #[test] + fn synth_needs_update_true_when_manifest_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } - Ok(()) -} + /// Manifest matches current fingerprint → fresh. Simulates + /// the post-`vw::synth` state on an unchanged tree — no + /// resynthesis required. + #[test] + fn synth_needs_update_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + assert!(!synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } -fn write_vhdl_ls_config( - workspace_dir: &Utf8Path, - managed_config: &VhdlLsConfig, -) -> Result<()> { - let mut existing_config = load_existing_vhdl_ls_config(workspace_dir)?; + /// After a checkpoint+manifest pair, rewriting a source with + /// NEW bytes invalidates the fingerprint. This is the primary + /// invalidation path — a real edit to a tracked file. + #[test] + fn synth_needs_update_true_when_source_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::fs::write(ws.join("hdl/top.vhd"), "-- vhdl updated\n").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } - // Remove any existing managed dependencies and add the new ones - for (name, library) in &managed_config.libraries { - existing_config - .libraries - .insert(name.clone(), library.clone()); + /// The regression this whole switch to content hashing fixes: + /// rewriting a tracked file with IDENTICAL bytes must NOT + /// invalidate the manifest. Simulates `make_wrapper` + /// regenerating `target/ip/*/wrapper.vhd` on every design.htcl + /// run — same stripped-header body, fresh mtime. + #[test] + fn synth_needs_update_false_after_identical_rewrite() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + // Overwrite with the same bytes the fixture wrote. Fresh + // mtime, unchanged content. Under the old mtime-based + // check this returned `true` (stale); under content + // hashing it stays `false`. + std::fs::write(ws.join("hdl/top.vhd"), "-- vhdl\n").unwrap(); + assert!(!synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); } - let toml_content = toml::to_string_pretty(&existing_config)?; - let config_path = workspace_dir.join("vhdl_ls.toml"); + /// Editing a workspace `.htcl` invalidates the manifest. + /// Exercises `list_workspace_htcl_files` — a source + /// enumerator distinct from the VHDL / XDC paths. + #[test] + fn synth_needs_update_true_when_htcl_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_synth_ws(&tmp); + let cp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(cp.parent().unwrap()).unwrap(); + std::fs::write(&cp, "").unwrap(); + write_synth_checkpoint_manifest(&ws, cp.as_std_path(), None).unwrap(); + std::fs::write(ws.join("design.htcl"), "# htcl updated\n").unwrap(); + assert!(synth_needs_update(&ws, cp.as_std_path(), None).unwrap()); + } - fs::write(&config_path, toml_content).map_err(|e| VwError::FileSystem { - message: format!("Failed to write vhdl_ls.toml file: {e}"), - })?; + /// Minimal workspace scaffold for `project_needs_wipe` tests: + /// a `vw.toml` and an `ip/module.htcl` (+ one submodule so + /// the recursive walk has something to cover). + fn make_project_ws(tmp: &tempfile::TempDir) -> Utf8PathBuf { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"prws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let ip = ws.join("ip"); + std::fs::create_dir_all(&ip).unwrap(); + std::fs::write( + ip.join("module.htcl"), + "namespace eval ip { proc configure {} unit {} }\n", + ) + .unwrap(); + std::fs::write(ip.join("cips.htcl"), "# cips\n").unwrap(); + Utf8PathBuf::from_path_buf(ws).unwrap() + } - Ok(()) -} + /// Materialize `//.xpr` as an empty + /// placeholder — `project_needs_wipe` short-circuits when + /// the `.xpr` is missing, so tests that want to exercise the + /// manifest branch need one present. + fn touch_placeholder_xpr(ws: &Utf8Path, name: &str) -> PathBuf { + let project_dir = vw_project_dir(ws); + let inner = project_dir.join(name); + std::fs::create_dir_all(inner.as_std_path()).unwrap(); + let xpr = inner.join(format!("{name}.xpr")); + std::fs::write(xpr.as_std_path(), "").unwrap(); + project_dir.into_std_path_buf() + } -/// Build a Rust library for a testbench. -/// Looks for Cargo.toml in the testbench directory, builds it, and returns the path to the .so file. -async fn build_rust_library( - bench_dir: &Utf8Path, - testbench_file: &Path, -) -> Result { - // Get the testbench directory - let testbench_dir = - testbench_file.parent().ok_or_else(|| VwError::Testbench { - message: format!( - "Testbench file {:?} has no parent directory???", - testbench_file - ), - })?; + /// Missing `.xpr` → always needs wipe (first-time-project + /// bootstrap). The manifest presence is irrelevant when the + /// `.xpr` isn't there — nothing to open. + #[test] + fn project_needs_wipe_true_when_xpr_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = vw_project_dir(&ws); + assert!( + project_needs_wipe(&ws, project_dir.as_std_path(), "prws").unwrap() + ); + } - // Look for Cargo.toml in the testbench directory - let cargo_toml_path = testbench_dir.join("Cargo.toml"); - if !cargo_toml_path.exists() { - return Err(VwError::Testbench { - message: format!( - "Cargo.toml not found in testbench directory: {:?}", - testbench_dir - ), - }); + /// `.xpr` present but manifest missing → wipe. This is the + /// "someone deleted the sidecar" or "pre-manifest-era + /// project" recovery path. + #[test] + fn project_needs_wipe_true_when_manifest_missing() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); } - // Parse Cargo.toml to get the package name - let cargo_toml_content = - fs::read_to_string(&cargo_toml_path).map_err(|e| { - VwError::FileSystem { - message: format!("Failed to read Cargo.toml: {e}"), - } - })?; + /// Fresh manifest matching current fingerprint → do NOT + /// wipe. Simulates the post-`vw::configure_ip` state on an + /// unchanged tree. + #[test] + fn project_needs_wipe_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } - let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?; - let package_name = cargo_toml.package.name; + /// Editing any `.htcl` under `/ip/` invalidates. Key + /// test — the invalidation trigger the user actually + /// controls (adding an IP, tweaking a configure_* parameter). + #[test] + fn project_needs_wipe_true_when_ip_htcl_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write(ws.join("ip/cips.htcl"), "# cips updated\n").unwrap(); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } - // Run cargo build in the testbench directory - let testbench_dir_owned = testbench_dir.to_path_buf(); - tokio::task::spawn_blocking(move || { - let output = std::process::Command::new("cargo") - .arg("build") - .current_dir(&testbench_dir_owned) - .output() - .map_err(|e| VwError::Testbench { - message: format!("Failed to execute cargo build: {e}"), - })?; + /// Editing `vw.toml` (target-part, deps list, etc.) also + /// invalidates — those changes usually mean the project + /// itself needs a fresh `create_project -part ...`. + #[test] + fn project_needs_wipe_true_when_vw_toml_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"prws\"\nversion = \"0.2.0\"\n", + ) + .unwrap(); + assert!(project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(VwError::Testbench { - message: format!("cargo build failed:\n{stderr}"), - }); - } + /// Editing a workspace htcl OUTSIDE ip/ (e.g. design.htcl) + /// must NOT invalidate the on-disk project. Design-level + /// changes are the synth checkpoint's concern; the project + /// scope stays narrow so we don't nuke the expensive BD/IP + /// state on every design edit. + #[test] + fn project_needs_wipe_false_when_non_ip_htcl_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + std::fs::write(ws.join("design.htcl"), "# design\n").unwrap(); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::fs::write(ws.join("design.htcl"), "# design updated\n").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } - Ok::<(), VwError>(()) - }) - .await - .map_err(|e| VwError::Testbench { - message: format!("Failed to execute cargo build task: {e}"), - })??; + /// The regression parallel to the synth case: rewriting an + /// `ip/*.htcl` with IDENTICAL bytes must not invalidate. + /// Content-hash based, not mtime. + #[test] + fn project_needs_wipe_false_after_identical_rewrite() { + let tmp = tempfile::tempdir().unwrap(); + let ws = make_project_ws(&tmp); + let project_dir = touch_placeholder_xpr(&ws, "prws"); + write_project_manifest(&ws, &project_dir, "prws").unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + std::fs::write(ws.join("ip/cips.htcl"), "# cips\n").unwrap(); + assert!(!project_needs_wipe(&ws, &project_dir, "prws").unwrap()); + } - // Find the .so file in the workspace target directory (parent of testbench dir) - let ext = if cfg!(target_os = "macos") { - "dylib" - } else { - "so" - }; - let lib_name = format!("lib{}.{ext}", package_name.replace('-', "_")); - let workspace_target = bench_dir.join("target").join("debug"); + /// Minimal workspace scaffold for `place_needs_update` tests: + /// a `vw.toml`, one place-scoped XDC, and a stand-in synth DCP + /// that the fingerprint folds in as a proxy for "everything + /// synth depended on". + fn make_place_ws( + tmp: &tempfile::TempDir, + ) -> (Utf8PathBuf, PathBuf, PathBuf) { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"plws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let xdc = ws.join("constraints").join("place"); + std::fs::create_dir_all(&xdc).unwrap(); + std::fs::write(xdc.join("place.xdc"), "# place xdc\n").unwrap(); + let synth_dcp = ws.join("target/synth/top.dcp"); + std::fs::create_dir_all(synth_dcp.parent().unwrap()).unwrap(); + std::fs::write(&synth_dcp, "").unwrap(); + let place_dcp = ws.join("target/place/top.dcp"); + ( + Utf8PathBuf::from_path_buf(ws).unwrap(), + place_dcp, + synth_dcp, + ) + } - let lib_path = workspace_target.join(&lib_name); + /// Missing checkpoint → stale. Same first-place rule the + /// synth/ip caches use. + #[test] + fn place_needs_update_true_when_checkpoint_missing() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } - if !lib_path.exists() { - return Err(VwError::Testbench { - message: format!( - "Built Rust library not found at expected path: {:?}", - lib_path - ), - }); + /// Fresh manifest → not stale. Verifies the write+check + /// round-trip on the place scope (place XDCs + synth DCP). + #[test] + fn place_needs_update_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + assert!(!place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); } - Ok(lib_path.into()) + /// Editing a place XDC invalidates. Trigger the user + /// controls most directly (tweak a place constraint, + /// re-place should fire). + #[test] + fn place_needs_update_true_when_place_xdc_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write( + ws.join("constraints/place/place.xdc"), + "# place xdc updated\n", + ) + .unwrap(); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Synth re-ran → synth DCP content changed → place + /// invalidates. The synth DCP is intentionally folded into + /// the place fingerprint as a proxy for "everything synth + /// depended on". + #[test] + fn place_needs_update_true_when_synth_checkpoint_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write(&synth_dcp, "different bytes").unwrap(); + assert!(place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Non-place workspace file changes must NOT invalidate the + /// place cache (design.htcl, hdl/, etc. are captured by the + /// synth stage; place scope is narrower). + #[test] + fn place_needs_update_false_when_non_place_htcl_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, place_dcp, synth_dcp) = make_place_ws(&tmp); + std::fs::write(ws.join("design.htcl"), "# design\n").unwrap(); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + write_place_checkpoint_manifest( + &ws, + place_dcp.as_path(), + synth_dcp.as_path(), + ) + .unwrap(); + std::fs::write(ws.join("design.htcl"), "# design updated\n").unwrap(); + assert!(!place_needs_update( + &ws, + place_dcp.as_path(), + synth_dcp.as_path() + ) + .unwrap()); + } + + /// Minimal workspace scaffold for `route_needs_update` tests: + /// a `vw.toml`, one route-scoped XDC, and a stand-in place DCP + /// that the fingerprint folds in as a proxy for "everything + /// place depended on". + fn make_route_ws( + tmp: &tempfile::TempDir, + ) -> (Utf8PathBuf, PathBuf, PathBuf) { + let ws = tmp.path().to_path_buf(); + std::fs::write( + ws.join("vw.toml"), + "[workspace]\nname = \"rtws\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let xdc = ws.join("constraints").join("route"); + std::fs::create_dir_all(&xdc).unwrap(); + std::fs::write(xdc.join("route.xdc"), "# route xdc\n").unwrap(); + let place_dcp = ws.join("target/place/top.dcp"); + std::fs::create_dir_all(place_dcp.parent().unwrap()).unwrap(); + std::fs::write(&place_dcp, "").unwrap(); + let route_dcp = ws.join("target/route/top.dcp"); + ( + Utf8PathBuf::from_path_buf(ws).unwrap(), + route_dcp, + place_dcp, + ) + } + + /// Missing checkpoint → stale. Same first-place rule the + /// place / synth caches use. + #[test] + fn route_needs_update_true_when_checkpoint_missing() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, route_dcp, place_dcp) = make_route_ws(&tmp); + assert!(route_needs_update( + &ws, + route_dcp.as_path(), + place_dcp.as_path() + ) + .unwrap()); + } + + /// Fresh manifest → not stale. Verifies the write+check + /// round-trip on the route scope (route XDCs + place DCP). + #[test] + fn route_needs_update_false_when_manifest_matches() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, route_dcp, place_dcp) = make_route_ws(&tmp); + std::fs::create_dir_all(route_dcp.parent().unwrap()).unwrap(); + std::fs::write(&route_dcp, "").unwrap(); + write_route_checkpoint_manifest( + &ws, + route_dcp.as_path(), + place_dcp.as_path(), + ) + .unwrap(); + assert!(!route_needs_update( + &ws, + route_dcp.as_path(), + place_dcp.as_path() + ) + .unwrap()); + } + + /// Editing a route XDC invalidates. Trigger the user + /// controls most directly (tweak a route constraint, + /// re-route should fire). + #[test] + fn route_needs_update_true_when_route_xdc_content_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, route_dcp, place_dcp) = make_route_ws(&tmp); + std::fs::create_dir_all(route_dcp.parent().unwrap()).unwrap(); + std::fs::write(&route_dcp, "").unwrap(); + write_route_checkpoint_manifest( + &ws, + route_dcp.as_path(), + place_dcp.as_path(), + ) + .unwrap(); + std::fs::write( + ws.join("constraints/route/route.xdc"), + "# route xdc updated\n", + ) + .unwrap(); + assert!(route_needs_update( + &ws, + route_dcp.as_path(), + place_dcp.as_path() + ) + .unwrap()); + } + + /// Place re-ran → place DCP content changed → route + /// invalidates. The place DCP is intentionally folded into + /// the route fingerprint as a proxy for "everything place + /// depended on". + #[test] + fn route_needs_update_true_when_place_checkpoint_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, route_dcp, place_dcp) = make_route_ws(&tmp); + std::fs::create_dir_all(route_dcp.parent().unwrap()).unwrap(); + std::fs::write(&route_dcp, "").unwrap(); + write_route_checkpoint_manifest( + &ws, + route_dcp.as_path(), + place_dcp.as_path(), + ) + .unwrap(); + std::fs::write(&place_dcp, "different bytes").unwrap(); + assert!(route_needs_update( + &ws, + route_dcp.as_path(), + place_dcp.as_path() + ) + .unwrap()); + } + + /// Non-route workspace file changes must NOT invalidate the + /// route cache — synth / place XDC edits belong to those + /// stages (and reach route via the DCP-proxy chain). + #[test] + fn route_needs_update_false_when_non_route_xdc_changes() { + let tmp = tempfile::tempdir().unwrap(); + let (ws, route_dcp, place_dcp) = make_route_ws(&tmp); + let place_xdc_dir = ws.join("constraints").join("place"); + std::fs::create_dir_all(&place_xdc_dir).unwrap(); + std::fs::write(place_xdc_dir.join("place.xdc"), "# place\n").unwrap(); + std::fs::create_dir_all(route_dcp.parent().unwrap()).unwrap(); + std::fs::write(&route_dcp, "").unwrap(); + write_route_checkpoint_manifest( + &ws, + route_dcp.as_path(), + place_dcp.as_path(), + ) + .unwrap(); + std::fs::write(place_xdc_dir.join("place.xdc"), "# place updated\n") + .unwrap(); + assert!(!route_needs_update( + &ws, + route_dcp.as_path(), + place_dcp.as_path() + ) + .unwrap()); + } } diff --git a/vw-lib/src/parts.rs b/vw-lib/src/parts.rs new file mode 100644 index 0000000..dc3dd74 --- /dev/null +++ b/vw-lib/src/parts.rs @@ -0,0 +1,401 @@ +//! Enumerate FPGA parts from a local Vivado install without launching +//! Vivado (which takes ~30s just to start up). +//! +//! The plaintext catalog lives at +//! `/data/parts/xilinx//public/ibis/FileMap.txt` +//! for every architecture Xilinx has published this file for +//! (7-series, Zynq-7000/UltraScale+, all UltraScale+, and Versal in +//! 2025.1 — pre-7-series legacy families ship only encrypted +//! `DeviceParts.xml`). Each FileMap.txt has a `pkg-file-mapping +//! { ... }` block with rows shaped ` .pkg`. +//! Translating the id from underscores to dashes yields the canonical +//! Vivado part id (`xcvp1202_vsva2785_2MP_e_S_` → +//! `xcvp1202-vsva2785-2MP-e-S`; `xa7s100_fgga484_1I` → +//! `xa7s100-fgga484-1I`). UltraScale+/Versal ids carry a trailing +//! separator underscore that must be stripped; 7-series ids don't. +//! Rows whose package column is the literal `In_Development` are +//! placeholders and get filtered out. + +use camino::Utf8PathBuf; +use std::path::PathBuf; + +/// Broad device series used for filtering in the picker chip. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PartSeries { + Versal, + KintexUP, + VirtexUP, + ArtixUP, + SpartanUP, + /// Zynq UltraScale+ MPSoC and RFSoC — same `zu` prefix, both + /// live under this bucket. Users needing MPSoC vs RFSoC narrow + /// via fuzzy search (`eg`/`ev` = MPSoC, `dr` = RFSoC). + ZynqUP, + Artix7, + Kintex7, + Virtex7, + Spartan7, + Zynq7, + Other, +} + +impl PartSeries { + pub fn label(self) -> &'static str { + match self { + Self::Versal => "Versal", + Self::KintexUP => "Kintex UP", + Self::VirtexUP => "Virtex UP", + Self::ArtixUP => "Artix UP", + Self::SpartanUP => "Spartan UP", + Self::ZynqUP => "Zynq UP", + Self::Artix7 => "Artix-7", + Self::Kintex7 => "Kintex-7", + Self::Virtex7 => "Virtex-7", + Self::Spartan7 => "Spartan-7", + Self::Zynq7 => "Zynq-7000", + Self::Other => "Other", + } + } + + /// Every variant, in the order the picker's Tab chip cycles + /// through them. Newer/bigger families first so common cases + /// need fewer Tab presses. + pub fn all() -> [PartSeries; 12] { + [ + Self::Versal, + Self::ZynqUP, + Self::KintexUP, + Self::VirtexUP, + Self::ArtixUP, + Self::SpartanUP, + Self::Artix7, + Self::Kintex7, + Self::Virtex7, + Self::Spartan7, + Self::Zynq7, + Self::Other, + ] + } + + /// Classify a canonical part id (dashed form) into a series + /// based on its `x[caq][r?]` prefix. Vivado embeds the + /// series in the first 3-4 chars of every part id. + fn from_part_id(part_id: &str) -> Self { + let bytes = part_id.as_bytes(); + if bytes.len() < 3 || bytes[0] != b'x' { + return Self::Other; + } + // Strip the grade prefix (`xa`/`xc`/`xq`/`xqr`) to leave + // just the family suffix. `xqr` (radiation-tolerant) comes + // first so `xq` doesn't shadow it. + let after = if bytes[1] == b'q' && bytes.get(2) == Some(&b'r') { + &bytes[3..] + } else if matches!(bytes[1], b'a' | b'c' | b'q') { + &bytes[2..] + } else { + return Self::Other; + }; + // UltraScale+ / Versal — 2-letter family code. + if let Some(&a) = after.first() { + if let Some(&b) = after.get(1) { + let series = match (a, b) { + (b'v', b'p' | b'c' | b'm' | b'e' | b'h' | b'r' | b'n') => { + Some(Self::Versal) + } + (b'k', b'u') => Some(Self::KintexUP), + (b'v', b'u') => Some(Self::VirtexUP), + (b'a', b'u') => Some(Self::ArtixUP), + (b's', b'u') => Some(Self::SpartanUP), + (b'z', b'u') => Some(Self::ZynqUP), + // `xcv80` is a Versal outlier — device-name-only + // classification, no 2-letter family shortcut. + _ => None, + }; + if let Some(s) = series { + return s; + } + } + // 7-series — `7` pattern after the grade. + if a == b'7' { + return match after.get(1) { + Some(&b'a') => Self::Artix7, + Some(&b'k') => Self::Kintex7, + Some(&b'v') => Self::Virtex7, + Some(&b's') => Self::Spartan7, + Some(&b'z') => Self::Zynq7, + _ => Self::Other, + }; + } + } + // `xcv80…` (Versal outlier with 1-letter code). + if after.starts_with(b"v80") { + return Self::Versal; + } + Self::Other + } + + /// Fallback classifier used when `from_part_id` returns `Other`. + /// Every FileMap.txt lives under a Xilinx architecture directory + /// name that identifies the family authoritatively — new part + /// prefixes (Kria SOMs `xck2*`, Versal Series 2 `xc2v*`, etc.) + /// still get classified correctly without needing the prefix + /// table to keep up. + fn from_arch_dir(dir: &str) -> Option { + match dir { + "versal" => Some(Self::Versal), + "kintexuplus" => Some(Self::KintexUP), + "virtexuplus" | "virtexuplus58g" | "virtexuplusHBM" => { + Some(Self::VirtexUP) + } + "spartanuplus" => Some(Self::SpartanUP), + "zynquplus" | "zynquplusRFSOC" => Some(Self::ZynqUP), + "artix7" => Some(Self::Artix7), + "kintex7" => Some(Self::Kintex7), + "virtex7" => Some(Self::Virtex7), + "spartan7" => Some(Self::Spartan7), + "zynq" => Some(Self::Zynq7), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PartEntry { + /// Canonical Vivado part id (e.g. `xcvp1202-vsva2785-2MP-e-S`). + pub id: String, + pub series: PartSeries, +} + +/// Resolve the Vivado install directory (containing `bin/`, `data/`, +/// `settings64.sh`). Preference order matches Xilinx tooling: env +/// override first, then whatever `vivado` on `$PATH` resolves to. +pub fn find_vivado_install() -> Option { + if let Ok(env) = std::env::var("XILINX_VIVADO") { + let p = Utf8PathBuf::from(env); + if p.join("data/parts").exists() { + return Some(p); + } + } + let output = std::process::Command::new("which") + .arg("vivado") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let path = String::from_utf8(output.stdout).ok()?; + let path = PathBuf::from(path.trim()); + let resolved = std::fs::canonicalize(&path).ok()?; + // `/bin/vivado` → `` + let install = resolved.parent()?.parent()?; + Utf8PathBuf::from_path_buf(install.to_path_buf()).ok() +} + +/// Walk every plaintext `FileMap.txt` under the install and pull +/// out every shipping part id. Returns them deduplicated and sorted +/// by canonical id — a part may appear in multiple FileMaps when +/// architectures overlap (Artix UP lives in the `kintexuplus` dir +/// despite having its own series prefix). +pub fn enumerate_parts(install: &Utf8PathBuf) -> Vec { + let arch_root = install.join("data/parts/xilinx"); + let Ok(entries) = std::fs::read_dir(&arch_root) else { + return Vec::new(); + }; + let mut all: Vec = Vec::new(); + for entry in entries.flatten() { + let dir_name = entry.file_name(); + let dir_name = dir_name.to_string_lossy(); + let filemap = entry.path().join("public/ibis/FileMap.txt"); + let Ok(contents) = std::fs::read_to_string(&filemap) else { + continue; + }; + let dir_fallback = PartSeries::from_arch_dir(&dir_name); + for mut part in parse_filemap(&contents) { + if part.series == PartSeries::Other { + if let Some(fallback) = dir_fallback { + part.series = fallback; + } + } + all.push(part); + } + } + all.sort_by(|a, b| a.id.cmp(&b.id)); + all.dedup_by(|a, b| a.id == b.id); + all +} + +/// Parse the `pkg-file-mapping { ... }` block out of a FileMap.txt +/// string. Handles both UltraScale+/Versal id shape (trailing +/// separator underscore that gets stripped) and 7-series shape +/// (no trailing underscore). +pub fn parse_filemap(contents: &str) -> Vec { + let mut out = Vec::new(); + let mut in_pkg_block = false; + for line in contents.lines() { + let trimmed = line.trim_start(); + if !in_pkg_block { + if trimmed.starts_with("pkg-file-mapping") { + in_pkg_block = true; + } + continue; + } + if trimmed.starts_with('}') { + break; + } + // Row shape: ` \s+.pkg`. Placeholder rows + // use `In_Development` as the package name — skip those. + let mut it = trimmed.split_whitespace(); + let (Some(ident), Some(pkg)) = (it.next(), it.next()) else { + continue; + }; + if pkg == "In_Development" { + continue; + } + let id = ident.strip_suffix('_').unwrap_or(ident).replace('_', "-"); + if id.is_empty() { + continue; + } + let series = PartSeries::from_part_id(&id); + out.push(PartEntry { id, series }); + } + out +} + +/// Sub-string, case-insensitive fuzzy filter used by the picker. +/// Splits the query on whitespace; every token must appear in the +/// part id (in any order, any position) for the entry to match. +/// Empty query returns everything. +pub fn matches_query(id: &str, query: &str) -> bool { + let hay = id.to_ascii_lowercase(); + query + .split_whitespace() + .all(|tok| hay.contains(&tok.to_ascii_lowercase())) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIXTURE_ULTRASCALE_PLUS: &str = "\ +# Header comment +# +ibs-file-mapping { + irrelevant ignored.ibs +} + +pkg-file-mapping { + xcvp1202_vsva2785_2MP_e_S_ xcvp1202_vsva2785.pkg + xcvp1202_vsva2785_1LP_i_L_ xcvp1202_vsva2785.pkg + xcau10p_ffvb676_1_e_ xcau10p_ffvb676.pkg + xcku15p_CIV_ffva1156_2_e_ In_Development + xcvu9p_flga2104_2_e_ xcvu9p_flga2104.pkg + xcsu35p_swra1493_1_e_ xcsu35p_swra1493.pkg + xczu9eg_ffvc900_2_e_ xczu9eg_ffvc900.pkg + xczu21dr_ffvd1156_2_e_ xczu21dr_ffvd1156.pkg + xqku15p_ffva1156_2_e_ xqku15p_ffva1156.pkg + malformed_row_no_pkg +} + +some-other-block { + xzz_should_not_appear foo.pkg +} +"; + + // 7-series: no trailing `_` on identifiers, speed grade like `_1I`. + const FIXTURE_7SERIES: &str = "\ +pkg-file-mapping { + xc7a100t_csg324_1 xc7a100t_csg324.pkg + xa7a100t_csg324_1I xc7a100t_csg324.pkg + xc7k325t_ffg676_2 xc7k325t_ffg676.pkg + xc7v2000t_flg1925_1 xc7v2000t_flg1925.pkg + xc7s100_fgga484_1 xc7s100_fgga484.pkg + xc7z020_clg484_1 xc7z020_clg484.pkg +} +"; + + #[test] + fn parses_pkg_block_and_translates_ids_ultrascale_plus() { + let parts = parse_filemap(FIXTURE_ULTRASCALE_PLUS); + let ids: Vec<&str> = parts.iter().map(|p| p.id.as_str()).collect(); + assert_eq!( + ids, + vec![ + "xcvp1202-vsva2785-2MP-e-S", + "xcvp1202-vsva2785-1LP-i-L", + "xcau10p-ffvb676-1-e", + "xcvu9p-flga2104-2-e", + "xcsu35p-swra1493-1-e", + "xczu9eg-ffvc900-2-e", + "xczu21dr-ffvd1156-2-e", + "xqku15p-ffva1156-2-e", + ] + ); + } + + #[test] + fn parses_pkg_block_and_translates_ids_7series() { + let parts = parse_filemap(FIXTURE_7SERIES); + let ids: Vec<&str> = parts.iter().map(|p| p.id.as_str()).collect(); + assert_eq!( + ids, + vec![ + "xc7a100t-csg324-1", + "xa7a100t-csg324-1I", + "xc7k325t-ffg676-2", + "xc7v2000t-flg1925-1", + "xc7s100-fgga484-1", + "xc7z020-clg484-1", + ] + ); + } + + #[test] + fn skips_in_development_and_out_of_block_rows() { + let parts = parse_filemap(FIXTURE_ULTRASCALE_PLUS); + assert!(parts.iter().all(|p| !p.id.contains("CIV"))); + assert!(parts.iter().all(|p| !p.id.contains("xzz"))); + } + + #[test] + fn classifies_series() { + let cases = [ + ("xcvp1202-vsva2785-2MP-e-S", PartSeries::Versal), + ("xcvc1902-vsva2197-2MP-e-S", PartSeries::Versal), + ("xcvm1502-vsva2197-2MP-e-S", PartSeries::Versal), + ("xcvh1782-vsva3697-2MP-e-S", PartSeries::Versal), + ("xave2002-nsvg1369-2LP-e-S", PartSeries::Versal), + ("xqrvc1902-vsva2197-1MP-i-L", PartSeries::Versal), + ("xcv80-lsva4737-2LHP-i-S", PartSeries::Versal), + ("xcku15p-ffva1156-2-e", PartSeries::KintexUP), + ("xqku15p-ffva1156-2-e", PartSeries::KintexUP), + ("xcvu9p-flga2104-2-e", PartSeries::VirtexUP), + ("xcau10p-ffvb676-1-e", PartSeries::ArtixUP), + ("xaau10p-ffvb676-1-e", PartSeries::ArtixUP), + ("xcsu35p-swra1493-1-e", PartSeries::SpartanUP), + ("xczu9eg-ffvc900-2-e", PartSeries::ZynqUP), + ("xczu21dr-ffvd1156-2-e", PartSeries::ZynqUP), + ("xc7a100t-csg324-1", PartSeries::Artix7), + ("xa7a100t-csg324-1I", PartSeries::Artix7), + ("xq7k325t-ffg676-2I", PartSeries::Kintex7), + ("xc7v2000t-flg1925-1", PartSeries::Virtex7), + ("xc7s100-fgga484-1", PartSeries::Spartan7), + ("xc7z020-clg484-1", PartSeries::Zynq7), + ("bogus", PartSeries::Other), + ]; + for (id, expected) in cases { + assert_eq!( + PartSeries::from_part_id(id), + expected, + "misclassified {id}" + ); + } + } + + #[test] + fn fuzzy_matches_across_tokens_case_insensitive() { + assert!(matches_query("xcvp1202-vsva2785-2MP-e-S", "vp1202")); + assert!(matches_query("xcvp1202-vsva2785-2MP-e-S", "VSVA 2mp")); + assert!(matches_query("xcvp1202-vsva2785-2MP-e-S", "")); + assert!(!matches_query("xcvp1202-vsva2785-2MP-e-S", "vp1202 nope")); + } +} diff --git a/vw-lib/src/sim/bridge.rs b/vw-lib/src/sim/bridge.rs index f432b7f..231ea38 100644 --- a/vw-lib/src/sim/bridge.rs +++ b/vw-lib/src/sim/bridge.rs @@ -109,7 +109,17 @@ fn expand_tilde(path: &str) -> String { } fn write_file(path: impl AsRef, content: &str) -> crate::Result<()> { - let mut f = fs::File::create(path.as_ref())?; + let path = path.as_ref(); + // Content-aware: skip the write when the file already holds exactly + // this content. Scaffolding runs before every `vw bench` (so a + // clean checkout self-heals), and an unconditional truncate+write + // would bump the mtime of `Cargo.toml`/`build.rs`/generated sources + // every run — forcing cargo to rebuild the bridge crate each time. + // Leaving unchanged files untouched keeps the incremental build hot. + if fs::read_to_string(path).is_ok_and(|existing| existing == content) { + return Ok(()); + } + let mut f = fs::File::create(path)?; f.write_all(content.as_bytes())?; Ok(()) } diff --git a/vw-lib/src/sim/bridge_assets/xyce.rs b/vw-lib/src/sim/bridge_assets/xyce.rs index 8c82c11..ee9c8f2 100644 --- a/vw-lib/src/sim/bridge_assets/xyce.rs +++ b/vw-lib/src/sim/bridge_assets/xyce.rs @@ -46,8 +46,8 @@ impl Xyce { return Err("xyce_open returned null".into()); } - // Set working directory to the netlist's parent directory so that - // relative file references (e.g., TABLE("main.dat")) resolve correctly. + // Keep Xyce's working directory at the netlist's parent so relative + // file references (e.g., TABLE("main.dat")) resolve correctly. if let Some(parent) = netlist_path.parent() { let dir = CString::new(parent.to_str().unwrap()).unwrap(); unsafe { xyce_set_working_directory(&mut ptr, dir.as_ptr()) }; @@ -60,10 +60,22 @@ impl Xyce { let prog = CString::new("Xyce").unwrap(); let netlist_c = CString::new(netlist).unwrap(); - let mut argv: Vec<*mut c_char> = vec![ - prog.as_ptr() as *mut c_char, - netlist_c.as_ptr() as *mut c_char, - ]; + // When the harness (vw) specifies an output directory, direct Xyce's + // output files there via `-o /` rather than + // letting them land next to the netlist. CStrings must outlive + // `xyce_initialize`, so bind them in this scope. + let dash_o = CString::new("-o").unwrap(); + let out_base_c = rust_cosim::output_dir().and_then(|dir| { + let base = dir.join(netlist_path.file_name()?); + CString::new(base.to_str()?).ok() + }); + + let mut argv: Vec<*mut c_char> = vec![prog.as_ptr() as *mut c_char]; + if let Some(out_base_c) = out_base_c.as_ref() { + argv.push(dash_o.as_ptr() as *mut c_char); + argv.push(out_base_c.as_ptr() as *mut c_char); + } + argv.push(netlist_c.as_ptr() as *mut c_char); let status = unsafe { xyce_initialize(&mut ptr, argv.len() as c_int, argv.as_mut_ptr()) }; diff --git a/vw-lib/src/sim/mod.rs b/vw-lib/src/sim/mod.rs index 734fd19..a026174 100644 --- a/vw-lib/src/sim/mod.rs +++ b/vw-lib/src/sim/mod.rs @@ -18,7 +18,7 @@ use camino::Utf8Path; use crate::nvc_helpers::{run_nvc_analysis, run_nvc_cosim, run_nvc_elab}; use crate::{ - analyze_ext_libraries, find_referenced_files, load_existing_vhdl_ls_config, + analyze_ext_libraries, find_referenced_files, render_vhdl_ls_config, sort_files_by_dependencies, FileCache, MistConfig, RecordProcessor, ToolsConfig, VhdlStandard, VwError, }; @@ -106,18 +106,20 @@ pub async fn run_analog_test( mist_config: &MistConfig, _tools: &Option, vhdl_std: VhdlStandard, + build_dir: &str, ) -> crate::Result<()> { - let vhdl_ls_config = load_existing_vhdl_ls_config(workspace_dir)?; + let vhdl_ls_config = render_vhdl_ls_config(workspace_dir, None, false)?; let mut processor = RecordProcessor::new(vhdl_std); let mut cache = FileCache::new(); - fs::create_dir_all(crate::BUILD_DIR)?; + fs::create_dir_all(build_dir)?; // Analyze external libraries analyze_ext_libraries( &vhdl_ls_config, &mut processor, vhdl_std, + build_dir, &mut cache, ) .await?; @@ -156,30 +158,36 @@ pub async fn run_analog_test( files.push(entity_file.to_string_lossy().to_string()); // Compile VHDL - run_nvc_analysis(vhdl_std, crate::BUILD_DIR, "work", &files, false).await?; - run_nvc_elab(vhdl_std, crate::BUILD_DIR, "work", entity_name, false) - .await?; + run_nvc_analysis(vhdl_std, build_dir, "work", &files, false).await?; + run_nvc_elab(vhdl_std, build_dir, "work", entity_name, false).await?; // Build the bridge crate let bridge_lib = build_bridge_library(bench_dir.as_std_path(), name).await?; let bridge_lib_str = bridge_lib.to_string_lossy().to_string(); + // Per-bench output directory under target/. The Xyce bridge writes its + // `.prn` straight here (via rust_cosim::output_dir() -> Xyce's `-o` flag), + // so nothing is copied out of the source tree afterward. + let output_dir = crate::bench_output_dir(workspace_dir, name); + fs::create_dir_all(&output_dir)?; + let output_dir_abs = output_dir + .canonicalize_utf8() + .unwrap_or_else(|_| output_dir.clone()); + // Run co-simulation run_nvc_cosim( vhdl_std, - crate::BUILD_DIR, + build_dir, "work", entity_name, &bridge_lib_str, + output_dir_abs.as_str(), false, ) .await?; - // Collect output - let output_dir = bench_dir.as_std_path().join("output"); - fs::create_dir_all(&output_dir)?; - + // Xyce has written .prn into output_dir; generate plots from it. let netlist_path = bench_dir.as_std_path().join(&mist_config.netlist); let prn_name = netlist_path .file_name() @@ -187,28 +195,16 @@ pub async fn run_analog_test( .to_string_lossy() .to_string() + ".prn"; - let prn_source = netlist_path.with_extension( - netlist_path - .extension() - .unwrap_or_default() - .to_string_lossy() - .to_string() - + ".prn", - ); - - if prn_source.exists() { - let prn_dest = output_dir.join(&prn_name); - fs::copy(&prn_source, &prn_dest)?; - } - // Auto-generate plots #[cfg(feature = "plot")] { - let prn_path = output_dir.join(&prn_name); + let prn_path = output_dir.as_std_path().join(&prn_name); if prn_path.exists() { - if let Err(e) = - plot::generate_plots(&netlist_path, &prn_path, &output_dir) - { + if let Err(e) = plot::generate_plots( + &netlist_path, + &prn_path, + output_dir.as_std_path(), + ) { eprintln!("Warning: plot generation failed: {e}"); } } diff --git a/vw-openapi-manager/Cargo.toml b/vw-openapi-manager/Cargo.toml new file mode 100644 index 0000000..aae58f4 --- /dev/null +++ b/vw-openapi-manager/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "vw-openapi-manager" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Manages the checked-in OpenAPI documents for the VW service APIs" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[[bin]] +name = "vw-openapi-manager" +path = "src/main.rs" + +[dependencies] +vw-api = { path = "../vw-api" } +vw-sync-api = { path = "../vw-sync-api" } +dropshot-api-manager = "0.7.2" +dropshot-api-manager-types = "0.7.2" +anyhow = "1.0" +camino.workspace = true +clap.workspace = true + +[dev-dependencies] +dropshot-api-manager = "0.7.2" diff --git a/vw-openapi-manager/src/lib.rs b/vw-openapi-manager/src/lib.rs new file mode 100644 index 0000000..0b4071c --- /dev/null +++ b/vw-openapi-manager/src/lib.rs @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Integration point between the vw service API traits and the +//! [Dropshot API manager](https://crates.io/crates/dropshot-api-manager). +//! +//! The manager owns the OpenAPI documents under `openapi/` at the root of the +//! repository: `cargo xtask openapi generate` writes them from the API traits in +//! `vw-api`, and `cargo xtask openapi check` fails if what is on disk no longer +//! matches. That check also runs as a test in this crate, so a stale document +//! turns up in `cargo test` rather than in a client that has quietly drifted +//! from the service. +//! +//! `vw-api-client` generates its progenitor clients from the `-latest.json` +//! symlink the manager maintains for each API. + +use anyhow::Context; +use camino::Utf8PathBuf; +use dropshot_api_manager::{Environment, ManagedApiConfig, ManagedApis}; +use dropshot_api_manager_types::{ManagedApiMetadata, Versions}; + +/// How a developer invokes this binary. The manager quotes it back in its own +/// guidance, so it needs to match the alias in `.cargo/config.toml`. +const COMMAND: &str = "cargo xtask openapi"; + +/// Where the managed documents live, relative to the repository root. +const OPENAPI_DIR: &str = "openapi"; + +/// The environment the manager runs in. +pub fn environment() -> anyhow::Result { + Environment::new(COMMAND, repo_root()?, OPENAPI_DIR) +} + +/// Every OpenAPI document the manager is responsible for. +/// +/// Both APIs are versioned rather than lockstep: `vw-api` declares its +/// supported versions with `api_versions!`, and clients out in the world will +/// not be upgraded in lockstep with the service. +pub fn all_apis() -> anyhow::Result { + ManagedApis::new(vec![ + ManagedApiConfig { + ident: "vw-user-api", + versions: Versions::new_versioned(vw_api::supported_versions()), + title: "VW user API", + metadata: ManagedApiMetadata { + description: Some( + "Manage your own vw build environments. Callers are \ + identified by a Github access token.", + ), + ..Default::default() + }, + api_description: vw_api::vw_user_api_mod::stub_api_description, + }, + ManagedApiConfig { + ident: "vw-admin-api", + versions: Versions::new_versioned(vw_api::supported_versions()), + title: "VW admin API", + metadata: ManagedApiMetadata { + description: Some( + "Manage every user's vw build environments. Restricted to \ + the operators named in the service's --admin-users \ + argument.", + ), + ..Default::default() + }, + api_description: vw_api::vw_admin_api_mod::stub_api_description, + }, + ManagedApiConfig { + ident: "vw-sync-api", + versions: Versions::new_versioned(vw_sync_api::supported_versions()), + title: "VW agent API", + metadata: ManagedApiMetadata { + description: Some( + "Receive source on a build instance. Reachable only from \ + vw-svc over the rack's internal network.", + ), + ..Default::default() + }, + api_description: vw_sync_api::vw_sync_api_mod::stub_api_description, + }, + ]) +} + +/// The root of the repository, one directory up from this crate. +fn repo_root() -> anyhow::Result { + let manifest_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")); + Ok(manifest_dir + .parent() + .with_context(|| { + format!( + "{manifest_dir} has no parent directory to use as the \ + repository root" + ) + })? + .to_owned()) +} diff --git a/vw-openapi-manager/src/main.rs b/vw-openapi-manager/src/main.rs new file mode 100644 index 0000000..f804f54 --- /dev/null +++ b/vw-openapi-manager/src/main.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Manages the checked-in OpenAPI documents for the vw service APIs. +//! +//! Run it through the workspace alias: +//! +//! ```text +//! cargo xtask openapi list # what documents are managed +//! cargo xtask openapi generate # write them from the API traits +//! cargo xtask openapi check # fail if what is on disk is out of date +//! ``` + +use std::process::ExitCode; + +use clap::Parser; +use dropshot_api_manager::App; + +fn main() -> anyhow::Result { + let app = App::parse(); + Ok(app.exec( + &vw_openapi_manager::environment()?, + &vw_openapi_manager::all_apis()?, + )) +} diff --git a/vw-openapi-manager/tests/openapi.rs b/vw-openapi-manager/tests/openapi.rs new file mode 100644 index 0000000..8aca417 --- /dev/null +++ b/vw-openapi-manager/tests/openapi.rs @@ -0,0 +1,24 @@ +// The OpenAPI documents under `openapi/` are what `vw-api-client` generates +// its progenitor clients from. Nothing forces them to be regenerated when an +// endpoint changes, so an out of date document would quietly leave the client +// describing an API the service no longer serves. Catch that here rather than +// at runtime. + +use dropshot_api_manager::test_util::{check_apis_up_to_date, CheckResult}; + +#[test] +fn openapi_documents_are_up_to_date() { + let environment = + vw_openapi_manager::environment().expect("resolve environment"); + let apis = vw_openapi_manager::all_apis().expect("collect managed apis"); + + match check_apis_up_to_date(&environment, &apis).expect("run check") { + CheckResult::Success => {} + CheckResult::NeedsUpdate => { + panic!("openapi documents are out of date; run `cargo xtask openapi generate`") + } + CheckResult::Failures => { + panic!("openapi documents failed validation; see the output above") + } + } +} diff --git a/vw-quote/Cargo.toml b/vw-quote/Cargo.toml new file mode 100644 index 0000000..f01e952 --- /dev/null +++ b/vw-quote/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vw-quote" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +syn = { version = "2", features = ["full"] } +quote = "1" + +[dev-dependencies] +vw-htcl = { path = "../vw-htcl" } diff --git a/vw-quote/src/lib.rs b/vw-quote/src/lib.rs new file mode 100644 index 0000000..b724e21 --- /dev/null +++ b/vw-quote/src/lib.rs @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `quote_htcl!` — generate htcl source code with interpolation. +//! +//! Analogous to the `quote` crate, but for htcl. Takes a string +//! literal of htcl source containing `#(expr)` interpolation markers +//! and produces a `String` of well-formed htcl at runtime. Each +//! interpolated value is passed through [`vw_htcl::emit::ToHtcl`] to +//! choose the right word form (bare vs. quoted vs. …) so generated +//! code is always parseable. +//! +//! ## Why a string literal? +//! +//! Rust's `TokenStream` doesn't preserve newlines, and newlines +//! terminate htcl commands — so a token-walking macro that reads +//! `quote_htcl! { proc x { … } { … } }` directly can't tell where one +//! statement ends and the next begins. Taking the input as a string +//! literal keeps the source text exact at zero cost in ergonomics. +//! +//! ## Syntax +//! +//! - `#(expr)` — interpolation slot. `expr` is parsed as a Rust +//! expression at macro time and emitted via +//! `vw_htcl::emit::ToHtcl::to_htcl(&expr)`. +//! - Anything else is literal htcl, copied verbatim except that `{` +//! and `}` in the template don't need any escaping (the macro +//! handles `format!` quoting for you). +//! +//! ## Example +//! +//! ```ignore +//! use vw_quote::quote_htcl; +//! let name = "greet"; +//! let who = "world"; +//! let s = quote_htcl!("\ +//! proc #(name) { +//! #(who) +//! } { puts hi } +//! "); +//! // s == "proc greet {\n world\n} { puts hi }\n" +//! // (the interpolated values get `ToHtcl`-formatted; here both are +//! // bare identifiers, so they emit as-is.) +//! ``` + +use proc_macro::TokenStream; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{parse_macro_input, Expr, LitStr}; + +#[proc_macro] +pub fn quote_htcl(input: TokenStream) -> TokenStream { + expand(input, Dialect::Htcl) +} + +/// Same template grammar as [`quote_htcl!`], but routes interpolated +/// values through [`vw_htcl::emit::ToTcl`] and produces pure Tcl +/// (no htcl-specific attribute handling). Use for compiler-emitted +/// runtime helpers — `repr` procs, `kwargs` shim glue, anything that +/// lives in the Tcl interpreter and should never look like htcl. +/// +/// The split exists so future Tcl-only behavior (typed `Tcl_Obj` +/// handle quoting, etc.) can land on `ToTcl` without changing +/// `quote_htcl!`'s contract. +#[proc_macro] +pub fn quote_tcl(input: TokenStream) -> TokenStream { + expand(input, Dialect::Tcl) +} + +/// Which interpolation trait the macro routes through. The template +/// parsing is shared verbatim — the only thing that differs is the +/// trait + method name used in the generated `format!` arguments. +#[derive(Clone, Copy)] +enum Dialect { + Htcl, + Tcl, +} + +fn expand(input: TokenStream, dialect: Dialect) -> TokenStream { + let lit = parse_macro_input!(input as LitStr); + let template_text = lit.value(); + let lit_span = lit.span(); + + let (template, exprs) = match split_template(&template_text, lit_span) { + Ok(parts) => parts, + Err(e) => return e.to_compile_error().into(), + }; + + // Escape literal `{`/`}` so the format string parser leaves them + // alone; replace each `#(…)` site with a positional placeholder. + let format_string = render_format_string(&template, exprs.len()); + let format_lit = LitStr::new(&format_string, Span::call_site()); + + let exprs: Vec = + exprs.into_iter().map(|e| e.to_token_stream()).collect(); + + let out = match dialect { + Dialect::Htcl => quote! {{ + // Bring the trait into scope so `(&expr).to_htcl()` resolves + // without the caller needing to import it. + #[allow(unused_imports)] + use ::vw_htcl::emit::ToHtcl as _; + ::std::format!( + #format_lit, + #( (&{ #exprs }).to_htcl() ),* + ) + }}, + Dialect::Tcl => quote! {{ + #[allow(unused_imports)] + use ::vw_htcl::emit::ToTcl as _; + ::std::format!( + #format_lit, + #( (&{ #exprs }).to_tcl() ),* + ) + }}, + }; + out.into() +} + +// --- template parsing ------------------------------------------------------ + +/// One piece of the parsed template. +enum Piece { + /// Verbatim text from the template. + Text(String), + /// An interpolation site; index into the parallel `exprs` Vec. + Interp, +} + +/// Split the template string into alternating text / interpolation +/// pieces, parsing each `#(…)` body as a Rust expression. +fn split_template( + text: &str, + lit_span: Span, +) -> syn::Result<(Vec, Vec)> { + let mut pieces = Vec::new(); + let mut exprs = Vec::new(); + let mut buf = String::new(); + + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let c = bytes[i]; + // Allow `##` doc comments and `#` plain comments to pass + // through unmolested: interpolation is `#(...)`, so we only + // engage when `#` is immediately followed by `(`. + if c == b'#' && i + 1 < bytes.len() && bytes[i + 1] == b'(' { + if !buf.is_empty() { + pieces.push(Piece::Text(std::mem::take(&mut buf))); + } + let body_end = match find_matching_paren(bytes, i + 1) { + Some(end) => end, + None => { + return Err(syn::Error::new( + lit_span, + "unterminated `#(...)` in quote_htcl! template", + )); + } + }; + // bytes[i+2 .. body_end] is the inside of the parens. + let expr_src = &text[i + 2..body_end]; + let expr: Expr = syn::parse_str(expr_src).map_err(|e| { + syn::Error::new( + lit_span, + format!( + "could not parse interpolation `{expr_src}` as a \ + Rust expression: {e}" + ), + ) + })?; + exprs.push(expr); + pieces.push(Piece::Interp); + i = body_end + 1; + continue; + } + // Push this char (handle UTF-8 boundary by stepping a full + // char rather than a byte). + let ch_start = i; + // Safe because `i` is always at a UTF-8 boundary (we only + // advance by full chars or past ASCII chars we recognized). + let ch = text[ch_start..].chars().next().unwrap(); + buf.push(ch); + i += ch.len_utf8(); + } + if !buf.is_empty() { + pieces.push(Piece::Text(buf)); + } + Ok((pieces, exprs)) +} + +/// Find the byte index of the `)` that matches an opening `(` at +/// `bytes[open]`. Tracks nested parens. Returns `None` on unterminated. +fn find_matching_paren(bytes: &[u8], open: usize) -> Option { + debug_assert_eq!(bytes[open], b'('); + let mut depth = 1usize; + let mut i = open + 1; + while i < bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + i += 1; + } + None +} + +/// Build the format string passed to `std::format!`, with literal +/// `{`/`}` doubled and `{0}` / `{1}` / … placeholders inserted at each +/// interpolation site. +fn render_format_string(pieces: &[Piece], n_interps: usize) -> String { + let mut out = String::new(); + let mut next_interp = 0usize; + let _ = n_interps; + for piece in pieces { + match piece { + Piece::Text(s) => { + for c in s.chars() { + match c { + '{' => out.push_str("{{"), + '}' => out.push_str("}}"), + other => out.push(other), + } + } + } + Piece::Interp => { + use std::fmt::Write; + write!(out, "{{{}}}", next_interp).unwrap(); + next_interp += 1; + } + } + } + out +} + +trait ToTokenStream { + fn to_token_stream(&self) -> TokenStream2; +} +impl ToTokenStream for Expr { + fn to_token_stream(&self) -> TokenStream2 { + quote! { #self } + } +} diff --git a/vw-quote/tests/basic.rs b/vw-quote/tests/basic.rs new file mode 100644 index 0000000..1e0866f --- /dev/null +++ b/vw-quote/tests/basic.rs @@ -0,0 +1,140 @@ +// Integration tests for `quote_htcl!` and `quote_tcl!`. A proc-macro +// crate can't use its own macros from within `src/`, so these tests +// live in `tests/` and pull `vw-quote` and `vw-htcl` as dev-deps. + +use vw_quote::{quote_htcl, quote_tcl}; + +#[test] +fn literal_passthrough() { + let s = quote_htcl!("puts hi\n"); + assert_eq!(s, "puts hi\n"); +} + +#[test] +fn simple_ident_interpolation() { + let name = "greet"; + let s = quote_htcl!("proc #(name) {} { puts hi }\n"); + assert_eq!(s, "proc greet {} { puts hi }\n"); +} + +#[test] +fn expression_interpolation() { + let width = 16u32; + let s = quote_htcl!("set w #(width)\n"); + assert_eq!(s, "set w 16\n"); +} + +#[test] +fn values_needing_quoting_get_quoted() { + let msg = "hello world"; + let s = quote_htcl!("puts #(msg)\n"); + assert_eq!(s, "puts \"hello world\"\n"); +} + +#[test] +fn dollar_in_value_is_escaped() { + let s = "$x"; + let out = quote_htcl!("puts #(s)\n"); + // The value `$x` has special chars, so it gets quoted with + // `\$` escaped — preserving it as the literal text "$x" at runtime. + assert_eq!(out, "puts \"\\$x\"\n"); +} + +#[test] +fn braces_in_template_pass_through() { + let name = "f"; + let s = quote_htcl!("proc #(name) {} {\n puts hi\n}\n"); + assert_eq!(s, "proc f {} {\n puts hi\n}\n"); +} + +#[test] +fn doc_comment_passes_through() { + let s = quote_htcl!("## A doc comment.\nputs hi\n"); + assert_eq!(s, "## A doc comment.\nputs hi\n"); +} + +#[test] +fn multiple_interpolations() { + let name = "greet"; + let arg = "world"; + let s = quote_htcl!("proc #(name) { #(arg) } { puts hi }\n"); + assert_eq!(s, "proc greet { world } { puts hi }\n"); +} + +#[test] +fn method_call_in_interpolation() { + struct P { + name: &'static str, + } + let p = P { name: "greet" }; + let s = quote_htcl!("proc #(p.name) {} { }\n"); + assert_eq!(s, "proc greet {} { }\n"); +} + +#[test] +fn output_parses_as_valid_htcl() { + let name = "greet"; + let msg = "hi there"; + let s = quote_htcl!("proc #(name) {} {\n puts #(msg)\n}\n"); + let parsed = vw_htcl::parse(&s); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); +} + +// --- quote_tcl! ------------------------------------------------------------- +// +// Mirror the quote_htcl! shape tests. The Tcl-dialect macro shares +// the same template parser, so the same template should produce the +// same output for the cases that overlap (which is most of them +// today — the Tcl/htcl split exists so they can DIVERGE later, not +// because their current behavior differs). + +#[test] +fn tcl_literal_passthrough() { + let s = quote_tcl!("puts hi\n"); + assert_eq!(s, "puts hi\n"); +} + +#[test] +fn tcl_simple_ident_interpolation() { + let name = "greet"; + let s = quote_tcl!("proc #(name) {} { puts hi }\n"); + assert_eq!(s, "proc greet {} { puts hi }\n"); +} + +#[test] +fn tcl_expression_interpolation() { + let width = 16u32; + let s = quote_tcl!("set w #(width)\n"); + assert_eq!(s, "set w 16\n"); +} + +#[test] +fn tcl_values_needing_quoting_get_quoted() { + let msg = "hello world"; + let s = quote_tcl!("puts #(msg)\n"); + assert_eq!(s, "puts \"hello world\"\n"); +} + +#[test] +fn tcl_braces_in_template_pass_through() { + let name = "f"; + let s = quote_tcl!("proc #(name) {} {\n puts hi\n}\n"); + assert_eq!(s, "proc f {} {\n puts hi\n}\n"); +} + +#[test] +fn tcl_multiple_interpolations() { + let name = "greet"; + let arg = "world"; + let s = quote_tcl!("proc #(name) { #(arg) } { puts hi }\n"); + assert_eq!(s, "proc greet { world } { puts hi }\n"); +} + +#[test] +fn tcl_emits_repr_proc_shape() { + // A representative use case from step 2b: emit a per-type + // repr proc body via quote_tcl!. + let mangled = "string"; + let s = quote_tcl!("proc #(mangled)::repr {v} {\n return $v\n}\n"); + assert_eq!(s, "proc string::repr {v} {\n return $v\n}\n"); +} diff --git a/vw-remote/Cargo.toml b/vw-remote/Cargo.toml new file mode 100644 index 0000000..64655e6 --- /dev/null +++ b/vw-remote/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "vw-remote" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Driving a Vivado worker that is on another machine" + +[dependencies] +async-trait.workspace = true +camino = { workspace = true, features = ["serde1"] } +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +toml.workspace = true +tokio.workspace = true +tokio-tungstenite.workspace = true +tracing.workspace = true +vw-eda = { path = "../vw-eda" } +vw-lib = { path = "../vw-lib" } +vw-bench = { path = "../vw-bench" } +vw-vivado = { path = "../vw-vivado" } + +[dev-dependencies] +tempfile.workspace = true +tokio = { workspace = true, features = ["full"] } diff --git a/vw-remote/src/backend.rs b/vw-remote/src/backend.rs new file mode 100644 index 0000000..463ec9a --- /dev/null +++ b/vw-remote/src/backend.rs @@ -0,0 +1,308 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! A worker on another machine, driven as though it were on this one. +//! +//! Implements the same [`EdaBackend`] the local Vivado worker implements, so +//! everything above it — `vw run`'s eval loop, the REPL's session, the block +//! renderer, the exit-code ladder — cannot tell the difference and does not +//! have to be told. The one thing it must keep faith with is streaming: a +//! caller that installed a sink gets its chunks while the command is still +//! running, exactly as it would locally, because that is the whole reason +//! anyone can bear to watch a synthesis run. + +use async_trait::async_trait; +use futures::{SinkExt, StreamExt}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::WebSocketStream; +use vw_eda::{ + BackendError, EdaBackend, EvalOutput, Request, RequestOp, Response, + ResponseResult, StdoutSink, StreamKind, +}; + +use crate::protocol::{SessionEvent, SessionRequest}; + +/// A way to stop whatever the worker is running, usable while it is running. +/// +/// Cloneable and independent of the backend on purpose. The backend is +/// borrowed for as long as an eval is in flight, so anything that had to go +/// through it could only interrupt a command that had already finished — which +/// is not an interrupt, it is a coincidence. +#[derive(Clone)] +pub struct InterruptHandle { + outgoing: tokio::sync::mpsc::UnboundedSender, +} + +impl InterruptHandle { + /// Ask the instance to abandon what it is running. + /// + /// Best effort by nature: the command may finish on its own between this + /// being sent and it arriving, and a session that has already ended has + /// nothing to interrupt. Neither is worth reporting to somebody who just + /// pressed Ctrl-C. + pub fn interrupt(&self) { + let _ = self.outgoing.send(SessionRequest::Interrupt); + } +} + +/// Where an agent's progress reports go. +/// +/// Separate from the output sink because these are not the build talking, they +/// are the machinery around it: dependencies being fetched, vivado starting. +/// A caller that shows them tells the developer why nothing is happening yet, +/// which for the half minute vivado takes to come up is the difference between +/// waiting and wondering. +pub type NoteSink = Box; + +/// A Vivado worker reached over a session. +pub struct RemoteBackend { + incoming: futures::stream::SplitStream>, + /// Everything leaving here goes through one task, so that a Ctrl-C can be + /// sent while an eval is still outstanding. + outgoing: tokio::sync::mpsc::UnboundedSender, + stdout_sink: Option, + note_sink: Option, + next_id: u64, + /// Set once the far end says the session is over, so a later call fails + /// with the reason rather than waiting for an answer that is not coming. + fatal: Option, +} + +impl RemoteBackend +where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ + /// Drive the worker on the far end of `socket`. + pub fn new(socket: WebSocketStream) -> RemoteBackend + where + S: 'static, + { + let (mut sink, incoming) = socket.split(); + let (outgoing, mut to_send) = + tokio::sync::mpsc::unbounded_channel::(); + + tokio::spawn(async move { + while let Some(request) = to_send.recv().await { + let Ok(text) = serde_json::to_string(&request) else { + continue; + }; + if sink.send(Message::Text(text)).await.is_err() { + break; + } + } + let _ = sink.close().await; + }); + + RemoteBackend { + incoming, + outgoing, + stdout_sink: None, + note_sink: None, + next_id: 1, + fatal: None, + } + } + + /// A handle for interrupting whatever this is running. + pub fn interrupt_handle(&self) -> InterruptHandle { + InterruptHandle { + outgoing: self.outgoing.clone(), + } + } + + fn alloc_id(&mut self) -> u64 { + let id = self.next_id; + self.next_id += 1; + id + } + + fn write(&mut self, request: Request) -> Result<(), BackendError> { + self.outgoing + .send(SessionRequest::Run { request }) + .map_err(|_| { + BackendError::Worker("the session has ended".to_owned()) + }) + } + + /// Read until the answer to `id` arrives, feeding everything else where it + /// belongs on the way. + /// + /// Returns the response and whatever output was produced while waiting. + /// The output is only accumulated when no sink is installed — with one, + /// the sink owns the chunks, which is the same bargain the local backend + /// strikes and what keeps `EvalOutput::stdout` from silently doubling + /// everything the caller has already printed. + async fn read_until( + &mut self, + id: u64, + ) -> Result<(Response, String), BackendError> { + let mut stdout = String::new(); + + loop { + let message = self.incoming.next().await.ok_or_else(|| { + BackendError::Worker( + "the session ended before the command answered".to_owned(), + ) + })?; + + let message = message.map_err(|e| { + BackendError::Worker(format!("reading from the session: {e}")) + })?; + + let text = match message { + Message::Text(text) => text, + Message::Binary(bytes) => String::from_utf8(bytes) + .map_err(|e| BackendError::Worker(e.to_string()))?, + // Pings are answered by the library; a close means the far end + // is gone and there is no answer coming. + Message::Close(_) => { + return Err(BackendError::Worker( + self.fatal.clone().unwrap_or_else(|| { + "the session was closed by the other end".to_owned() + }), + )) + } + _ => continue, + }; + + match serde_json::from_str::(&text)? { + SessionEvent::Chunk { kind, data } => { + match self.stdout_sink.as_mut() { + Some(sink) => sink(kind, &data), + None => stdout.push_str(&data), + } + } + SessionEvent::Note { message } => { + // A caller with somewhere particular to put these gets + // them there. Otherwise they join the output, which is + // what a full-screen caller wants: it has scrollback and + // no stderr to write to without tearing its own display. + if let Some(sink) = self.note_sink.as_mut() { + sink(&message); + } else if let Some(sink) = self.stdout_sink.as_mut() { + sink(StreamKind::Info, &format!("{message}\n")); + } else { + tracing::info!("{message}"); + } + } + SessionEvent::Response(response) if response.id == id => { + return Ok((response, stdout)) + } + // An answer to something else. Nothing issues two requests at + // once — `eval` takes `&mut self` — so this is a duplicate or + // a stale reply, and dropping it is better than mistaking it + // for the one being waited on. + SessionEvent::Response(response) => { + tracing::warn!( + "ignoring a response for {} while waiting for {id}", + response.id, + ); + } + SessionEvent::Fatal { message } => { + self.fatal = Some(message.clone()); + return Err(BackendError::Worker(message)); + } + } + } + } + + /// Install a sink for the agent's progress reports. + /// + /// Not on [`EdaBackend`] because a local worker has nothing to report: the + /// waiting it does is on this machine, where the caller can already see + /// it. Only a session across a network has a gap to explain. + pub fn set_note_sink(&mut self, sink: NoteSink) { + self.note_sink = Some(sink); + } + + /// Fail immediately if the session has already been declared over. + fn check_alive(&self) -> Result<(), BackendError> { + match &self.fatal { + Some(message) => Err(BackendError::Tcl { + message: message.clone(), + code: Some("VW_SESSION_DEAD".to_owned()), + info: None, + stdout: String::new(), + }), + None => Ok(()), + } + } +} + +#[async_trait] +impl EdaBackend for RemoteBackend +where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ + fn name(&self) -> &str { + "vivado (remote)" + } + + async fn eval(&mut self, tcl: &str) -> Result { + self.check_alive()?; + + let id = self.alloc_id(); + self.write(Request { + id, + op: RequestOp::Eval { tcl: tcl.into() }, + })?; + + let (response, stdout) = self.read_until(id).await?; + + match response.result { + ResponseResult::Ok { result, .. } => { + let value = match result { + serde_json::Value::String(s) => s, + other => other.to_string(), + }; + Ok(EvalOutput { value, stdout }) + } + ResponseResult::Err { error, .. } => Err(BackendError::Tcl { + message: error.message, + code: error.code, + info: error.info, + stdout, + }), + } + } + + async fn send( + &mut self, + mut request: Request, + ) -> Result { + self.check_alive()?; + + if request.id == 0 { + request.id = self.alloc_id(); + } + let id = request.id; + self.write(request)?; + let (response, _stdout) = self.read_until(id).await?; + Ok(response) + } + + fn set_stdout_sink(&mut self, sink: StdoutSink) { + self.stdout_sink = Some(sink); + } + + async fn shutdown(&mut self) -> Result<(), BackendError> { + // A session that has already died has nothing to shut down, and saying + // so would turn a clean exit into an error the user has to read. + if self.fatal.is_some() { + return Ok(()); + } + + let id = self.alloc_id(); + let _ = self.write(Request { + id, + op: RequestOp::Shutdown, + }); + // The far end tears Vivado down and closes; either an answer or the + // close will end this, and neither is worth failing over. + let _ = self.read_until(id).await; + + Ok(()) + } +} diff --git a/vw-remote/src/bench.rs b/vw-remote/src/bench.rs new file mode 100644 index 0000000..1b0dba5 --- /dev/null +++ b/vw-remote/src/bench.rs @@ -0,0 +1,151 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Running a workspace's testbenches on the instance that holds it. +//! +//! The same shape as a vivado session and for the same reason: a batch of +//! benches takes minutes and finishes one at a time, so the developer has to +//! see each result as it lands rather than a verdict at the end. What crosses +//! the socket is `vw-bench`'s own event stream, so the panel on a developer's +//! terminal is driven by exactly the events it would be driven by locally. +//! +//! Nothing about *what* to run is decided here. Discovery reads the tree, and +//! the tree is on the instance. + +use camino::Utf8Path; +use futures::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::WebSocketStream; + +/// What an instance sends back while running a batch. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BenchEvent { + /// Something that happened to a bench. + Progress { event: vw_bench::Event }, + /// Every bench has finished. + Done { passed: usize, failed: usize }, + /// The batch could not be run at all. + Fatal { message: String }, +} + +/// Run the batch described by `request` and report as it goes. +/// +/// `launch` says how the instance starts a single bench; the caller supplies +/// it because only the caller knows what binary it is. +pub async fn serve( + socket: WebSocketStream, + root: &Utf8Path, + request: vw_bench::Request, + launch: vw_bench::Launch, +) -> Result<(), crate::SessionError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let (mut outgoing, mut incoming) = socket.split(); + let (events, mut to_send) = + tokio::sync::mpsc::unbounded_channel::(); + + // One task writes, as elsewhere: results arrive while the batch is still + // running and must go out then, not after. + let writer = tokio::spawn(async move { + while let Some(event) = to_send.recv().await { + let Ok(text) = serde_json::to_string(&event) else { + continue; + }; + if outgoing.send(Message::Text(text)).await.is_err() { + break; + } + } + let _ = outgoing.close().await; + }); + + // A developer who walks away should not leave an instance running a batch + // nobody will read. Watched alongside the run rather than after it. + let departed = tokio::spawn(async move { + while let Some(Ok(message)) = incoming.next().await { + if matches!(message, Message::Close(_)) { + break; + } + } + }); + + let running = run(root, request, launch, events.clone()); + tokio::pin!(running); + + tokio::select! { + () = &mut running => {} + _ = departed => { + tracing::info!("client left; abandoning the batch"); + } + } + + drop(events); + let _ = writer.await; + + Ok(()) +} + +/// The batch itself, reporting into `events`. +async fn run( + root: &Utf8Path, + request: vw_bench::Request, + launch: vw_bench::Launch, + events: tokio::sync::mpsc::UnboundedSender, +) { + let standard = match request.standard.parse::() { + Ok(standard) => standard, + Err(e) => { + let _ = events.send(BenchEvent::Fatal { + message: format!( + "'{}' is not a vhdl standard: {e}", + request.standard + ), + }); + return; + } + }; + + let names = match vw_bench::discover(root, &request) { + Ok(names) => names, + Err(e) => { + let _ = events.send(BenchEvent::Fatal { + message: e.to_string(), + }); + return; + } + }; + + if names.is_empty() { + let _ = events.send(BenchEvent::Progress { + event: vw_bench::Event::Discovered { names }, + }); + let _ = events.send(BenchEvent::Done { + passed: 0, + failed: 0, + }); + return; + } + + if let Err(e) = vw_bench::prepare(root, standard).await { + let _ = events.send(BenchEvent::Fatal { + message: e.to_string(), + }); + return; + } + + let relay = events.clone(); + let summary = + vw_bench::run(root, names, request.concurrency, launch, move |event| { + let _ = relay.send(BenchEvent::Progress { event }); + }) + .await; + + let _ = events.send(BenchEvent::Done { + passed: summary.passed, + failed: summary.failed, + }); +} diff --git a/vw-remote/src/driver.rs b/vw-remote/src/driver.rs new file mode 100644 index 0000000..5b4218d --- /dev/null +++ b/vw-remote/src/driver.rs @@ -0,0 +1,695 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Building the driver on the machine it runs on. +//! +//! The helios instance is where the driver's target is native and where its +//! pinned toolchain is installed, so the build happens there and its output +//! comes back as it is produced — the same bargain as a vivado run. +//! +//! **Why cargo is a process rather than a library.** A driver pins its +//! toolchain in `rust-toolchain.toml`, and that file is honoured by the rustup +//! shim, not by cargo itself. Linking cargo in would mean building with +//! whatever cargo this agent was compiled against, silently ignoring the pin — +//! for a kernel module compiled with `code-model: kernel` that is worse than +//! not building at all. Cargo also says of itself that it is "not intended for +//! external use" and "may make major changes to its APIs". The one thing the +//! library would buy — structured diagnostics — cargo already offers any +//! caller through `--message-format=json`. + +use camino::{Utf8Path, Utf8PathBuf}; +use futures::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, BufReader}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::WebSocketStream; + +/// How the agent stores what a build produced. +/// +/// Passed in because storing is not this module's business — it knows what was +/// built, not where anything keeps things. Called before the build is reported +/// finished, so a developer whose command has returned can go and fetch what +/// it made. +pub type Uploader = Box< + dyn Fn(Vec) -> futures::future::BoxFuture<'static, usize> + + Send, +>; + +/// What the instance sends back while a build runs. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DriverEvent { + /// A new part of the build has started. + /// + /// A driver is not one cargo invocation. Userland and a kernel module are + /// built for different targets, and one `cargo build` produces artifacts + /// for exactly one target — so there are as many invocations as there are + /// targets, and it is worth saying which one is talking. + Building { unit: String }, + /// One line of cargo's output, as cargo wrote it. + /// + /// Forwarded verbatim, colour and all, so what a developer sees is what + /// they would have seen building on their own machine. Nothing here parses + /// or reformats it — cargo already says these things better than a relay + /// could. + Line { text: String }, + /// The build finished. + Done { + success: bool, + /// The exit status, when there was one. Absent if cargo was killed by + /// a signal. + code: Option, + }, + /// The build produced something worth keeping, and it has been stored. + Produced { + artifacts: Vec, + stored: usize, + }, + /// The build could not be run at all. + Fatal { message: String }, +} + +/// What cargo says about something it built. +#[derive(serde::Deserialize)] +#[serde(tag = "reason", rename_all = "kebab-case")] +enum CargoMessage { + /// A file cargo produced. + CompilerArtifact { + package_id: String, + target: CargoTarget, + #[serde(default)] + filenames: Vec, + }, + /// Something rustc had to say, already formatted the way rustc formats it. + CompilerMessage { message: RustcMessage }, + #[serde(other)] + Other, +} + +/// What a produced file was built as. +#[derive(serde::Deserialize)] +struct CargoTarget { + #[serde(default)] + kind: Vec, +} + +impl CargoTarget { + /// Whether this is a build script rather than something the driver is + /// made of. + /// + /// Cargo compiles every `build.rs` into a binary and reports it like any + /// other artifact. It is a step in the build, not a product of it — it + /// runs once on the machine that compiled it and means nothing anywhere + /// else, while weighing tens of megabytes unstripped. + fn is_build_script(&self) -> bool { + self.kind.iter().any(|kind| kind == "custom-build") + } +} + +#[derive(serde::Deserialize)] +struct RustcMessage { + #[serde(default)] + rendered: Option, +} + +/// Whether a file cargo produced is a deliverable rather than an intermediate. +/// +/// A driver's outputs are things that run or load — a binary, a kernel module. +/// An `rlib` is scaffolding for the next compilation and means nothing on +/// another machine, and `.d` files are make fragments naming paths that only +/// exist on the builder. +fn is_deliverable(path: &Utf8Path) -> bool { + !matches!(path.extension(), Some("rlib" | "rmeta" | "d")) +} + +/// What to build. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct BuildParams { + /// Build with optimizations. + pub release: bool, + /// Anything else to put on cargo's command line. + /// + /// Split on whitespace, so a value containing a space cannot be expressed. + /// No cargo flag the driver needs has one, and the alternative is half a + /// shell's quoting rules in a query parameter. + pub args: Vec, +} + +/// Run a build and report as it goes. +pub async fn serve( + socket: WebSocketStream, + root: &Utf8Path, + params: BuildParams, + upload: Uploader, +) -> Result, crate::SessionError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let (mut outgoing, mut incoming) = socket.split(); + let (events, mut to_send) = + tokio::sync::mpsc::unbounded_channel::(); + + let writer = tokio::spawn(async move { + while let Some(event) = to_send.recv().await { + let Ok(text) = serde_json::to_string(&event) else { + continue; + }; + if outgoing.send(Message::Text(text)).await.is_err() { + break; + } + } + let _ = outgoing.close().await; + }); + + // A developer who walks away should not leave a build running on an + // instance nobody is watching. + let departed = tokio::spawn(async move { + while let Some(Ok(message)) = incoming.next().await { + if matches!(message, Message::Close(_)) { + break; + } + } + }); + + let building = build(root, ¶ms, events.clone(), upload); + tokio::pin!(building); + + // A build that finishes after the developer has gone still produced + // something, and it still belongs in the store — but one abandoned part + // way through produced nothing worth keeping. + let produced = tokio::select! { + produced = &mut building => produced, + _ = departed => { + tracing::info!("client left; abandoning the build"); + Vec::new() + } + }; + + drop(events); + let _ = writer.await; + + Ok(produced) +} + +/// One place cargo has to be run, and what to call it. +struct Unit { + /// The directory to run from. This is the whole point: cargo reads + /// `.cargo/config.toml` from the current directory and its ancestors, and + /// explicitly does not read one belonging to a workspace member when it is + /// invoked from the workspace root. A member whose config selects a target + /// therefore cannot be built correctly any other way. + directory: Utf8PathBuf, + name: String, +} + +/// What `cargo metadata` tells us about a driver workspace. +#[derive(serde::Deserialize)] +struct Metadata { + packages: Vec, + #[serde(default)] + workspace_members: Vec, + #[serde(default)] + workspace_default_members: Vec, +} + +#[derive(serde::Deserialize)] +struct Package { + id: String, + name: String, + manifest_path: Utf8PathBuf, +} + +impl Package { + fn directory(&self) -> Utf8PathBuf { + self.manifest_path + .parent() + .map(Utf8Path::to_owned) + .unwrap_or_default() + } + + /// Whether this member's own cargo config selects a build target. + /// + /// This is vw's signal for "compiled for somewhere other than here". A + /// kernel module sets it because it is not userland; nothing else has a + /// reason to. It is a signal a project already has to set for its own + /// build to work, rather than one more file to keep in step. + fn is_separately_targeted(&self) -> bool { + let config = self.directory().join(".cargo/config.toml"); + let Ok(text) = std::fs::read_to_string(&config) else { + return false; + }; + let Ok(parsed) = text.parse::() else { + return false; + }; + parsed + .get("build") + .and_then(toml::Value::as_table) + .is_some_and(|build| build.contains_key("target")) + } +} + +/// Ask cargo what this workspace is made of. +fn metadata(root: &Utf8Path) -> Option { + let output = std::process::Command::new("cargo") + .args(["metadata", "--no-deps", "--format-version", "1"]) + .current_dir(root.as_std_path()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + serde_json::from_slice(&output.stdout).ok() +} + +/// Every cargo invocation a driver build needs, in the order to run them. +/// +/// vw's rule, and the whole of it: **a workspace member whose own +/// `.cargo/config.toml` selects a build target is built separately, from its +/// own directory.** Everything else is built together as the workspace's +/// default members. +/// +/// Two things force this and neither is a matter of taste. One cargo +/// invocation compiles for exactly one target, so a driver with a kernel +/// module and userland tooling is at least two invocations however it is +/// arranged. And a member's cargo config only applies when cargo runs from +/// that directory, so the invocation for such a member has to start there — +/// which, as a bonus, is also what keeps workspace feature unification from +/// pulling `std` into a `no_std` build. +/// +/// Deriving it from a file the project already needs means there is nothing +/// extra to declare, and nothing that can disagree with how the project +/// actually builds. +fn units(root: &Utf8Path) -> Vec { + let mut units = vec![Unit { + directory: root.to_owned(), + name: "workspace".to_owned(), + }]; + + let Some(metadata) = metadata(root) else { + return units; + }; + + let mut separate: Vec = metadata + .packages + .iter() + .filter(|package| metadata.workspace_members.contains(&package.id)) + .filter(|package| package.is_separately_targeted()) + .map(|package| Unit { + directory: package.directory(), + name: package.name.clone(), + }) + .collect(); + separate.sort_by(|a, b| a.name.cmp(&b.name)); + + units.append(&mut separate); + units +} + +/// A member that is built for its own target and also in the workspace's +/// default members. +/// +/// Worth catching before anything runs. Cargo will build it for the host +/// instead, quietly ignoring the target its config asks for, and the failure +/// arrives much later as a duplicate `panic_impl` lang item or a missing +/// `core` — which reads as a dependency problem and is not one. +fn contradiction(root: &Utf8Path) -> Option { + let metadata = metadata(root)?; + + let offender = metadata + .packages + .iter() + .filter(|package| { + metadata.workspace_default_members.contains(&package.id) + }) + .find(|package| package.is_separately_targeted())?; + + Some(format!( + "`{name}` sets its own build target in {directory}/.cargo/config.toml, \ + but it is also one of this workspace's `default-members`. Cargo does \ + not read a member's config when it runs from the workspace root, so \ + building it that way compiles it for this machine instead of for the \ + target it asks for. Remove `{name}` from `default-members` — vw \ + builds it separately, from its own directory, which is the only way \ + that config takes effect.", + name = offender.name, + directory = offender + .directory() + .strip_prefix(root) + .unwrap_or(&offender.directory()), + )) +} + +/// Run every part of the build, stopping at the first failure. +/// +/// Returns what it produced, by absolute path, so the caller can put it +/// somewhere it will outlive the instance. +async fn build( + root: &Utf8Path, + params: &BuildParams, + events: tokio::sync::mpsc::UnboundedSender, + upload: Uploader, +) -> Vec { + // Said before anything is built, because the alternative is a compiler + // error several minutes from now that names none of this. + if let Some(contradiction) = contradiction(root) { + let _ = events.send(DriverEvent::Fatal { + message: contradiction, + }); + return Vec::new(); + } + + // Cargo is asked which of its members are the workspace's, so a + // dependency's artifacts are not mistaken for the driver's. + let members: std::collections::HashSet = metadata(root) + .map(|metadata| metadata.workspace_members.into_iter().collect()) + .unwrap_or_default(); + + let mut produced = Vec::new(); + for unit in units(root) { + let _ = events.send(DriverEvent::Building { + unit: unit.name.clone(), + }); + + match build_one(&unit.directory, params, &events, &members).await { + Some((true, mut artifacts)) => produced.append(&mut artifacts), + // Reported already; there is no sense building the kernel module + // against a userland that did not compile. + Some((false, _)) | None => return Vec::new(), + } + } + + if !produced.is_empty() { + // Stored before the build is called finished. Otherwise a developer + // whose command has just returned successfully would go looking for + // the artifacts and find nothing, which is a race they have no way to + // know about. + let stored = upload(produced.clone()).await; + let _ = events.send(DriverEvent::Produced { + artifacts: produced.iter().map(ToString::to_string).collect(), + stored, + }); + } + + let _ = events.send(DriverEvent::Done { + success: true, + code: Some(0), + }); + + produced +} + +/// Spawn cargo in one directory and forward everything it says. +/// +/// Returns whether it succeeded, or `None` if it could not be run at all — in +/// which case the failure has already been reported. +async fn build_one( + root: &Utf8Path, + params: &BuildParams, + events: &tokio::sync::mpsc::UnboundedSender, + members: &std::collections::HashSet, +) -> Option<(bool, Vec)> { + let mut command = tokio::process::Command::new("cargo"); + command.arg("build"); + if params.release { + command.arg("--release"); + } + // Forced on: cargo turns colour off when its output is not a terminal, and + // here it never is — but there is a terminal at the far end of this, and + // it is the one that matters. + command.args(["--color", "always"]); + // Structured, so the exact set of files this produced comes from cargo + // rather than from guessing at target directory layout. Diagnostics still + // arrive pre-rendered, colour and all, so nothing about what a developer + // sees changes. + command.args(["--message-format", "json-diagnostic-rendered-ansi"]); + command.args(¶ms.args); + + let child = command + .current_dir(root.as_std_path()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + // So a build does not outlive the session that asked for it. + .kill_on_drop(true) + .spawn(); + + let mut child = match child { + Ok(child) => child, + Err(e) => { + let _ = events.send(DriverEvent::Fatal { + message: format!("cannot run cargo on this instance: {e}"), + }); + return None; + } + }; + + // The two streams say different things under `--message-format=json`. + // Cargo's own progress goes to stderr as ordinary text; rustc's + // diagnostics and the record of what was built come down stdout as JSON. + // Both are forwarded as they arrive, so the order a developer sees is the + // order things happened. + let structured = child.stdout.take().map(|stdout| { + tokio::spawn(read_messages(stdout, events.clone(), members.clone())) + }); + let plain = child + .stderr + .take() + .map(|stderr| tokio::spawn(forward(stderr, events.clone()))); + + let status = child.wait().await; + + // Drained before reporting the result, so the last line of a failing build + // never arrives after the verdict on it. + let mut artifacts = Vec::new(); + if let Some(structured) = structured { + if let Ok(found) = structured.await { + artifacts = found; + } + } + if let Some(plain) = plain { + let _ = plain.await; + } + + match status { + Ok(status) if status.success() => Some((true, artifacts)), + Ok(status) => { + let _ = events.send(DriverEvent::Done { + success: false, + code: status.code(), + }); + Some((false, Vec::new())) + } + Err(e) => { + let _ = events.send(DriverEvent::Fatal { + message: format!("waiting for cargo: {e}"), + }); + None + } + } +} + +/// Read cargo's structured output, forwarding what a person should see and +/// keeping what was built. +/// +/// Only the workspace's own members count. A dependency compiled along the way +/// produces artifacts too, and none of them are the driver. +async fn read_messages( + reader: R, + events: tokio::sync::mpsc::UnboundedSender, + members: std::collections::HashSet, +) -> Vec +where + R: AsyncRead + Unpin, +{ + let mut produced = Vec::new(); + let mut lines = BufReader::new(reader).lines(); + + while let Ok(Some(line)) = lines.next_line().await { + let Ok(message) = serde_json::from_str::(&line) else { + // Not something we understand. Cargo occasionally writes plain + // text here, and passing it on is better than swallowing it. + let _ = events.send(DriverEvent::Line { text: line }); + continue; + }; + + match message { + CargoMessage::CompilerMessage { message } => { + if let Some(rendered) = message.rendered { + // Already formatted by rustc, so it reads exactly as it + // would on the developer's own machine. + for text in rendered.lines() { + let _ = events.send(DriverEvent::Line { + text: text.to_owned(), + }); + } + } + } + CargoMessage::CompilerArtifact { + package_id, + target, + filenames, + } if members.contains(&package_id) && !target.is_build_script() => { + produced.extend( + filenames.into_iter().filter(|path| is_deliverable(path)), + ); + } + _ => {} + } + } + + produced +} + +/// Send every line of `reader` as it appears. +async fn forward( + reader: R, + events: tokio::sync::mpsc::UnboundedSender, +) where + R: AsyncRead + Unpin, +{ + let mut lines = BufReader::new(reader).lines(); + while let Ok(Some(text)) = lines.next_line().await { + if events.send(DriverEvent::Line { text }).is_err() { + return; + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn a_release_build_says_so() { + let params = BuildParams { + release: true, + args: vec!["-p".to_owned(), "module".to_owned()], + }; + + // The shape the agent will hand cargo, checked here because getting it + // wrong means building the wrong thing on a machine nobody is looking + // at. + let mut expected = vec!["build", "--release", "--color", "always"]; + expected.extend(params.args.iter().map(String::as_str)); + + assert_eq!( + expected, + ["build", "--release", "--color", "always", "-p", "module"], + ); + } + + /// A driver workspace with `userland` and, optionally, a member built for + /// its own target. + fn workspace( + separately_targeted: bool, + in_default_members: bool, + ) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + + let members = if in_default_members { + r#"["userland", "kmod"]"# + } else { + r#"["userland"]"# + }; + std::fs::write( + root.join("Cargo.toml"), + format!( + "[workspace]\nmembers = [\"userland\", \"kmod\"]\n\ + default-members = {members}\nresolver = \"2\"\n" + ), + ) + .expect("write"); + + for member in ["userland", "kmod"] { + let package = root.join(member); + std::fs::create_dir_all(package.join("src")).expect("mkdir"); + std::fs::write( + package.join("Cargo.toml"), + format!( + "[package]\nname = \"{member}\"\nversion = \"0.1.0\"\n\ + edition = \"2021\"\n" + ), + ) + .expect("write"); + std::fs::write(package.join("src/lib.rs"), "").expect("write"); + } + + if separately_targeted { + let config = root.join("kmod/.cargo"); + std::fs::create_dir_all(&config).expect("mkdir"); + std::fs::write( + config.join("config.toml"), + "[build]\ntarget = \"x86_64-unknown-none.json\"\n\n\ + [unstable]\nbuild-std = [\"core\", \"alloc\"]\n", + ) + .expect("write"); + } + + (dir, root) + } + + #[test] + fn a_member_with_its_own_target_is_built_on_its_own() { + // vw's whole rule. The kernel module's cargo config only applies when + // cargo runs from its directory, so that is where it is run from. + let (_dir, root) = workspace(true, false); + + let names: Vec = + units(&root).into_iter().map(|unit| unit.name).collect(); + + assert_eq!(names, ["workspace", "kmod"]); + } + + #[test] + fn a_workspace_that_is_all_userland_is_one_build() { + // The common case, and it should cost nothing: no member asks for a + // different target, so one invocation covers it. + let (_dir, root) = workspace(false, false); + + let names: Vec = + units(&root).into_iter().map(|unit| unit.name).collect(); + + assert_eq!(names, ["workspace"]); + } + + #[test] + fn a_member_in_default_members_that_wants_its_own_target_is_refused() { + // The mistake worth catching: cargo silently builds it for the host + // and the failure surfaces minutes later as a duplicate lang item, + // which reads as a dependency problem and is not one. + let (_dir, root) = workspace(true, true); + + let complaint = contradiction(&root).expect("should be caught"); + + assert!(complaint.contains("kmod"), "{complaint}"); + assert!(complaint.contains("default-members"), "{complaint}"); + } + + #[test] + fn a_workspace_with_nothing_contradictory_is_left_alone() { + let (_dir, root) = workspace(true, false); + assert!(contradiction(&root).is_none()); + + let (_dir, root) = workspace(false, false); + assert!(contradiction(&root).is_none()); + } + + #[test] + fn an_event_says_which_kind_it_is() { + let done = serde_json::to_string(&DriverEvent::Done { + success: false, + code: Some(101), + }) + .expect("serialize"); + + assert!(done.contains(r#""kind":"done""#), "{done}"); + assert!(matches!( + serde_json::from_str::(&done), + Ok(DriverEvent::Done { + success: false, + code: Some(101) + }), + )); + } +} diff --git a/vw-remote/src/lib.rs b/vw-remote/src/lib.rs new file mode 100644 index 0000000..75bfc71 --- /dev/null +++ b/vw-remote/src/lib.rs @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Running a build on a machine that is not this one. +//! +//! The split is between what depends on the source you are editing and what +//! depends on the tree being built. Parsing `design.htcl`, lowering it to Tcl +//! and attributing a diagnostic back to a line number all belong to the first +//! and stay on the developer's machine, which is why an error still points at +//! a file they can open. Spawning Vivado, answering `vw::vhdl_design_sources` +//! and deciding whether a checkpoint is still good all belong to the second +//! and happen on the instance, because that is where the files and the +//! `target/` directory are. +//! +//! In between is the protocol `vw-eda` already defined for talking to a local +//! worker, carried over a websocket instead of a pipe. It streams because it +//! always streamed. + +mod backend; +pub mod bench; +pub mod driver; +pub mod protocol; +mod session; + +pub use backend::{InterruptHandle, NoteSink, RemoteBackend}; +pub use bench::BenchEvent; +pub use driver::{BuildParams, DriverEvent}; +pub use protocol::{SessionEvent, SessionParams, SessionRequest}; +pub use session::{serve, workspace_root, SessionError}; diff --git a/vw-remote/src/protocol.rs b/vw-remote/src/protocol.rs new file mode 100644 index 0000000..669c73e --- /dev/null +++ b/vw-remote/src/protocol.rs @@ -0,0 +1,193 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! What a client and an agent say to each other over a session. +//! +//! Deliberately thin. `vw-eda` already defines what driving a TCL worker looks +//! like — a [`Request`] in, output chunks and a [`Response`] back — and that +//! protocol was built to stream, because a synthesis run produces output for +//! minutes before it produces a result. Moving the worker to another machine +//! does not change any of that. It changes the pipe. +//! +//! So this adds two things and no more: an envelope that says which of the two +//! kinds of thing is coming back, and a way for the agent to report a failure +//! that belongs to no particular request — Vivado dying, or never starting. + +use serde::{Deserialize, Serialize}; +use vw_eda::{Request, Response, StreamKind}; + +/// What a client sends. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum SessionRequest { + /// Something for the worker to do — the same [`Request`] a local backend + /// would receive, which is the point: the client is driving the same + /// worker it always drove. + Run { request: Request }, + + /// Abandon whatever is running, but keep the session. + /// + /// What Ctrl-C means in the REPL. Not a request for the worker, which is + /// busy and by definition not reading — it is a request about the worker, + /// and the agent acts on it by signalling the process the way an + /// interactive user's terminal would if vivado were on their own machine. + /// The eval then comes back as an error, the interpreter survives, and the + /// developer keeps everything they had loaded. + Interrupt, +} + +/// What an agent sends back. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum SessionEvent { + /// Output, as it is produced. + /// + /// Already classified, because classification reads Vivado's own message + /// format out of the byte stream and the agent is the side holding that + /// stream. Sending raw text and classifying at the far end would mean two + /// implementations that have to agree. + Chunk { kind: StreamKind, data: String }, + + /// The result of a request. + Response(Response), + + /// Something worth saying that is not output and not a result — a stale + /// project wiped, dependencies fetched, a fallback taken while starting. + /// + /// These happen before any request is in flight, so they cannot be + /// reported as one failing. + Note { message: String }, + + /// The session cannot continue. + /// + /// Distinct from an error response: a response means a command failed and + /// the worker is still there to take another. This means there is no + /// worker, and every request still outstanding will never be answered. + Fatal { message: String }, +} + +/// How a client asks for a session to be set up. +/// +/// Everything the agent needs to spawn a worker the way this run wants it, +/// and nothing it can work out for itself: the tree, the workspace config and +/// the dependency cache are all already on its side, so none of them are here. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SessionParams { + /// `--part`, for a workspace whose parts are declared at the top level. + pub part: Option, + /// `--variant`, for a workspace that declares variants. + pub variant: Option, + /// Attach the Tcl call stack to INFO messages as well as warnings and + /// errors. + pub info_with_stack: bool, + /// Forward Vivado's unclassified chatter — its banner, source echo and + /// idle output — rather than reading and discarding it. + pub verbose: bool, +} + +impl SessionParams { + /// Render as query parameters for the session URL. + /// + /// Hand-rolled rather than pulled from a query-string crate: there are + /// four fields, they are all scalars, and the agent parses them back with + /// the same list in view. + pub fn to_query(&self) -> Vec<(&'static str, String)> { + let mut query = Vec::new(); + if let Some(part) = &self.part { + query.push(("part", part.clone())); + } + if let Some(variant) = &self.variant { + query.push(("variant", variant.clone())); + } + if self.info_with_stack { + query.push(("info_with_stack", "true".to_owned())); + } + if self.verbose { + query.push(("verbose", "true".to_owned())); + } + query + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn an_event_says_which_kind_it_is() { + let chunk = SessionEvent::Chunk { + kind: StreamKind::CriticalWarning, + data: "CRITICAL WARNING: [Synth 8-7080]\n".to_owned(), + }; + let json = serde_json::to_string(&chunk).expect("serialize"); + + assert!(json.contains(r#""event":"chunk""#), "{json}"); + assert!(json.contains(r#""kind":"critical_warning""#), "{json}"); + + let back: SessionEvent = + serde_json::from_str(&json).expect("deserialize"); + match back { + SessionEvent::Chunk { kind, .. } => { + assert_eq!(kind, StreamKind::CriticalWarning) + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_response_survives_the_trip() { + let event = SessionEvent::Response(vw_eda::Response::ok( + 7, + serde_json::json!("done"), + )); + let json = serde_json::to_string(&event).expect("serialize"); + + let back: SessionEvent = + serde_json::from_str(&json).expect("deserialize"); + match back { + SessionEvent::Response(r) => assert_eq!(r.id, 7), + other => panic!("{other:?}"), + } + } + + #[test] + fn an_interrupt_is_distinguishable_from_work() { + let interrupt = serde_json::to_string(&SessionRequest::Interrupt) + .expect("serialize"); + let work = serde_json::to_string(&SessionRequest::Run { + request: Request { + id: 1, + op: vw_eda::RequestOp::Eval { + tcl: "synth_design".to_owned(), + }, + }, + }) + .expect("serialize"); + + assert!(interrupt.contains(r#""op":"interrupt""#), "{interrupt}"); + assert!(work.contains(r#""op":"run""#), "{work}"); + + // And the round trip keeps them apart, which is the whole point: an + // interrupt mistaken for work would be queued behind the very thing + // it is trying to stop. + assert!(matches!( + serde_json::from_str::(&interrupt), + Ok(SessionRequest::Interrupt), + )); + assert!(matches!( + serde_json::from_str::(&work), + Ok(SessionRequest::Run { .. }), + )); + } + + #[test] + fn only_what_was_asked_for_is_in_the_query() { + let params = SessionParams { + variant: Some("metro".to_owned()), + ..Default::default() + }; + + assert_eq!(params.to_query(), [("variant", "metro".to_owned())]); + } +} diff --git a/vw-remote/src/session.rs b/vw-remote/src/session.rs new file mode 100644 index 0000000..a1df012 --- /dev/null +++ b/vw-remote/src/session.rs @@ -0,0 +1,728 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The agent's half: a real Vivado worker, driven by whoever is on the socket. +//! +//! Everything Vivado needs to be told is worked out here rather than sent: +//! which part, which variant, where the sources are, whether a checkpoint is +//! still good. All of those are answers about the tree, and the tree is here. +//! The client sends the two flags it cannot know — `--part` and `--variant` — +//! and nothing else. +//! +//! Reading and writing are separated on purpose. A command can run for minutes +//! and produce output the whole time, so the side that forwards output cannot +//! be the side that is blocked waiting for the command to finish. + +use camino::{Utf8Path, Utf8PathBuf}; +use futures::{SinkExt, StreamExt}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::mpsc::{self, UnboundedSender}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::WebSocketStream; +use vw_eda::{EdaBackend, RequestOp}; + +use crate::protocol::{SessionEvent, SessionParams, SessionRequest}; + +#[derive(Debug, thiserror::Error)] +pub enum SessionError { + #[error("resolving what to build: {0}")] + Selection(String), + #[error("starting vivado")] + Spawn(#[source] vw_eda::BackendError), + #[error("talking to the client")] + Socket(#[source] tokio_tungstenite::tungstenite::Error), +} + +/// Run a session against the workspace at `root` until the client is done. +/// +/// Returns when the client asks to shut down or goes away. Vivado is torn down +/// either way — a worker whose client has vanished is holding a great deal of +/// memory for nobody. +pub async fn serve( + socket: WebSocketStream, + root: &Utf8Path, + params: SessionParams, +) -> Result<(), SessionError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let (mut outgoing, mut incoming) = socket.split(); + let (events, mut to_send) = mpsc::unbounded_channel::(); + + // One task owns writing. Output produced while a command runs goes out as + // it is produced rather than piling up behind the command — which is the + // difference between watching a synthesis run and staring at a blank + // terminal for twenty minutes. + let writer = tokio::spawn(async move { + while let Some(event) = to_send.recv().await { + let text = match serde_json::to_string(&event) { + Ok(text) => text, + Err(e) => { + tracing::error!("cannot encode a session event: {e}"); + continue; + } + }; + if outgoing.send(Message::Text(text)).await.is_err() { + // The client is gone. Nothing left to say to it. + break; + } + } + let _ = outgoing.close().await; + }); + + let result = match start(&events, root, ¶ms).await { + Ok(backend) => { + // Built before anything is running, because interrupting means + // reaching the worker while the backend is borrowed by the very + // command being interrupted. + let pid = backend.child_pid(); + let interrupt: Interrupter = Box::new(move || match pid { + Some(pid) => vw_vivado::interrupt_process_group(pid), + None => tracing::warn!( + "asked to interrupt, but the worker has no pid to signal" + ), + }); + drive(&mut incoming, &events, Box::new(backend), interrupt).await + } + Err(e) => Err(e), + }; + + if let Err(e) = &result { + // The client is owed a reason. It cannot see this instance's log, and + // a session that simply closed would look like a network fault. + let _ = events.send(SessionEvent::Fatal { + message: e.to_string(), + }); + } + + drop(events); + let _ = writer.await; + + result +} + +/// Pump requests through a worker until the client is done with it. +/// +/// Takes the worker rather than making one so that the part worth testing — +/// what happens when the developer walks away mid-command — can be tested +/// without a vivado installation. +async fn drive( + incoming: &mut futures::stream::SplitStream>, + events: &UnboundedSender, + mut backend: Box, + interrupt: Interrupter, +) -> Result<(), SessionError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + while let Some(message) = incoming.next().await { + // A read failure here means the client went away between commands — + // a terminal closed, a laptop shut, a Ctrl-C at an idle prompt. It + // ends the session, but calling it a failure would put an error in + // the log for every developer who finished what they were doing. + let Ok(message) = message else { break }; + + let text = match message { + Message::Text(text) => text, + Message::Binary(bytes) => match String::from_utf8(bytes) { + Ok(text) => text, + Err(e) => { + tracing::warn!("discarding a non-utf8 request: {e}"); + continue; + } + }, + // The client hung up. Not a failure — it is how a run ends when + // the user interrupts one. + Message::Close(_) => break, + _ => continue, + }; + + let request = match serde_json::from_str::(&text) { + Ok(SessionRequest::Run { request }) => request, + // Nothing is running, so there is nothing to stop. A Ctrl-C that + // lands in the gap between commands is the ordinary case of + // pressing it at an idle prompt. + Ok(SessionRequest::Interrupt) => continue, + Err(e) => { + tracing::warn!("discarding an unreadable request: {e}"); + continue; + } + }; + + let shutting_down = matches!(request.op, RequestOp::Shutdown); + let id = request.id; + + // The command and the socket are watched together. A synthesis run + // takes minutes, and for all of them the developer may press Ctrl-C or + // close the lid — so the socket cannot go unread until the command + // finishes, or the answer to "stop" would be "in a little while". + // + // The command's future is pinned outside the loop so that handling an + // interrupt does not cancel it. That matters: an interrupt is meant to + // make the command come back with an error, and the session carries on + // with everything the developer had loaded. Dropping the future would + // throw away the answer they are waiting for. + // Scoped so the borrow of `backend` ends with the command, leaving + // it free to be dropped below if the developer has gone. + let outcome = { + let running = backend.send(request); + tokio::pin!(running); + + loop { + let interruption = tokio::select! { + result = &mut running => break Ran(result), + interruption = next_interruption(incoming) => interruption, + }; + + match interruption { + Interruption::Left => break Left, + Interruption::Interrupt => { + // Exactly what the developer's own terminal would do + // if vivado were on their machine: its Tcl traps + // SIGINT into `interp cancel`, so the eval aborts and + // the interpreter lives. + tracing::info!("interrupting request {id}"); + interrupt(); + } + } + } + }; + + match outcome { + Ran(Ok(response)) => { + let _ = events.send(SessionEvent::Response(response)); + } + Ran(Err(e)) => { + // The worker itself failed, not the command — a command that + // fails comes back as an error response. There is nothing left + // to run requests against. + let _ = events.send(SessionEvent::Fatal { + message: format!("the vivado worker failed: {e}"), + }); + tracing::error!("worker failed answering request {id}: {e}"); + break; + } + Left => { + // Killed rather than asked to stop. `shutdown` sends vivado a + // request and waits for the reply, and a vivado in the middle + // of `synth_design` will not read it for another twenty + // minutes — by which time it has finished the work nobody is + // waiting for and burned an instance doing it. Dropping the + // backend kills the process. + tracing::info!( + "client left while request {id} was running; killing vivado" + ); + drop(backend); + return Ok(()); + } + } + + if shutting_down { + break; + } + } + + let _ = backend.shutdown().await; + + Ok(()) +} + +/// What became of a request. +use Outcome::{Left, Ran}; +enum Outcome { + /// The worker answered, for better or worse. + Ran(Result), + /// The developer went away while it was still running. + Left, +} + +/// How to cut short whatever the worker is running. +/// +/// Injected rather than derived from the worker so that the behaviour worth +/// testing — an interrupt reaching a command that is already in flight — does +/// not need a real process to signal. +type Interrupter = Box; + +/// Why a running command's peace was disturbed. +enum Interruption { + /// The developer asked for it to stop. + Interrupt, + /// There is no developer any more. + Left, +} + +/// Resolve when something arrives for a command that is already running. +/// +/// A close frame is the polite way a client leaves; a connection that simply +/// ends is what happens when the process is killed outright. Neither leaves +/// anyone to send an answer to, so they are the same event. +async fn next_interruption( + incoming: &mut futures::stream::SplitStream>, +) -> Interruption +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + loop { + let message = match incoming.next().await { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => { + return Interruption::Left + } + Some(Ok(Message::Text(text))) => text, + // Anything else is not something a client sends while it is + // waiting on an answer, and is not a reason to disturb the + // command. + Some(Ok(_)) => continue, + }; + + match serde_json::from_str::(&message) { + Ok(SessionRequest::Interrupt) => return Interruption::Interrupt, + // A second command while the first is outstanding. The protocol + // does not allow it and acting on it would interleave two evals + // in one interpreter. + Ok(SessionRequest::Run { .. }) => { + tracing::warn!("ignoring a request sent over a running one"); + } + Err(e) => tracing::warn!("discarding an unreadable message: {e}"), + } + } +} + +/// Bring up Vivado for this session. +async fn start( + events: &UnboundedSender, + root: &Utf8Path, + params: &SessionParams, +) -> Result { + let note = |message: String| { + let _ = events.send(SessionEvent::Note { message }); + }; + + // Dependencies are fetched here rather than synchronized. They are named + // by revision in `vw.lock`, so the instance can get them itself and get + // exactly what the developer's machine would — using the credentials the + // sync that preceded this put in place. + fetch_dependencies(root, ¬e).await; + + let selection = vw_vivado::resolve_workspace_selection( + root, + params.part.as_deref(), + params.variant.as_deref(), + ) + .map_err(SessionError::Selection)?; + + for message in &selection.notes { + note(message.clone()); + } + + // Only consulted on demand: nothing has been shipped to this Vivado yet, + // so an empty map is the truth, and `compile_htcl_module` will load what + // it needs from the tree. + let preload: vw_vivado::SharedPreload = + std::sync::Arc::new(std::sync::RwLock::new(Default::default())); + let cw_count: vw_vivado::SharedCriticalWarningCount = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + + let rpc_handler = vw_vivado::make_handler_full( + Some(root.as_std_path().to_path_buf()), + selection.active_variant.clone(), + preload, + cw_count.clone(), + ); + + let raw_log = + match vw_vivado::raw_log_path_for_workspace(root.as_std_path()) { + Ok(path) => Some(path), + Err(e) => { + note(format!("raw vivado log unavailable: {e}")); + None + } + }; + + // Said before the wait rather than after it. Vivado takes the better part + // of a minute to come up, and a minute of silence on a remote build reads + // as a hang — the developer cannot see the process starting the way they + // could if it were on their own machine. + note("starting vivado".to_owned()); + + let mut backend = + vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose: params.verbose, + info_with_stack: params.info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project: selection.auto_project, + raw_log, + ..Default::default() + }) + .await + .map_err(SessionError::Spawn)?; + + note("vivado is ready".to_owned()); + + // Everything the worker produces goes straight out. The counter the + // checkpoint gates read lives on this side too, because the RPC that + // reads it is answered on this side. + let sink_events = events.clone(); + backend.set_stdout_sink(Box::new(move |kind, chunk: &str| { + if matches!(kind, vw_eda::StreamKind::CriticalWarning) { + cw_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + let _ = sink_events.send(SessionEvent::Chunk { + kind, + data: chunk.to_owned(), + }); + })); + + Ok(backend) +} + +/// Make sure the workspace's dependencies are on this instance. +/// +/// Best effort: a failure here is reported and the run continues, because the +/// run may not need the missing dependency and failing now would be a worse +/// answer than failing where it is actually used. +async fn fetch_dependencies(root: &Utf8Path, note: &impl Fn(String)) { + if vw_lib::dependencies_present(root) { + return; + } + + note("fetching missing dependencies".to_owned()); + + let credentials = vw_lib::get_access_credentials_from_netrc("github.com") + .ok() + .flatten(); + if credentials.is_none() { + note( + "no github credentials on this instance; a private dependency \ + will not be fetchable" + .to_owned(), + ); + } + + if let Err(e) = vw_lib::update_workspace_with_token(root, credentials).await + { + note(format!("could not fetch dependencies: {e}")); + } +} + +/// Where a session's workspace lives on an instance. +/// +/// The same tree synchronization writes to. A build has to see what was +/// pushed, and there is only one copy. +pub fn workspace_root(tree: &Utf8Path) -> Utf8PathBuf { + tree.to_owned() +} + +#[cfg(test)] +mod test { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::time::Duration; + + use futures::SinkExt; + use tokio::net::{TcpListener, TcpStream}; + use tokio_tungstenite::tungstenite::protocol::Role; + use vw_eda::Request; + + use super::*; + + /// A worker whose command can be cut short, the way vivado's can. + /// + /// Stands in for the real mechanism without needing a process to signal: + /// the session normally interrupts by sending SIGINT to vivado's process + /// group, and what the caller observes is the eval coming back early with + /// the interpreter still alive. + struct InterruptibleWorker { + cancelled: Arc, + } + + #[async_trait::async_trait] + impl EdaBackend for InterruptibleWorker { + fn name(&self) -> &str { + "interruptible" + } + + async fn eval( + &mut self, + _tcl: &str, + ) -> Result { + unreachable!("the session only ever calls send") + } + + async fn send( + &mut self, + request: Request, + ) -> Result { + tokio::select! { + () = self.cancelled.notified() => Ok(vw_eda::Response::err( + request.id, + vw_eda::ErrorPayload { + message: "interrupted".to_owned(), + code: None, + info: None, + }, + )), + () = tokio::time::sleep(Duration::from_secs(30)) => { + Ok(vw_eda::Response::ok( + request.id, + serde_json::json!("ran to completion"), + )) + } + } + } + + fn set_stdout_sink(&mut self, _sink: vw_eda::StdoutSink) {} + + async fn shutdown(&mut self) -> Result<(), vw_eda::BackendError> { + Ok(()) + } + } + + /// A worker that takes a very long time and notices being dropped. + /// + /// Stands in for a vivado in the middle of `synth_design`: it will not + /// answer for a good while, and the only way to stop it is to kill it. + struct SlowWorker { + killed: Arc, + } + + impl Drop for SlowWorker { + fn drop(&mut self) { + self.killed.store(true, Ordering::SeqCst); + } + } + + #[async_trait::async_trait] + impl EdaBackend for SlowWorker { + fn name(&self) -> &str { + "slow" + } + + async fn eval( + &mut self, + _tcl: &str, + ) -> Result { + unreachable!("the session only ever calls send") + } + + async fn send( + &mut self, + request: Request, + ) -> Result { + tokio::time::sleep(Duration::from_secs(30)).await; + Ok(vw_eda::Response::ok(request.id, serde_json::json!("late"))) + } + + fn set_stdout_sink(&mut self, _sink: vw_eda::StdoutSink) {} + + async fn shutdown(&mut self) -> Result<(), vw_eda::BackendError> { + // Vivado's real shutdown asks the interpreter to exit and waits + // for it to answer, which a busy one will not do. Modelled as + // never returning, because that is what it amounts to. + tokio::time::sleep(Duration::from_secs(30)).await; + Ok(()) + } + } + + /// A session with a worker on one end and a socket the test drives. + async fn session( + killed: Arc, + ) -> ( + tokio::task::JoinHandle<()>, + tokio_tungstenite::WebSocketStream, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + + let served = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + stream, + Role::Server, + None, + ) + .await; + let (_outgoing, mut incoming) = socket.split(); + let (events, _drain) = mpsc::unbounded_channel(); + drive( + &mut incoming, + &events, + Box::new(SlowWorker { killed }), + Box::new(|| {}), + ) + .await + .expect("drive"); + }); + + let stream = TcpStream::connect(address).await.expect("connect"); + let client = tokio_tungstenite::WebSocketStream::from_raw_socket( + stream, + Role::Client, + None, + ) + .await; + + (served, client) + } + + #[tokio::test] + async fn an_interrupt_reaches_a_command_that_is_already_running() { + // The REPL's Ctrl-C. It has to arrive while the command is in flight — + // an interrupt delivered after the thing it was meant to stop has + // finished is not an interrupt — and the command must come back with + // an error rather than the session being torn down, because the + // developer still has an hour of loaded state in it. + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let cancelled = Arc::new(tokio::sync::Notify::new()); + let signal = Arc::clone(&cancelled); + + let served = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let socket = tokio_tungstenite::WebSocketStream::from_raw_socket( + stream, + Role::Server, + None, + ) + .await; + let (_outgoing, mut incoming) = socket.split(); + let (events, mut seen) = mpsc::unbounded_channel(); + + let worker = InterruptibleWorker { + cancelled: Arc::clone(&signal), + }; + let interrupt: Interrupter = + Box::new(move || signal.notify_waiters()); + + drive(&mut incoming, &events, Box::new(worker), interrupt) + .await + .expect("drive"); + + drop(events); + let mut collected = Vec::new(); + while let Some(event) = seen.recv().await { + collected.push(event); + } + collected + }); + + let stream = TcpStream::connect(address).await.expect("connect"); + let mut client = tokio_tungstenite::WebSocketStream::from_raw_socket( + stream, + Role::Client, + None, + ) + .await; + + client + .send(Message::Text( + serde_json::to_string(&SessionRequest::Run { + request: Request { + id: 1, + op: RequestOp::Eval { + tcl: "synth_design".to_owned(), + }, + }, + }) + .expect("encode"), + )) + .await + .expect("send"); + tokio::time::sleep(Duration::from_millis(100)).await; + client + .send(Message::Text( + serde_json::to_string(&SessionRequest::Interrupt) + .expect("encode"), + )) + .await + .expect("interrupt"); + tokio::time::sleep(Duration::from_millis(200)).await; + drop(client); + + let events = tokio::time::timeout(Duration::from_secs(5), served) + .await + .expect("the command should have been cut short") + .expect("session task"); + + // It answered — with an error, as an interrupted eval does — rather + // than running its full thirty seconds. + assert!( + events.iter().any(|event| matches!( + event, + SessionEvent::Response(r) + if matches!(r.result, vw_eda::ResponseResult::Err { .. }) + )), + "expected an interrupted response, got {events:?}", + ); + } + + #[tokio::test] + async fn a_client_that_leaves_takes_the_worker_with_it() { + let killed = Arc::new(AtomicBool::new(false)); + let (served, mut client) = session(Arc::clone(&killed)).await; + + // Ask for something long, then walk away — the developer pressing + // Ctrl-C twenty seconds into a synthesis run. + let request = SessionRequest::Run { + request: Request { + id: 1, + op: RequestOp::Eval { + tcl: "synth_design".to_owned(), + }, + }, + }; + client + .send(Message::Text( + serde_json::to_string(&request).expect("encode"), + )) + .await + .expect("send"); + tokio::time::sleep(Duration::from_millis(100)).await; + drop(client); + + // Promptly, not when the command it no longer cares about finishes. + // The worker holds an instance's worth of memory and would otherwise + // spend twenty minutes producing something nobody will collect. + tokio::time::timeout(Duration::from_secs(5), served) + .await + .expect("the session should end when the client does") + .expect("session task"); + + assert!( + killed.load(Ordering::SeqCst), + "the worker should have been killed, not asked to stop", + ); + } + + #[tokio::test] + async fn a_close_frame_ends_it_too() { + // The polite version of the same thing, which is what a client that + // gets to run its own shutdown sends. + let killed = Arc::new(AtomicBool::new(false)); + let (served, mut client) = session(Arc::clone(&killed)).await; + + let request = SessionRequest::Run { + request: Request { + id: 1, + op: RequestOp::Eval { + tcl: "synth_design".to_owned(), + }, + }, + }; + client + .send(Message::Text( + serde_json::to_string(&request).expect("encode"), + )) + .await + .expect("send"); + tokio::time::sleep(Duration::from_millis(100)).await; + client.close(None).await.expect("close"); + + tokio::time::timeout(Duration::from_secs(5), served) + .await + .expect("the session should end on a close frame") + .expect("session task"); + + assert!(killed.load(Ordering::SeqCst)); + } +} diff --git a/vw-remote/tests/streaming.rs b/vw-remote/tests/streaming.rs new file mode 100644 index 0000000..561a3ff --- /dev/null +++ b/vw-remote/tests/streaming.rs @@ -0,0 +1,202 @@ +// The property the remote flow lives or dies by: output reaches the developer +// while the command is still running. +// +// A synthesis run produces messages for minutes before it produces a result. +// If those only arrived when the command finished, watching a remote build +// would mean watching nothing at all, and the whole thing would be worse than +// useless — it would look hung. So this drives a `RemoteBackend` against an +// agent that deliberately takes its time, and checks that the chunks land as +// they are sent rather than in a heap at the end. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use futures::{SinkExt, StreamExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio_tungstenite::tungstenite::protocol::Role; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::WebSocketStream; +use vw_eda::{EdaBackend, StreamKind}; +use vw_remote::{RemoteBackend, SessionEvent, SessionRequest}; + +/// How long the pretend worker waits between chunks. +const GAP: Duration = Duration::from_millis(200); + +/// An agent that answers one eval with `chunks`, spaced out in time, and then +/// a result. +async fn agent_that_dawdles(chunks: Vec<(StreamKind, &'static str)>) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr").to_string(); + + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = + WebSocketStream::from_raw_socket(stream, Role::Server, None).await; + + // Wait to be asked for something. + let frame = socket.next().await.expect("a request").expect("frame"); + let request = match frame { + Message::Text(text) => { + match serde_json::from_str::(&text) + .expect("a request") + { + SessionRequest::Run { request } => request, + other => panic!("unexpected request: {other:?}"), + } + } + other => panic!("unexpected frame: {other:?}"), + }; + + for (kind, data) in chunks { + tokio::time::sleep(GAP).await; + let event = SessionEvent::Chunk { + kind, + data: data.to_owned(), + }; + socket + .send(Message::Text( + serde_json::to_string(&event).expect("encode"), + )) + .await + .expect("send chunk"); + } + + tokio::time::sleep(GAP).await; + let done = SessionEvent::Response(vw_eda::Response::ok( + request.id, + serde_json::json!("finished"), + )); + socket + .send(Message::Text(serde_json::to_string(&done).expect("encode"))) + .await + .expect("send response"); + }); + + address +} + +async fn connect(address: &str) -> RemoteBackend { + let stream = TcpStream::connect(address).await.expect("connect"); + RemoteBackend::new( + WebSocketStream::from_raw_socket(stream, Role::Client, None).await, + ) +} + +#[tokio::test] +async fn output_arrives_while_the_command_is_still_running() { + let address = agent_that_dawdles(vec![ + (StreamKind::Stdout, "starting synthesis\n"), + (StreamKind::Info, "INFO: [Synth 8-6157] done elaborating\n"), + (StreamKind::Stdout, "wrapping up\n"), + ]) + .await; + let mut backend = connect(&address).await; + + // When each chunk reached the caller, measured from the moment the command + // was issued. + let seen: Arc>> = + Arc::new(Mutex::new(Vec::new())); + let started = Instant::now(); + let recorder = Arc::clone(&seen); + backend.set_stdout_sink(Box::new(move |kind, chunk: &str| { + recorder.lock().expect("lock").push(( + started.elapsed(), + kind, + chunk.to_owned(), + )); + })); + + let output = backend.eval("synth_design").await.expect("eval"); + let finished = started.elapsed(); + let seen = seen.lock().expect("lock").clone(); + + assert_eq!(seen.len(), 3, "every chunk should have arrived: {seen:?}"); + + // The point of the whole exercise. The first chunk must have been in the + // caller's hands long before the command answered — not merely delivered + // in the right order once it was over. + assert!( + seen[0].0 < finished / 2, + "the first chunk arrived at {:?}, but the command only finished at \ + {finished:?} — that is not streaming, that is a transcript", + seen[0].0, + ); + + // And spaced the way the agent sent them, rather than bunched together at + // the end, which is what a buffer somewhere in the middle would produce. + for pair in seen.windows(2) { + let gap = pair[1].0 - pair[0].0; + assert!( + gap > GAP / 2, + "chunks {:?} and {:?} arrived {gap:?} apart; something is holding \ + them", + pair[0].2, + pair[1].2, + ); + } + + assert_eq!( + seen[1].1, + StreamKind::Info, + "the severity survived the trip" + ); + assert_eq!(output.value, "finished"); + assert!( + output.stdout.is_empty(), + "with a sink installed the chunks belong to it, not to the result — \ + otherwise the caller prints everything twice", + ); +} + +#[tokio::test] +async fn without_a_sink_the_output_comes_back_with_the_result() { + // What `vw check`'s in-process runs and any other non-rendering caller + // relies on: no sink means the output is not thrown away, it is collected. + let address = agent_that_dawdles(vec![ + (StreamKind::Stdout, "one\n"), + (StreamKind::Stdout, "two\n"), + ]) + .await; + let mut backend = connect(&address).await; + + let output = backend.eval("puts one; puts two").await.expect("eval"); + + assert_eq!(output.stdout, "one\ntwo\n"); +} + +#[tokio::test] +async fn a_worker_that_dies_is_reported_rather_than_waited_on() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr").to_string(); + + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = + WebSocketStream::from_raw_socket(stream, Role::Server, None).await; + let _ = socket.next().await; + let event = SessionEvent::Fatal { + message: "vivado exited during elaboration".to_owned(), + }; + socket + .send(Message::Text( + serde_json::to_string(&event).expect("encode"), + )) + .await + .expect("send"); + }); + + let mut backend = connect(&address).await; + let failure = backend.eval("synth_design").await.expect_err("should fail"); + + assert!( + failure + .to_string() + .contains("vivado exited during elaboration"), + "the reason should survive: {failure}", + ); + + // And a second command should fail at once with the same reason rather + // than hanging on an answer that is never coming. + let again = backend.eval("place_design").await.expect_err("should fail"); + assert!(again.to_string().contains("vivado exited"), "{again}"); +} diff --git a/vw-repl/Cargo.toml b/vw-repl/Cargo.toml new file mode 100644 index 0000000..c8c85b7 --- /dev/null +++ b/vw-repl/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "vw-repl" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Interactive REPL for htcl scripts, driven by a long-lived Vivado worker" + +[dependencies] +vw-htcl = { path = "../vw-htcl" } +vw-eda = { path = "../vw-eda" } +vw-vivado = { path = "../vw-vivado" } +vw-lib = { path = "../vw-lib" } +camino.workspace = true +ratatui.workspace = true +crossterm.workspace = true +tui-textarea.workspace = true +tokio.workspace = true +thiserror.workspace = true +dirs.workspace = true +futures.workspace = true +tracing.workspace = true +arboard = "3" +base64 = "0.22" +winnow.workspace = true +nucleo-matcher.workspace = true +serde_json.workspace = true +serde.workspace = true +toml.workspace = true + +# Unix-only. Used by the Ctrl-C eval-interrupt path in `app.rs` — +# `libc::kill(pid, SIGINT)` sends SIGINT directly to the Vivado +# child (cached pid from `WorkerEvent::Started`) so cancellation +# fires immediately even while `worker_task` is blocked in +# `backend.eval().await` on the very eval we're trying to cancel. +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-repl/src/app.rs b/vw-repl/src/app.rs new file mode 100644 index 0000000..53bad80 --- /dev/null +++ b/vw-repl/src/app.rs @@ -0,0 +1,5026 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! REPL state machine + event loop. +//! +//! Single tokio task drives both the ratatui screen and the Vivado +//! worker. Inputs are crossterm key events; outputs are eval results +//! from the worker plus our own scrollback updates. A `tokio::select!` +//! arbitrates the two so neither side blocks the other. +//! +//! A Vivado eval can take seconds to minutes. The UI stays +//! responsive throughout: the input area locks (`eval_in_flight`) +//! but the screen still redraws, the worker's stdout still streams +//! into scrollback as it arrives, and Ctrl-C cancels the in-flight +//! eval (sent as a TCL interrupt to the worker). + +use std::time::Duration; + +use crossterm::event::{ + DisableBracketedPaste, DisableMouseCapture, EnableBracketedPaste, + EnableMouseCapture, Event, KeyCode, KeyEvent, KeyEventKind, KeyModifiers, + KeyboardEnhancementFlags, MouseButton, MouseEvent, MouseEventKind, + PopKeyboardEnhancementFlags, PushKeyboardEnhancementFlags, +}; +use crossterm::terminal::{ + disable_raw_mode, enable_raw_mode, EnterAlternateScreen, + LeaveAlternateScreen, +}; +use crossterm::ExecutableCommand; +use futures::StreamExt; +use ratatui::backend::CrosstermBackend; +use ratatui::layout::Rect; +use ratatui::Terminal; +use tokio::sync::mpsc; +use tui_textarea::{Input, TextArea}; + +use crate::config::{self, CollapseMode}; +use crate::history::History; +use crate::lower::Origin; +use crate::session::{Session, SessionBatch}; +use crate::ui::{self, WorkerStatusView}; +use crate::{ReplError, ReplOptions}; + +/// A multi-line entry with more than this many source lines +/// auto-collapses into a `▶`-marked placeholder at push time, +/// Mathematica-notebook style. Shift-click still toggles the +/// state, so the raw content is one gesture away — the threshold +/// just keeps a wall of text from dominating scrollback when a +/// large `list` or `Properties` value comes back. Chosen at 100 +/// because that's roughly the transition point where a viewport- +/// filling block stops being scannable and starts being intrusive. +pub const COLLAPSE_AUTO_THRESHOLD: usize = 100; + +/// Decide the initial [`ScrollbackEntry::collapse_state`] for an +/// entry with `text`. Every multi-line entry gets a `Some(bool)` +/// so it's toggleable via Shift+click; single-line entries get +/// `None` because a placeholder around one row of content adds +/// affordance without meaningful eliding. Above the auto-threshold +/// the initial state is collapsed. +fn compute_collapse_state(text: &str, mode: CollapseMode) -> Option { + let lines = text.lines().count(); + if lines < 2 { + return None; + } + match mode { + // Aggressive: every multi-line entry starts collapsed — + // `▶`-marked placeholders that expand on demand. Turns + // scrollback into a compact index. See + // [`crate::config::CollapseMode`]. + CollapseMode::Aggressive => Some(true), + // Normal: only wall-of-text entries auto-collapse; smaller + // multi-line output stays inline for at-a-glance scanning. + CollapseMode::Normal => Some(lines > COLLAPSE_AUTO_THRESHOLD), + } +} + +/// Split a tagged-diagnostic message into `(leading, trailing)`. +/// +/// A "tagged diagnostic line" has the shape +/// `: [] ` — e.g. +/// `ERROR: [Common 17-107] Cannot change read-only property …` +/// or `CRITICAL WARNING: [Project 1-486] Could not resolve …`. +/// Vivado (and our own eval-error renderer) sometimes appends +/// wrapped continuations, a ` Resolution: …` hint, or a +/// backtrace on the following lines — all valuable, but bulky. +/// +/// `leading` is the tagged first line (returned even when the +/// message is a single line, in which case `trailing` is `None`). +/// `trailing` is everything after the first line, trailing +/// newlines stripped; `None` when the message is a single line +/// or the tail is whitespace-only. +/// +/// Downstream, `leading` pushes as its OWN scrollback entry so +/// it stays at full brightness — one-liner, non-collapsible, +/// eye-catching gutter — while `trailing` pushes separately and +/// can auto-collapse / dim like any other multi-line entry. +/// This is the fix for the "critical error line got dimmed and +/// blended into the surrounding chatter" bug. +fn split_leading_diagnostic(text: &str) -> (String, Option) { + match text.split_once('\n') { + Some((head, tail)) => { + let tail = tail.trim_end_matches('\n'); + if tail.trim().is_empty() { + (head.to_string(), None) + } else { + (head.to_string(), Some(tail.to_string())) + } + } + None => (text.to_string(), None), + } +} + +/// Split a diagnostic's rendered text into `(body, stack)` at the +/// first line that looks like a stack frame — ` at :`. +/// The body is everything before that line (message + any wrapped +/// continuations); the stack is that line and everything after, +/// trailing newline stripped. `None` for stack means the diagnostic +/// carried no frames (traceless INFOs, some plain WARNINGs). +/// +/// The two-space indent + literal `at ` prefix is the shape +/// [`install_proc_body_wrap`] attaches in `vivado-shim.tcl` — see +/// its `format_stack` helper. If that format ever drifts, both +/// sides must stay in sync. +fn split_body_and_stack(text: &str) -> (String, Option) { + let needle = "\n at "; + match text.find(needle) { + // The `\n` at `idx` closes the body line; the stack begins + // at `idx + 1` so the ` at ` prefix is preserved on the + // first stack frame. + Some(idx) => { + let body = text[..idx].to_string(); + let stack = text[idx + 1..].trim_end_matches('\n').to_string(); + if stack.is_empty() { + (body, None) + } else { + (body, Some(stack)) + } + } + None => (text.to_string(), None), + } +} + +/// What category an entry in the scrollback log belongs to. Drives +/// the per-line gutter prefix and color. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScrollbackKind { + /// Echo of an input the user submitted. + Input, + /// A return value from a successful eval. + Result, + /// Captured stdout from `puts` etc. during an eval. + Stdout, + /// An error — TCL-level or REPL-level. Also the visual bucket + /// for Vivado CRITICAL WARNINGs, which the block classifier + /// keeps semantically distinct for log-level filtering but + /// which render with the same red ✗ treatment (per the + /// stream handler's Severity match). Entries in this kind + /// that came from a CW are additionally flagged via + /// [`ScrollbackEntry::is_critical_warning`] so the + /// diagnostics finder can offer a separate `Critical` filter + /// checkbox on top of the Error filter. + Error, + /// A pre-flight warning the user should see before the + /// underlying eval result — e.g. "this call uses keyword args + /// but isn't a loaded htcl wrapper." Distinct color from + /// notices so it actually pulls the eye. + Warning, + /// Internal notice (`vivado: ready`, `:load`, `:restart`, etc.). + Notice, + /// Non-diagnostic Vivado chatter that landed as its own single- + /// line NONE block — the `----` divider after an INFO, + /// `Attempting to get a license…` status echoes, single-line + /// section banners. Rendered dimmed dark-gray with a plain + /// ` ` prefix so it reads as "background noise" without + /// pretending to be an INFO (which would carry `· `) or a + /// user-facing Stdout entry (which would be bright white). + /// Multi-line NONE blocks go through the collapsible path + /// instead — see `push_none_block`. + Chatter, +} + +/// Tracks where each echoed top-level statement's Input entry +/// lives in scrollback AND which lowered command-index in the +/// batch is its last. When that command finishes evaluating, the +/// Input entry's timer freezes — giving accurate per-statement +/// durations in multi-statement load batches instead of all +/// entries sharing the whole-batch wall time. +#[derive(Clone, Debug)] +struct InputBoundary { + /// Position in `scrollback` where this boundary's echo lives, + /// or `None` when the echo is still queued. Non-first entries + /// start `None` and get pushed by `advance_input_timers` when + /// the prior boundary closes — so a `:load` batch prints + /// linearly: command, its output, then the *next* command. + scrollback_idx: Option, + /// Snippet captured up front so the deferred push has the + /// exact text `dispatch_eval_with_echo` would have used + /// eagerly. + snippet: String, + /// Eval-index in `pending_origins` of the last lowered + /// command that originated from this top-level statement. + /// When `pending_eval_index` reaches this value (i.e. the + /// command at this index has just finished), the entry's + /// timer should freeze. `None` when no lowered command was + /// attributed to this boundary — e.g. a `src` whose target + /// file lowered to zero Tcl commands. Such boundaries are + /// skipped when the prior boundary closes: nothing to wait + /// for and nothing to echo. + last_command_idx: Option, + /// Set to true once we've stamped this entry's `completed_at`, + /// so we don't re-stamp on subsequent EvalDones. + completed: bool, +} + +#[derive(Clone, Debug)] +pub struct ScrollbackEntry { + pub kind: ScrollbackKind, + pub text: String, + /// When this entry was pushed. Only set for `Input` entries + /// — used by the renderer to right-justify a `Ns` / + /// `M:SS` / `H:MM:SS` elapsed-time marker on the first + /// line. Non-input entries don't get timed and leave this + /// `None`. + pub started_at: Option, + /// When the corresponding eval finished. `None` while the + /// eval is still running (renderer shows live-updating + /// elapsed time from `started_at`); `Some(t)` freezes the + /// timer at the final duration once the batch completes. + pub completed_at: Option, + /// Non-`None` marks this entry as a collapsible NONE-severity + /// block (Vivado's non-diagnostic output: tables, banners, + /// section headers). `Some(true)` = collapsed (renderer shows + /// a single `▶` placeholder with a preview + hidden-line + /// count); `Some(false)` = expanded (all lines render dimmed + /// with a `▼` marker on the first line). `None` = normal + /// entry, no collapse handling. + /// + /// Only NONE blocks get this treatment — diagnostics are + /// always shown at full fidelity so a user scanning + /// scrollback for a WARNING never has to expand anything to + /// find it. + pub collapse_state: Option, + /// True when this entry originated from a Vivado CRITICAL + /// WARNING (not a plain ERROR). The two share + /// [`ScrollbackKind::Error`] for rendering — same red gutter + /// — but the diagnostics finder uses this flag to let the + /// user filter Critical warnings independently of plain + /// Errors. Default `false` for entries pushed by paths that + /// don't know severity (Input echoes, Result returns, + /// synthetic Notices). + pub is_critical_warning: bool, + /// Index in `scrollback` of the [`ScrollbackKind::Input`] + /// entry this row belongs to, or `None` when the row + /// predates any input (startup notices) or IS itself an + /// Input. Set once at push time — the "current input" is + /// the last `Input` pushed, and every subsequent + /// non-`Input` entry inherits that idx as its parent. + /// + /// Drives the Mathematica-style "collapse everything under + /// this command" grouping. Renderer skips rows whose + /// parent's `group_collapsed` is `true`; diagnostic search + /// uses this to group results by command AND to expand the + /// parent group when the user jumps to a hidden result. + pub parent_input_idx: Option, + /// Only meaningful for [`ScrollbackKind::Input`] entries. + /// When `true`, every subsequent entry whose + /// `parent_input_idx` points at this input is hidden from + /// the renderer and skipped by mouse / diagnostic-jump + /// math. Toggled by Shift-click on the Input row itself. + /// Defaults to `false` — a freshly-pushed command shows its + /// output live. + pub group_collapsed: bool, + /// Only meaningful for [`ScrollbackKind::Input`] entries. + /// Number of `Error` / `CriticalWarning` children currently + /// attributed to this input's group. Renderer shows a red + /// `✗` badge on the collapsed Input row when this is >0, + /// so users can see something went wrong without expanding. + /// Bumped by `push` / `push_diag` when they attribute a + /// child to a parent input; never decremented (a scrollback + /// entry is never re-classified after push). + pub error_child_count: u32, + /// Only meaningful for [`ScrollbackKind::Input`] entries. + /// Sibling of [`error_child_count`] for `Warning` children. + /// Renderer shows an orange `⚠` badge — same glyph and + /// color as the [`ScrollbackKind::Warning`] gutter — on + /// the collapsed Input row when this is >0, so users can + /// see there are non-fatal issues without expanding. + pub warning_child_count: u32, +} + +/// Drag-selection over scrollback rows. Coordinates are `(row, col)` +/// indices into the post-wrap line list (see +/// [`crate::render::wrap_lines`]) — i.e. the same indexing the +/// renderer uses for `Paragraph::scroll`. `anchor` is where the user +/// pressed; `cursor` updates while dragging. The range may be +/// inverted (cursor before anchor) — callers normalize via +/// [`Selection::ordered`] before applying. +#[derive(Clone, Copy, Debug)] +pub struct Selection { + pub anchor: (usize, usize), + pub cursor: (usize, usize), +} + +impl Selection { + /// Return `(start, end)` with `start <= end` so callers don't + /// have to special-case backwards drags. + pub fn ordered(&self) -> ((usize, usize), (usize, usize)) { + if self.anchor <= self.cursor { + (self.anchor, self.cursor) + } else { + (self.cursor, self.anchor) + } + } +} + +/// REPL meta-command catalog. Each entry is `(label, hint)` where +/// `label` is the full `:command` token (including the leading +/// colon, since that's what the user types and what Tab-completion +/// replaces) and `hint` is a one-line description shown in the +/// completion popup. +/// +/// Keep in sync with the `match` in [`App::run_meta_command`] and +/// with the cheat-sheet rows in [`crate::popup::HELP_ROWS`]. +pub const META_COMMANDS: &[(&str, &str)] = &[ + (":load", "evaluate a file's contents in this session"), + (":libs", "list loaded libraries + symbol counts"), + (":quit", "exit the REPL"), + (":exit", "exit the REPL (alias of :quit)"), + (":q", "exit the REPL (alias of :quit)"), + ( + ":restart", + "restart the Vivado worker (not yet implemented)", + ), +]; + +#[derive(Clone, Debug)] +pub struct ReverseSearch { + /// The substring the user is searching for. + pub query: String, + /// Index in [`History::entries`] of the current match. + pub match_index: Option, + /// The matched entry's text, cloned for the UI's static lifetime. + pub match_text: String, +} + +pub struct App { + opts: ReplOptions, + input: TextArea<'static>, + history: History, + /// Where the up/down (Ctrl-P/Ctrl-N) history walk is currently + /// positioned. `None` means "composing a fresh entry" — the + /// state any new keypress (other than Ctrl-P/N themselves) + /// drops back into so editing after walking history doesn't + /// keep recalling stale entries. `Some(i)` indexes + /// `History::entries()` directly. + history_cursor: Option, + /// In-progress draft saved when the user first walks back into + /// history with Ctrl-P. Restored on Ctrl-N past the newest + /// entry, so the draft they were typing isn't lost. + history_draft: String, + /// Session state, shareable across threads so background + /// prepare tasks (`dispatch_eval_with_echo`) can hold a read + /// guard for the ~seconds-to-minutes of a large `src` import + /// while the main event loop's frequent lookups (Ctrl-P + /// history walk, tab-completion, signature-help refresh, + /// input-completeness check) grab their own concurrent read + /// guards without contention. + /// + /// `RwLock` — not `Mutex` — because the main thread's typing- + /// time reads MUST run in parallel with a long-running + /// background prepare's read. A single writer (`commit` on a + /// successful prepare) briefly acquires the write lock; that + /// happens on the main task after the prepare returns, so + /// there's no active reader to wait for. + session: std::sync::Arc>, + /// Shared "already loaded in Vivado" map. Handed to the RPC + /// handler at spawn time; refreshed from + /// `session.loaded_paths()` after every `commit` so + /// `compile_htcl_module` skips re-shipping files whose + /// procs are already installed. Correctness invariant lives + /// with [`vw_vivado::SharedPreload`]. + preload: vw_vivado::SharedPreload, + scrollback: Vec, + /// Segments Vivado's classified stream into per-chunk Diagnostic + /// entries and grouped NONE-block collapsibles before pushing + /// into scrollback. Same accumulator vw-cli uses, but here it + /// lives on `App` because Stream events flow through the main + /// task's single event loop — no cross-thread sharing needed. + /// Diagnostic blocks and NONE blocks land in scrollback via + /// [`Self::push`] and [`Self::push_none_block`] respectively. + block_acc: vw_vivado::BlockAccumulator, + scrollback_scroll: u16, + /// Whether the terminal is currently capturing mouse events. + /// Default ON since we implement our own drag-to-select + + /// clipboard copy (see [`Selection`]). F2 toggles it off for + /// users who'd rather use terminal-native selection (which + /// requires capture to be disabled because the protocol is + /// all-or-nothing). + mouse_capture: bool, + /// Last scrollback render area, captured by `ui::draw_scrollback` + /// each frame. Lets mouse handlers map screen coords back to + /// scrollback-local cells without round-tripping through the UI. + scrollback_area: Option, + /// Active drag-selection in scrollback. Coordinates are + /// `(row, col)` indices into the post-wrap scrollback line list + /// — see `render::wrap_lines`. `None` outside of an active drag + /// or after a copy completes. + selection: Option, + /// Tail-follow mode. When `true`, the renderer pins the + /// effective scroll offset to the bottom of the wrapped-row + /// list — same model as `tail -f` or a fresh terminal. Manual + /// scroll-up flips this off so the user can read older content + /// without the view jumping out from under them; scrolling + /// back down to the bottom flips it back on. Submitting a new + /// command resets to `true`. + scrollback_follow: bool, + /// The effective scroll offset the renderer used on the most + /// recent frame. Written by `ui::draw_scrollback`, read by the + /// mouse / keyboard scroll handlers so a manual move from + /// tail-follow mode "takes over" the rendered position rather + /// than jumping back to whatever stale value is in + /// `scrollback_scroll`. + last_rendered_scroll: u16, + /// The `max_scroll` (= wrapped rows − viewport height) the + /// renderer computed on the most recent frame. Written by + /// `ui::draw_scrollback`; consulted by [`Self::scroll_by`] so a + /// downward scroll that lands at (or past) the bottom re-engages + /// tail-follow. This is safe now that the wrapped-row count is + /// exact — the old auto-re-engage misfired on large output + /// because it compared raw `text.lines().count()` against a + /// heuristic threshold. + last_max_scroll: u16, + /// Scrollback entry index the user last jumped to via the + /// diagnostic-finder popup (Ctrl-F → Enter). While `Some`, the + /// renderer paints a persistent left-gutter marker on every + /// wrapped row of that entry so the user can spot it in a + /// busy log. Alt-C clears it. `None` on startup and after + /// clear; not affected by scrolling or new entries appending. + marker_entry: Option, + /// One-shot request from the popup layer to scroll the + /// specified scrollback entry into view on the next frame. + /// Consumed by `ui::draw_scrollback` — it computes the + /// wrapped-row offset of that entry (from the same per-entry + /// count pass it already does) and writes it into + /// `scrollback_scroll` + disengages `scrollback_follow`. + /// Kept as a `usize` scrollback-idx rather than a pre-computed + /// row offset because the offset depends on area.width, which + /// the popup key handler doesn't know. + pending_jump: Option, + reverse_search: Option, + /// Active LSP-style popup over the input editor (completion, + /// signature help, hover). When `Some`, the key handler routes + /// navigation / dismissal keys to the popup BEFORE the catch-all + /// editor handoff. See [`crate::popup`]. + popup: Option, + worker_state: WorkerState, + /// How to cancel the eval in flight, cached from + /// `WorkerEvent::Started`. Called by the Ctrl-C handler rather + /// than going through `worker_tx`, which is blocked by the very + /// eval being cancelled. `None` before Started — in which case + /// Ctrl-C falls back to its non-eval behavior (clear the current + /// input). Local sessions signal vivado's process group; remote + /// ones ask the instance to do the same thing. + interrupt: Option, + worker_tx: mpsc::Sender, + eval_rx: mpsc::UnboundedReceiver, + /// Sender-side of the worker-event channel, kept on App so + /// spawned background tasks (currently: `prepare` on a + /// blocking thread) can post their completion back to the + /// event loop without a separate channel. The receiver + /// (`eval_rx`) is drained inside the main select loop. + event_tx: mpsc::UnboundedSender, + /// Origins of every command we shipped to the worker in the + /// current batch, in eval order, paired with an index for the + /// next not-yet-acknowledged command. Lets the stream handler + /// tag a mid-eval Vivado warning with `at : + /// in ` even when Vivado bypasses our Tcl-level + /// stack capture (the IP_Flow C++ property validators don't + /// call `::common::send_msg_id`, so neither shim override + /// fires — without this fallback those warnings would arrive + /// stack-less). + pending_origins: Vec, + /// Parallel to `pending_origins`: the expected return type of + /// each shipped command, when statically resolvable. Used by + /// the `EvalDone` handler to (a) skip the heuristic formatter + /// when the value is already type-formatted by the wrapped + /// Tcl, and (b) suppress the Result push entirely for + /// `unit`-typed expressions. + pending_return_types: Vec>, + /// Parallel to `pending_return_types`, one entry per lowered + /// command. True when the command is a top-level `set VAR + /// ` — the app suppresses the Result echo for those: + /// binding is a plumbing operation, not a display, and + /// echoing the raw value can leak the internal tagged Tcl + /// list form (`{Scalar x}` rather than the parens repr). + pending_is_set_binding: Vec, + /// For per-Input-entry timer freezing: one entry per + /// echoed top-level statement in the current batch, in + /// source order. Each carries the scrollback index of its + /// Input entry and the eval-index of the LAST lowered + /// command that came from that statement. When EvalDone + /// fires for that command index, we freeze the entry's + /// timer AND start the next entry's timer (so per-statement + /// durations in a multi-statement load batch are accurate + /// instead of all reading the whole-batch wall time). + pending_input_boundaries: Vec, + pending_eval_index: usize, + /// The batch we shipped to the worker but haven't yet seen a + /// result for. Held aside so a successful eval (and only a + /// successful one) commits to the session — and so the error + /// renderer can look up procs declared in this in-flight + /// batch (which aren't yet in `session`) when drilling into a + /// Tcl stack frame. + pending_batch: Option, + /// Set when `:quit` (or Ctrl-D on an empty buffer) fires, so the + /// outer loop bails out after the current frame. + exit: bool, + /// Auto-collapse policy for multi-line scrollback entries. + /// Loaded from `/.vw/repl.toml` at startup and + /// consulted by [`compute_collapse_state`] on every `push`. + collapse_mode: CollapseMode, + /// Index in `scrollback` of the most recently pushed + /// [`ScrollbackKind::Input`] entry, or `None` when no user + /// input has been submitted yet (startup notices only). + /// Every non-Input push after an Input records this index + /// as its `parent_input_idx`, forming a + /// Mathematica-notebook-style group under each command. + /// Renderer + click handler use these parent pointers to + /// hide / reveal a whole group as a unit. + current_input_idx: Option, +} + +enum WorkerState { + Starting, + Ready, + Running, + Down, +} + +/// Commands sent from the UI to the worker task. A batch ships one +/// or more lowered htcl statements; the worker fires `eval` per +/// item, sends one [`WorkerEvent::EvalDone`] per item, and stops at +/// the first failure so we don't keep running a script after it's +/// hit an error. +enum WorkerCmd { + EvalBatch(Vec), + Shutdown, +} + +/// Events sent from the worker task back to the UI. +enum WorkerEvent { + /// Vivado spawn succeeded. Carries the child's OS pid so the + /// UI can send SIGINT for eval cancellation without going + /// through the worker channel (which is blocked by the + /// in-flight eval it would need to cancel). `None` when the + /// PTY layer couldn't report a pid — cancellation degrades + /// to a no-op in that case. + Started { + interrupt: crate::Interrupt, + }, + /// One streaming chunk from the worker, with its source-of- + /// origin tag so the UI can render Vivado WARNING/ERROR lines + /// distinctly from user `puts` output. + Stream { + kind: vw_vivado::StreamKind, + data: String, + }, + /// One item of a batch completed. `origin` is the htcl source + /// location the lowered Tcl came from so the renderer can show + /// `file:line` rather than a Tcl stack trace pointing at the + /// shim. `last_in_batch` lets the UI know when to commit to + /// the session document. + EvalDone { + origin: crate::lower::Origin, + result: Result, + last_in_batch: bool, + }, + StartFailed(vw_eda::BackendError), + /// The background `prepare` (parse + validate + lower) + /// finished. `dispatch_eval_with_echo` spawns prepare on a + /// blocking thread so the event loop can render the input + /// echo + tick the timer while the (potentially minute-scale) + /// work runs. When this arrives, `handle_prepare_done` takes + /// over: surfaces warnings, commits or ships to the worker. + /// + /// `text` and `echo` are threaded through so the completion + /// handler has everything it needs without re-consulting the + /// input state (which may have moved on). + PrepareDone { + text: String, + echo: bool, + result: Result, + }, +} + +pub async fn run( + opts: ReplOptions, + worker: crate::Worker, +) -> Result<(), ReplError> { + enable_raw_mode()?; + let mut stdout = std::io::stdout(); + stdout.execute(EnterAlternateScreen)?; + // Mouse capture ON by default — the app implements its own + // drag-to-select + clipboard copy so users get text selection + // back (helix-style: app-level highlight rendered with + // `Modifier::REVERSED`, copied to the OS clipboard on mouse + // release). F2 toggles capture off if a user would rather use + // their terminal's native selection. + stdout.execute(EnableMouseCapture)?; + // Bracketed paste — the terminal wraps pasted content in + // sentinel byte sequences so we can distinguish it from + // manually-typed input. Without this, a pasted multi-line + // block delivers embedded `\n`s as raw Enter events, each of + // which the app treats as "submit" — the exact bug this + // enables us to fix. + stdout.execute(EnableBracketedPaste)?; + // Kitty keyboard protocol (minimal set) — asks the terminal + // to disambiguate keys that would otherwise collide (Ctrl+I + // vs. Tab, etc.). Doesn't help with Shift+Enter — this + // terminal (and many others) sends Shift+Enter as Ctrl+J + // instead of a distinct CSI-u sequence, so the outer key + // handler binds Ctrl+J to `insert_newline`. Unsupported + // flags are ignored, so this is safe everywhere. + let _ = stdout.execute(PushKeyboardEnhancementFlags( + KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES, + )); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + let result = run_inner(&mut terminal, opts, worker).await; + + disable_raw_mode()?; + let mut stdout = std::io::stdout(); + // No-op if capture was already disabled via F2. + let _ = stdout.execute(DisableMouseCapture); + let _ = stdout.execute(DisableBracketedPaste); + let _ = stdout.execute(PopKeyboardEnhancementFlags); + stdout.execute(LeaveAlternateScreen)?; + terminal.show_cursor()?; + result +} + +async fn run_inner( + terminal: &mut Terminal>, + opts: ReplOptions, + worker: crate::Worker, +) -> Result<(), ReplError> { + let (worker_tx, worker_rx) = mpsc::channel::(8); + let (event_tx, eval_rx) = mpsc::unbounded_channel::(); + let info_with_stack = opts.info_with_stack; + // At `--log-level=debug` the user wants the unclassified PTY + // firehose (banners, source echo, idle chatter) too; anything + // less filters it out. The firehose can't go to stderr under + // the TUI's alternate screen — that fd is the render surface — + // so route it to a per-process tempfile the user can `tail -f` + // from another terminal. Distinct from the raw byte-log wired + // downstream: this file holds only the leftover unclassified + // lines, whereas the raw log holds every byte. + let verbose = matches!(opts.log_level, vw_vivado::LogLevel::Debug); + let verbose_log_path = if verbose { + Some( + std::env::temp_dir() + .join(format!("vw-repl-vivado-{}.log", std::process::id())), + ) + } else { + None + }; + // Clone before moving into worker_task — App keeps its own + // handle so background prepare tasks can post PrepareDone + // events back onto the same channel the worker uses. + let event_tx_for_app = event_tx.clone(); + // Workspace-root discovery mirrors `vw run` / `vw check`: + // walk up from the initial-load file if provided, else from + // the current cwd, looking for the nearest `vw.toml`. Used + // to answer the htcl `vw::workspace_root` RPC — served + // through the Vivado shim's rpc_call primitive at eval time. + let rpc_workspace_root: Option = { + let start_dir = opts + .initial_load + .as_ref() + .and_then(|p| { + p.as_std_path().parent().map(std::path::Path::to_path_buf) + }) + .or_else(|| std::env::current_dir().ok()); + start_dir + .and_then(|d| vw_lib::find_workspace_dir(&d)) + .map(|p| p.into_std_path_buf()) + }; + // Load `/.vw/repl.toml` — currently just the `[ui] collapse` + // knob. Absent / malformed / no-workspace all fall back to + // defaults; a config file is optional infrastructure, not a + // startup dependency. + let repl_config = config::load(rpc_workspace_root.as_deref()); + // Shared preload map — grows as batches commit. Both the RPC + // handler (inside worker_task) and App hold clones of the same + // Arc so `compile_htcl_module` sees loaded-file updates as + // soon as `App::sync_preload_from_session` publishes them. + // See `vw_vivado::SharedPreload` for the correctness invariant. + let preload: vw_vivado::SharedPreload = std::sync::Arc::new( + std::sync::RwLock::new(std::collections::HashMap::new()), + ); + tokio::spawn(worker_task( + worker_rx, + event_tx, + verbose, + verbose_log_path.clone(), + info_with_stack, + opts.part.clone(), + opts.variant.clone(), + rpc_workspace_root, + preload.clone(), + worker, + )); + + let mut app = App::new( + opts, + worker_tx, + eval_rx, + event_tx_for_app, + repl_config.ui.collapse, + preload, + ); + if let Some(p) = verbose_log_path { + app.push( + ScrollbackKind::Notice, + format!( + "verbose output streaming to {} — `tail -f` from \ + another terminal", + p.display() + ), + ); + } + let mut crossterm_events = crossterm::event::EventStream::new(); + + loop { + terminal.draw(|f| ui::draw(f, &mut app))?; + if app.exit { + let _ = app.worker_tx.send(WorkerCmd::Shutdown).await; + return Ok(()); + } + + tokio::select! { + // Bias toward user input over worker events. Without + // this, a streaming eval (hundreds of stream chunks per + // second from Vivado) drowns out individual keystrokes: + // tokio::select! picks randomly when multiple branches + // are ready, and the worker channel is ready far more + // often. Result: Tab during eval looks dead because the + // keypress queues behind a long run of worker events. + // Biased order guarantees a ready crossterm event always + // wins, then the drain phase below catches up on worker + // events before the next draw. + biased; + maybe_event = crossterm_events.next() => { + match maybe_event { + Some(Ok(ev)) => app.handle_terminal_event(ev).await, + Some(Err(e)) => { + app.push(ScrollbackKind::Error, format!("terminal: {e}")); + } + None => { + app.exit = true; + } + } + } + Some(event) = app.eval_rx.recv() => { + app.handle_worker_event(event).await; + } + _ = tokio::time::sleep(Duration::from_millis(250)) => { + // Periodic wake: lets the spinner / "starting" status + // animate even when nothing else is happening. + } + } + // Drain additional pending events from BOTH streams + // before the next draw. Without this, a burst of N + // mouse-wheel events or worker stream chunks each + // triggers its own draw — even though only the final + // state matters visually — and the queue bloats faster + // than draws can keep up. + // + // The biased `select!` plus a wildcard always-ready + // branch acts as a non-blocking "is anything pending?" + // check: if neither real branch is immediately ready, + // the wildcard wins and we break out. Capped at 256 + // events per cycle so a sustained event firehose still + // yields back to drawing periodically (the user sees + // forward progress instead of "frozen until the whole + // burst is processed"). + for _ in 0..256 { + let made_progress = tokio::select! { + biased; + Some(maybe_event) = crossterm_events.next() => { + match maybe_event { + Ok(ev) => app.handle_terminal_event(ev).await, + Err(e) => app.push( + ScrollbackKind::Error, + format!("terminal: {e}"), + ), + } + true + } + Some(event) = app.eval_rx.recv() => { + app.handle_worker_event(event).await; + true + } + _ = std::future::ready(()) => false, + }; + if !made_progress { + break; + } + } + } +} + +impl App { + fn new( + opts: ReplOptions, + worker_tx: mpsc::Sender, + eval_rx: mpsc::UnboundedReceiver, + event_tx: mpsc::UnboundedSender, + collapse_mode: CollapseMode, + preload: vw_vivado::SharedPreload, + ) -> Self { + let mut input = TextArea::default(); + input.set_cursor_line_style(ratatui::style::Style::default()); + Self { + opts, + input, + history: History::load_default(), + history_cursor: None, + history_draft: String::new(), + session: std::sync::Arc::new( + std::sync::RwLock::new(Session::new()), + ), + preload, + scrollback: Vec::new(), + block_acc: vw_vivado::BlockAccumulator::new(), + scrollback_scroll: 0, + mouse_capture: true, + scrollback_area: None, + selection: None, + scrollback_follow: true, + last_rendered_scroll: 0, + last_max_scroll: 0, + marker_entry: None, + pending_jump: None, + reverse_search: None, + popup: None, + worker_state: WorkerState::Starting, + interrupt: None, + worker_tx, + eval_rx, + event_tx, + pending_batch: None, + pending_origins: Vec::new(), + pending_return_types: Vec::new(), + pending_is_set_binding: Vec::new(), + pending_input_boundaries: Vec::new(), + pending_eval_index: 0, + exit: false, + collapse_mode, + current_input_idx: None, + } + } + + /// Refresh the shared preload map from the current session's + /// `loaded_paths()`. Called immediately after every + /// `session.commit(...)` so the next `compile_htcl_module` + /// RPC sees the latest set of files installed in the Vivado + /// interpreter. + /// + /// Called AFTER commit (not before / concurrently) so the map + /// only names files whose lowered Tcl has been eval'd by + /// Vivado — the safety rule spelled out on + /// `vw_vivado::SharedPreload`. Wholesale replace (not merge) + /// so a file removed from the session — e.g. via a hot-edit + /// path in the future — drops out of the preload set too. + fn sync_preload_from_session(&self) { + let paths = { + let s = self.session.read().unwrap(); + s.loaded_paths() + }; + if let Ok(mut g) = self.preload.write() { + *g = paths; + } + } + + // --- queries used by ui.rs --------------------------------------- + + pub fn scrollback(&self) -> &[ScrollbackEntry] { + &self.scrollback + } + pub fn scrollback_scroll(&self) -> u16 { + self.scrollback_scroll + } + pub fn input_mut(&mut self) -> &mut TextArea<'static> { + &mut self.input + } + pub fn input_line_count(&self) -> usize { + self.input.lines().len() + } + pub fn reverse_search(&self) -> Option<&ReverseSearch> { + self.reverse_search.as_ref() + } + pub fn mouse_capture(&self) -> bool { + self.mouse_capture + } + pub fn selection(&self) -> Option { + self.selection + } + /// Called by `ui::draw_scrollback` each frame so subsequent + /// mouse events can translate absolute screen coords into + /// scrollback-local rows/cols. + pub fn set_scrollback_area(&mut self, area: Rect) { + self.scrollback_area = Some(area); + } + + pub fn scrollback_follow(&self) -> bool { + self.scrollback_follow + } + + /// Renderer-side writeback: records the scroll offset that was + /// actually used to paint the current frame. Mouse / keyboard + /// scroll handlers anchor their deltas off this so transitioning + /// out of tail-follow doesn't jump back to a stale + /// `scrollback_scroll` value. + pub fn set_last_rendered_scroll(&mut self, offset: u16) { + self.last_rendered_scroll = offset; + } + + /// Renderer-side writeback for the current frame's `max_scroll` + /// (wrapped rows − viewport height). Consulted by + /// [`Self::scroll_by`] so a downward wheel/PageDown that lands + /// at the bottom re-engages tail-follow. + pub fn set_last_max_scroll(&mut self, max_scroll: u16) { + self.last_max_scroll = max_scroll; + } + + /// Renderer-invoked writeback used by the pending-jump path — + /// the popup handler doesn't know area.width so it can't + /// compute the target scroll offset itself, and instead + /// stashes a `pending_jump` scrollback index that the + /// renderer translates into an offset and writes back here. + /// General-purpose scroll changes go through + /// [`Self::scroll_by`]; this setter is not a substitute. + pub fn set_scrollback_scroll(&mut self, offset: u16) { + self.scrollback_scroll = offset; + } + + /// Scrollback entry index the user last jumped to (or `None` + /// if the marker has been cleared or was never set). Consulted + /// by `ui::draw_scrollback` to paint the persistent gutter + /// marker on that entry's wrapped rows. + pub fn marker_entry(&self) -> Option { + self.marker_entry + } + + /// Consume any pending "scroll this entry into view" request + /// from the popup layer. Returns `Some(idx)` exactly once per + /// jump request; subsequent frames return `None`. The renderer + /// uses the per-entry wrapped-row count it computes anyway to + /// translate `idx` into an absolute scroll offset — that + /// translation needs area.width, which is why the popup can't + /// pre-compute it. + pub fn take_pending_jump(&mut self) -> Option { + self.pending_jump.take() + } + + /// Toggle terminal mouse capture. Writes the enable/disable + /// sequence directly to stdout — the alternate-screen / raw-mode + /// context that `run()` set up is still active. + fn toggle_mouse_capture(&mut self) { + let mut stdout = std::io::stdout(); + let _ = if self.mouse_capture { + stdout.execute(DisableMouseCapture) + } else { + stdout.execute(EnableMouseCapture) + }; + self.mouse_capture = !self.mouse_capture; + } + pub fn worker_state(&self) -> WorkerStatusView { + match self.worker_state { + WorkerState::Starting => WorkerStatusView::Starting, + WorkerState::Ready => WorkerStatusView::Ready, + WorkerState::Running => WorkerStatusView::Running, + WorkerState::Down => WorkerStatusView::Down, + } + } + pub fn eval_in_flight(&self) -> bool { + matches!(self.worker_state, WorkerState::Running) + } + + /// Whether the parser considers the current input buffer ready + /// to ship. Drives the input-area title and Enter behavior. + pub fn input_is_complete(&self) -> bool { + let buf = self.current_input_text(); + let session = self.session.read().unwrap(); + is_buffer_complete(&buf, &session.signature_table()) + } + + fn current_input_text(&self) -> String { + self.input.lines().join("\n") + } + + /// Translate the input editor's `(row, col)` cursor into a byte + /// offset within `current_input_text()`. Returns `None` when the + /// cursor is past EOF (shouldn't happen — TextArea keeps it in + /// bounds — but defensive). + fn cursor_byte_offset(&self) -> Option { + let (row, col) = self.input.cursor(); + let buffer = self.current_input_text(); + let line_idx = vw_htcl::line_index::LineIndex::new(&buffer); + Some(line_idx.offset_of(vw_htcl::line_index::LineCol { + line: row as u32, + character: col as u32, + })) + } + + /// Map an input-buffer (row, col) to a screen cell within the + /// input editor's rendered area. Used to anchor popups (slice 4+) + /// just below the cursor. Returns `None` when no scrollback area + /// has been captured yet (shouldn't happen post-first-render but + /// defensive against early key events). + fn cursor_screen_cell(&self) -> Option<(u16, u16)> { + // We don't have direct access to the input area Rect here + // (only the scrollback's). The popup anchor instead uses a + // best-effort approximation: assume the input area starts + // just below the scrollback and the popup will clamp to the + // frame in the renderer. Concretely: row = bottom of the + // visible scrollback (where the input border sits) + cursor + // row in the editor, col = cursor col + the input area's + // left edge (we use scrollback's x which they share). + let area = self.scrollback_area?; + let (row, col) = self.input.cursor(); + // +1 to step past the input box's top border, +area.y to land + // inside the input region. The renderer's popup positioning + // does additional clamping so over- and under-shoots are safe. + let screen_y = area.y + area.height + 1 + row as u16; + let screen_x = area.x + 1 + col as u16; + Some((screen_x, screen_y)) + } + + /// Trigger a completion popup at the current cursor position. No-op + /// when there are no completions to show. Called from the Tab key + /// handler. + fn trigger_completion(&mut self) { + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + // Parse ONLY the in-flight input. Earlier we tried merging + // session.merged_source() (~6MB after `src @vivado-cmd`) + // into the analysis source so `util::` would see + // session-known procs. That had two fatal problems: + // + // 1. Per-Tab cost was a multi-MB parse + a multi-MB + // `cmdline::analyze` walk-back. The UI froze for + // seconds; queued keypresses (ctrl-D, backspace) + // drained after the parse finished. + // 2. `cmdline::analyze` balances `[` / `]` but doesn't + // know about `#` comments. The auto-generated Vivado + // docs contain `[get_hw_sysmons]`, `[Common 17-39]`, + // etc. inside `## doc-comment` blocks; an unmatched + // bracket in those docs put the analyzer in + // "inside-a-substitution" state forever, blowing past + // every newline and never finding the command + // boundary. End result: `partial="util::"` but + // `head_words` was the entire 6 MB session. + // + // Cheaper, correct approach: analyze only the in-flight + // input (small, fast, no rogue brackets), then pull + // candidate proc names directly out of + // `Session::signature_table()` — that's a HashMap built + // from already-parsed batches; O(N) iteration over + // existing data instead of an MB-scale reparse + walk. + let parsed = vw_htcl::parser::parse(&input); + let cmd_line = vw_htcl::cmdline::analyze(&input, offset); + let session_guard = self.session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + // The currently-shipping batch hasn't committed yet — session + // commit only happens on EvalDone(last_in_batch=true). During + // the prime.htcl load (e.g. `src @vivado-cmd` taking 50+ + // seconds), every proc the user wants to complete on (util::*, + // create_*, …) lives in `pending_batch.document` but NOT in + // session.signature_table(). Surface those too — they're + // already parsed; cost is one `signature_table` walk over the + // in-flight document's stmts. + let pending_sigs: std::collections::HashMap< + String, + &vw_htcl::ProcSignature, + > = self + .pending_batch + .as_ref() + .map(|b| vw_htcl::signature_table(&b.document)) + .unwrap_or_default(); + let mut items: Vec = Vec::new(); + // Meta-command branch: `:load`, `:quit`, etc. Detected by + // a leading `:` on the partial — these are App-side + // commands, not htcl, so they live above the cmdline + // analyzer's notion of command position. + if cmd_line.partial.starts_with(':') { + for (label, hint) in META_COMMANDS { + if label.starts_with(cmd_line.partial) { + items.push(vw_htcl::complete::Completion { + label: label.to_string(), + kind: vw_htcl::complete::CompletionKind::Proc, + detail: Some(hint.to_string()), + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } else if cmd_line.in_command_position() { + // Proc-name completion: union of session + pending + + // in-flight proc names, filtered by the partial prefix. + let mut names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + for name in session_sigs.keys() { + names.insert(name.clone()); + } + for name in pending_sigs.keys() { + names.insert(name.clone()); + } + for name in input_sigs.keys() { + names.insert(name.clone()); + } + for name in names { + if name.starts_with(cmd_line.partial) { + items.push(vw_htcl::complete::Completion { + label: name, + kind: vw_htcl::complete::CompletionKind::Proc, + detail: None, + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } else if let Some(cmd_name) = cmd_line.command_name() { + // Flag completion: look up the called proc's signature + // in either source and emit its flag args. Matches the + // shape of `vw_htcl::complete::complete_at`'s flag path + // but uses our union of session+input signatures. + let sig = session_sigs + .get(cmd_name) + .copied() + .or_else(|| pending_sigs.get(cmd_name).copied()) + .or_else(|| input_sigs.get(cmd_name).copied()); + if let Some(sig) = sig { + let used: Vec<&str> = cmd_line.used_flags().collect(); + let needle_no_dash = cmd_line + .partial + .strip_prefix('-') + .unwrap_or(cmd_line.partial); + // Required (no @default) flags first, then optional, + // alphabetical within each group. Matches the + // signature-help popup's ordering so the user sees + // the same priority across surfaces. + for &i in &sorted_arg_indices(sig) { + let arg = &sig.args[i]; + let label = format!("-{}", arg.name); + if used.contains(&label.as_str()) { + continue; + } + if arg.name.starts_with(needle_no_dash) { + // Detail shows the type + default when + // available, so the completion popup row + // hints at what each flag expects without + // requiring the user to open hover. Format + // mirrors the sig-help line: `type = value`. + let detail = build_flag_detail(arg); + items.push(vw_htcl::complete::Completion { + label, + kind: vw_htcl::complete::CompletionKind::Flag, + detail, + documentation: None, + replace: cmd_line.partial_span, + insert_text: None, + snippet: false, + }); + } + } + } + } + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + if let Some(popup) = crate::popup::CompletionPopup::new(items, anchor) { + self.popup = Some(crate::popup::PopupState::Completion(popup)); + } + } + + /// Route a key event to the active popup. Returns `true` when the + /// key was consumed (navigation / accept / dismiss); `false` lets + /// the key fall through to the rest of the handler. + fn handle_popup_key(&mut self, key: crossterm::event::KeyEvent) -> bool { + use crossterm::event::KeyCode; + let Some(popup) = self.popup.as_mut() else { + return false; + }; + match popup { + crate::popup::PopupState::Completion(comp) => { + match key.code { + KeyCode::Up => { + comp.move_up(); + true + } + KeyCode::Down => { + comp.move_down(); + true + } + KeyCode::Esc => { + self.popup = None; + true + } + KeyCode::Enter if !key.modifiers.is_empty() => { + // Any modifier on Enter (Shift, Alt, Ctrl, + // combos) is the "keep typing" escape + // hatch. Don't let the popup consume it — + // dismiss the popup so the outer handler + // can insert a newline. Terminals differ + // on which modifier they attach; accept + // any of them. + self.popup = None; + false + } + KeyCode::Char('j') + if key.modifiers.contains( + crossterm::event::KeyModifiers::CONTROL, + ) => + { + // Shift+Enter on legacy terminals arrives + // as Ctrl+J — see the outer handler's + // matching branch for the rationale. Let + // it through so the outer handler can + // insert a newline. + self.popup = None; + false + } + KeyCode::Enter => { + if let Some(item) = comp.current().cloned() { + self.apply_completion(&item); + } + self.popup = None; + true + } + KeyCode::Tab => { + // Tab re-triggers — just cycle for now. + comp.move_down(); + true + } + _ => { + // Any other key dismisses the popup and falls + // through to the editor — typical IDE + // behavior where you can keep typing past the + // popup to refine your input. + self.popup = None; + false + } + } + } + crate::popup::PopupState::Help(_) => { + // Any keystroke dismisses the help modal. We CONSUME + // the dismissing key (return true) so it doesn't + // also act on the input — pressing Ctrl-H to open + // then any other key to close shouldn't accidentally + // type the close key into the editor. + self.popup = None; + true + } + crate::popup::PopupState::SignatureHelp(sig) => { + // Signature help is the background auto-show; it + // doesn't consume keys, falls through to the editor. + // Esc lets users hide it without clearing input. + if key.code == KeyCode::Esc { + self.popup = None; + return true; + } + // Shift-↑ / Shift-↓: scroll through args when the + // signature is too tall to fit. Picked over + // Ctrl-↑/Ctrl-↓ because macOS reserves those for + // Mission Control. We consume these chords (return + // true) so they don't ALSO scroll the scrollback. + // Step by 1 — fine-grained because each arg is a + // self-contained row. + if key.modifiers.contains(KeyModifiers::SHIFT) { + match key.code { + KeyCode::Up => { + sig.scroll_offset = + sig.scroll_offset.saturating_sub(1); + return true; + } + KeyCode::Down => { + sig.scroll_offset = + sig.scroll_offset.saturating_add(1); + return true; + } + _ => {} + } + } + false + } + crate::popup::PopupState::Hover(_) => { + // Hover dismisses on any keystroke. We consume the + // key so the dismissing keystroke doesn't also act + // on the input — Ctrl-Y to open + any key to close + // shouldn't smuggle that key into the buffer. + self.popup = None; + true + } + crate::popup::PopupState::SymbolSearch(picker) => { + use crate::symbol_search::PickerView; + match key.code { + KeyCode::Esc => { + self.popup = None; + true + } + KeyCode::Up => { + picker.move_up(); + true + } + KeyCode::Down => { + picker.move_down(); + true + } + KeyCode::Tab => { + picker.toggle_view(); + true + } + KeyCode::Backspace => { + if picker.view == PickerView::Symbols { + picker.pop_char(); + } + true + } + KeyCode::Char(c) + if !key.modifiers.contains(KeyModifiers::CONTROL) + && picker.view == PickerView::Symbols => + { + picker.push_char(c); + true + } + KeyCode::Enter => { + match picker.view { + PickerView::Symbols => { + if let Some(sym) = + picker.current_symbol().cloned() + { + self.popup = None; + self.insert_at_cursor_replacing_word( + &sym.name, + ); + } else { + self.popup = None; + } + } + PickerView::Libraries => { + picker.apply_library_filter(); + } + } + true + } + _ => true, // swallow other keys; popup stays open + } + } + crate::popup::PopupState::DiagnosticSearch(picker) => { + match (key.code, key.modifiers) { + (KeyCode::Esc, _) => { + self.popup = None; + true + } + (KeyCode::Up, _) => { + picker.move_up(); + true + } + (KeyCode::Down, _) => { + picker.move_down(); + true + } + (KeyCode::Char('e'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Error); + true + } + (KeyCode::Char('w'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Warning); + true + } + (KeyCode::Char('n'), m) + if m.contains(KeyModifiers::CONTROL) => + { + picker.toggle_kind(ScrollbackKind::Notice); + true + } + (KeyCode::Char('k'), m) + if m.contains(KeyModifiers::CONTROL) => + { + // Ctrl-K toggles the Critical-warning + // subset filter. Only meaningful inside + // the popup — outside, Ctrl-K is + // scrollback-up, which the popup handler + // shadows while active. + picker.toggle_critical(); + true + } + (KeyCode::Backspace, _) => { + picker.pop_char(); + true + } + (KeyCode::Char(c), m) + if !m.contains(KeyModifiers::CONTROL) => + { + picker.push_char(c); + true + } + (KeyCode::Enter, _) => { + // Snapshot the target BEFORE dropping the + // popup — `picker` borrows through + // `self.popup`; setting `self.popup = None` + // invalidates it. + let target = picker + .current() + .map(|it| (it.scrollback_idx, it.kind)); + self.popup = None; + if let Some((idx, kind)) = target { + self.jump_to_scrollback_entry(idx, kind); + } + true + } + _ => true, + } + } + } + } + + /// Insert / replace text from a chosen completion. Replaces the + /// byte range `item.replace` (the partial word under the cursor, + /// or a zero-width insertion point) with `item.label`. + fn apply_completion(&mut self, item: &vw_htcl::complete::Completion) { + let buffer = self.current_input_text(); + let start = item.replace.start as usize; + let end = (item.replace.end as usize).min(buffer.len()); + if start > buffer.len() { + return; + } + let mut new_buffer = String::with_capacity( + buffer.len() - (end - start) + item.label.len(), + ); + new_buffer.push_str(&buffer[..start]); + new_buffer.push_str(&item.label); + new_buffer.push_str(&buffer[end..]); + // Place cursor just after the inserted label. + let new_cursor_byte = start + item.label.len(); + self.replace_input_with_cursor(new_buffer, new_cursor_byte); + } + + /// Replace the input buffer with `text` and move the cursor to + /// the byte offset `cursor_byte`. Cursor offset is translated to + /// (row, col) via `LineIndex`. Used by completion accept. + fn replace_input_with_cursor(&mut self, text: String, cursor_byte: usize) { + use tui_textarea::TextArea; + let line_idx = vw_htcl::line_index::LineIndex::new(&text); + // Build the textarea fresh from the new content (tui-textarea + // doesn't offer a "replace everything" API; recreating is the + // documented way per its issue tracker). + let lines: Vec = + text.split('\n').map(|s| s.to_string()).collect(); + let mut ta = TextArea::new(lines); + ta.set_cursor_line_style(ratatui::style::Style::default()); + // Position cursor. + let lc = line_idx.position(cursor_byte as u32); + let target_row = lc.line as usize; + let target_col = lc.character as usize; + // Use the textarea's `move_cursor` API. + while ta.cursor().0 < target_row { + ta.move_cursor(tui_textarea::CursorMove::Down); + } + while ta.cursor().0 > target_row { + ta.move_cursor(tui_textarea::CursorMove::Up); + } + while ta.cursor().1 < target_col { + ta.move_cursor(tui_textarea::CursorMove::Forward); + } + while ta.cursor().1 > target_col { + ta.move_cursor(tui_textarea::CursorMove::Back); + } + self.input = ta; + self.history_cursor = None; + } + + /// Whether a popup is currently open (renderer queries this to + /// decide whether to draw the popup overlay layer). + pub fn popup_state(&self) -> Option<&crate::popup::PopupState> { + self.popup.as_ref() + } + + /// Refresh signature help after a buffer-mutating keystroke. + /// Looks up the proc the cursor sits in by name across session, + /// pending batch, and in-flight input; populates a + /// `PopupState::SignatureHelp` with its args + active parameter. + /// Dismisses any existing sig-help popup when nothing matches. + /// + /// Coexistence rule: never displaces a Completion or Help popup + /// (the user explicitly opened those; sig-help is the background + /// auto-show). + fn refresh_signature_help(&mut self) { + // Don't fight explicit popups. + if matches!( + self.popup, + Some(crate::popup::PopupState::Completion(_)) + | Some(crate::popup::PopupState::Help(_)) + ) { + return; + } + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + self.dismiss_signature_help(); + return; + }; + let cmd_line = vw_htcl::cmdline::analyze(&input, offset); + let Some(name) = cmd_line.command_name() else { + self.dismiss_signature_help(); + return; + }; + // Find the proc's signature + doc comments. Signatures live + // in session.signature_table() / pending / in-flight; doc + // comments require walking the source document (kept on the + // Command, not the ProcSignature). + let parsed = vw_htcl::parser::parse(&input); + // Arc-clone the session handle first — this drops the + // borrow on `self` immediately, so we can call `self.push` / + // `self.dismiss_signature_help` etc. later without the + // read guard blocking the mutable borrow of self. + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + let pending_doc = self.pending_batch.as_ref().map(|b| &b.document); + let pending_sigs = pending_doc + .map(|d| vw_htcl::signature_table(d)) + .unwrap_or_default(); + let sig = session_sigs + .get(name) + .copied() + .or_else(|| pending_sigs.get(name).copied()) + .or_else(|| input_sigs.get(name).copied()); + let Some(sig) = sig else { + self.dismiss_signature_help(); + return; + }; + // Look up doc comments by walking docs in priority order + // (input → pending → session). Most recent wins, which + // matches Tcl's "later proc shadows earlier" semantics that + // the lowerer already uses. + let mut doc_comments: &[String] = &[]; + if let Some(d) = lookup_proc_doc_comments(&parsed.document, name) { + doc_comments = d; + } else if let Some(doc) = pending_doc { + if let Some(d) = lookup_proc_doc_comments(doc, name) { + doc_comments = d; + } + } else { + // Walk session batches in reverse (newest first). + for batch in session_guard.batches_for_doc_search() { + if let Some(d) = lookup_proc_doc_comments(&batch.document, name) + { + doc_comments = d; + break; + } + } + } + // Build the display permutation (required → optional, + // alphabetical within each group) and reorder args + the + // active-parameter index accordingly. + let display_order = sorted_arg_indices(sig); + let active_decl = compute_active_parameter(sig, &cmd_line); + let active = active_decl.and_then(|orig| { + display_order + .iter() + .position(|&i| i == orig as usize) + .map(|i| i as u32) + }); + let args: Vec = display_order + .iter() + .map(|&i| { + let a = &sig.args[i]; + crate::popup::SigHelpArg { + name: a.name.clone(), + type_str: a.type_annotation.as_ref().map(render_type), + default_str: format_default_value(a), + } + }) + .collect(); + let return_type = sig.return_type.as_ref().map(render_type); + let doc_brief = vw_htcl::doc::brief(doc_comments); + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + // Preserve any user-set scroll offset across refreshes — + // manual Ctrl-↑/↓ scrolling shouldn't be reset by every + // keystroke. The renderer clamps the offset to a valid + // range, so an offset that's stale for the new arg list + // (e.g. switching to a smaller proc) silently snaps back. + let prev_scroll = match self.popup.as_ref() { + Some(crate::popup::PopupState::SignatureHelp(p)) => p.scroll_offset, + _ => 0, + }; + let popup = crate::popup::SignatureHelpPopup { + proc_name: name.to_string(), + args, + return_type, + doc_brief, + active: active.map(|a| a as usize), + anchor, + scroll_offset: prev_scroll, + }; + self.popup = Some(crate::popup::PopupState::SignatureHelp(popup)); + } + + /// Open the fuzzy symbol picker (Ctrl-T). Builds a fresh + /// `SymbolIndex` snapshot at open time — it stays stable while + /// the popup is alive so the result-row indices don't shift + /// out from under the user. The index is small to build + /// (walks already-parsed Documents) so per-open cost is fine. + fn trigger_symbol_search(&mut self) { + let input = self.current_input_text(); + let parsed = vw_htcl::parser::parse(&input); + let session_guard = self.session.read().unwrap(); + let index = + std::sync::Arc::new(crate::symbol_index::SymbolIndex::build( + &session_guard, + self.pending_batch.as_ref(), + Some(&parsed.document), + )); + let picker = crate::symbol_search::SymbolPicker::new(index); + self.popup = Some(crate::popup::PopupState::SymbolSearch(picker)); + } + + /// Ctrl-F opens the diagnostics finder. Snapshots the current + /// scrollback (Error/Warning/Notice only) and hands it to the + /// picker. Snapshot semantics: rows appended after open aren't + /// visible in this session of the picker; user reopens to see + /// them. Prevents result-index churn while typing a query + /// against a still-streaming scrollback. + fn trigger_diagnostic_search(&mut self) { + let picker = crate::diag_search::DiagnosticPicker::from_scrollback( + &self.scrollback, + ); + self.popup = Some(crate::popup::PopupState::DiagnosticSearch(picker)); + } + + /// Set the marker on `idx`, request the next render to scroll + /// that entry into view, and disengage tail-follow. Called + /// when the diagnostics-finder popup Accept fires; the + /// scrolling itself happens in the renderer next frame + /// (needs area.width to translate entry-idx → row offset). + /// `_kind` is captured for future use — right now the marker + /// styling is kind-agnostic (fixed color), but a per-kind + /// tint would use it. + fn jump_to_scrollback_entry(&mut self, idx: usize, _kind: ScrollbackKind) { + if idx >= self.scrollback.len() { + return; + } + // Expand the containing input group if it's collapsed. + // Without this, jumping to a diagnostic scrolls to a row + // that renders as 0 rows (child of a collapsed group), so + // the marker lands on empty space and the user sees + // nothing near the target. Expanding first makes the + // target row actually visible. + let parent_idx = self.scrollback[idx].parent_input_idx; + if let Some(pidx) = parent_idx { + if let Some(parent) = self.scrollback.get_mut(pidx) { + if matches!(parent.kind, ScrollbackKind::Input) { + parent.group_collapsed = false; + } + } + } + self.marker_entry = Some(idx); + self.pending_jump = Some(idx); + self.scrollback_follow = false; + } + + /// Alt-C clears the persistent marker. No-op when no marker + /// is set; harmless to press repeatedly. + fn clear_marker(&mut self) { + self.marker_entry = None; + } + + /// Insert `text` at the current cursor position, replacing the + /// identifier-under-cursor (if any). Used by the symbol-picker + /// Enter handler to insert a chosen symbol name in place of the + /// partial word the user is typing. + fn insert_at_cursor_replacing_word(&mut self, text: &str) { + let buffer = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + let off = offset as usize; + // Find word boundaries around the cursor (same rule as + // `ident_under_cursor`). + let bytes = buffer.as_bytes(); + let is_word_byte = |b: u8| -> bool { + b.is_ascii_alphanumeric() || b == b'_' || b == b':' + }; + let mut start = off; + while start > 0 && is_word_byte(bytes[start - 1]) { + start -= 1; + } + let mut end = off; + while end < bytes.len() && is_word_byte(bytes[end]) { + end += 1; + } + let mut new_buffer = + String::with_capacity(buffer.len() - (end - start) + text.len()); + new_buffer.push_str(&buffer[..start]); + new_buffer.push_str(text); + new_buffer.push_str(&buffer[end..]); + let new_cursor = start + text.len(); + self.replace_input_with_cursor(new_buffer, new_cursor); + } + + /// Hover-under-cursor: open a popup showing the proc / + /// variable / enum the cursor is on. Tries + /// [`vw_htcl::hover_at`] on the in-flight input first (catches + /// local vars, in-buffer proc decls, enum decls). Falls back to + /// a session-aware lookup: extracts the identifier under the + /// cursor and resolves it against + /// `Session::signature_table()` / pending / in-flight, so + /// `Ctrl-Y` on a `util::props` call surfaces the library's docs + /// even though the proc was defined in a separate session + /// batch. + fn trigger_hover(&mut self) { + let input = self.current_input_text(); + let Some(offset) = self.cursor_byte_offset() else { + return; + }; + let parsed = vw_htcl::parser::parse(&input); + let anchor = self.cursor_screen_cell().unwrap_or((0, 0)); + // Pass 1: in-document hover. Handles ProcDef, ProcArgDef, + // CallSite when the proc is in the buffer, CallArg, + // LocalVar, EnumDef. + if let Some(target) = + vw_htcl::hover_at(&parsed.document, &input, offset) + { + if let Some(popup) = hover_target_to_popup(target, anchor) { + self.popup = Some(crate::popup::PopupState::Hover(popup)); + return; + } + } + // Pass 2: session-aware proc lookup by identifier under + // cursor. Covers the common REPL case: cursor on a name + // referencing a session-loaded library proc. + let Some(name) = ident_under_cursor(&input, offset) else { + return; + }; + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let session_sigs = session_guard.signature_table(); + let pending_doc = self.pending_batch.as_ref().map(|b| &b.document); + let pending_sigs = pending_doc + .map(|d| vw_htcl::signature_table(d)) + .unwrap_or_default(); + let input_sigs = vw_htcl::signature_table(&parsed.document); + let sig = session_sigs + .get(name) + .copied() + .or_else(|| pending_sigs.get(name).copied()) + .or_else(|| input_sigs.get(name).copied()); + let Some(sig) = sig else { return }; + // Doc comments: walk newest source first (input → pending → + // session newest-first). + let mut doc_comments: &[String] = &[]; + if let Some(d) = lookup_proc_doc_comments(&parsed.document, name) { + doc_comments = d; + } else if let Some(doc) = pending_doc { + if let Some(d) = lookup_proc_doc_comments(doc, name) { + doc_comments = d; + } + } else { + for batch in session_guard.batches_for_doc_search() { + if let Some(d) = lookup_proc_doc_comments(&batch.document, name) + { + doc_comments = d; + break; + } + } + } + let title = render_proc_title(name, sig); + let body = vw_htcl::doc::reflow_doc_comments(doc_comments); + self.popup = + Some(crate::popup::PopupState::Hover(crate::popup::HoverPopup { + title, + body, + anchor, + })); + } + + /// Dismiss the signature-help popup if one is active. Leaves + /// Completion / Help popups alone. + fn dismiss_signature_help(&mut self) { + if matches!( + self.popup, + Some(crate::popup::PopupState::SignatureHelp(_)) + ) { + self.popup = None; + } + } + + /// Walk the input history by `delta` (negative = older, + /// positive = newer). Readline-style: first step back from the + /// "composing" position saves the current draft; stepping + /// past the newest entry restores it. Empty history is a no-op. + fn history_step(&mut self, delta: i32) { + let entries = self.history.entries(); + if entries.is_empty() { + return; + } + let cursor = match (self.history_cursor, delta) { + (None, d) if d >= 0 => return, // already at draft, can't go newer + (None, _) => { + // Stepping back from the draft for the first time — + // capture the in-progress text so Ctrl-N past the + // newest entry can restore it. + self.history_draft = self.current_input_text(); + entries.len().saturating_sub(1) + } + (Some(i), d) => { + let new = i as i32 + d; + if new < 0 { + 0 + } else if new >= entries.len() as i32 { + // Past the newest entry — drop back to draft. + self.history_cursor = None; + let draft = std::mem::take(&mut self.history_draft); + self.replace_input_with(&draft); + return; + } else { + new as usize + } + } + }; + self.history_cursor = Some(cursor); + let text = entries[cursor].clone(); + self.replace_input_with(&text); + } + + /// Reset the input buffer to `text`, placing the cursor at the + /// end. Used by history navigation and reverse-search accept. + fn replace_input_with(&mut self, text: &str) { + self.input = TextArea::default(); + for (i, line) in text.lines().enumerate() { + if i > 0 { + self.input.insert_newline(); + } + self.input.insert_str(line); + } + // If `text` ended with a newline, `lines()` drops it; preserve. + if text.ends_with('\n') { + self.input.insert_newline(); + } + } + + // --- event handling --------------------------------------------- + + fn handle_mouse_event(&mut self, mouse: MouseEvent) { + // Wheel events scroll the scrollback buffer. 3 lines per + // notch is the de-facto terminal-emulator default and + // matches what feels natural when you've held the wheel + // for half a second. Keyboard scroll (Ctrl-J/K) still + // jumps 5 — the wheel is finer-grained because the user + // can keep spinning. + // + // Direction: `scrollback_scroll` is ratatui's `scroll.y`, + // which counts lines skipped from the TOP of the buffer. + // Wheel-up should reveal older content above the viewport + // (terminal convention), which means moving the viewport + // UP through the buffer — i.e. SUBTRACTING from + // `scrollback_scroll`. Wheel-down does the reverse. + match mouse.kind { + MouseEventKind::ScrollUp => { + self.scroll_by(-3); + return; + } + MouseEventKind::ScrollDown => { + self.scroll_by(3); + return; + } + _ => {} + } + + // Drag-selection lives within the scrollback area only. + // Outside it, mouse events are ignored — the input box has + // its own selection model via tui-textarea and we don't + // want a click on a status bar to start a scrollback drag. + let Some(area) = self.scrollback_area else { + return; + }; + let in_area = mouse.column >= area.x + && mouse.column < area.x + area.width + && mouse.row >= area.y + && mouse.row < area.y + area.height; + + match mouse.kind { + MouseEventKind::Down(MouseButton::Left) if in_area => { + self.selection = Some(Selection { + anchor: self.cell_to_buffer(mouse.column, mouse.row, area), + cursor: self.cell_to_buffer(mouse.column, mouse.row, area), + }); + } + MouseEventKind::Drag(MouseButton::Left) + if self.selection.is_some() => + { + // Auto-scroll when the drag wanders past the + // top or bottom edge of the scrollback area so + // selections can extend beyond the current + // viewport. Crossterm fires drag events per + // cell of mouse movement, so the user wiggles + // the mouse at the edge to keep scrolling; + // simpler than tracking a "held at edge" timer + // and good enough for selection-extension UX. + let bottom = area.y + area.height; + if mouse.row >= bottom { + self.scroll_by(3); + } else if mouse.row < area.y { + self.scroll_by(-3); + } + // Clamp to the area: dragging outside still + // updates the cursor to the edge so selection + // can extend through the visible viewport even + // when the mouse strays. + let col = mouse + .column + .clamp(area.x, area.x + area.width.saturating_sub(1)); + let row = mouse + .row + .clamp(area.y, area.y + area.height.saturating_sub(1)); + let cursor = self.cell_to_buffer(col, row, area); + if let Some(sel) = self.selection.as_mut() { + sel.cursor = cursor; + } + } + MouseEventKind::Up(MouseButton::Left) => { + if let Some(sel) = self.selection.take() { + // Shift + pure click (no drag) on a collapsible + // block toggles its expanded state. Detect "pure + // click" by comparing anchor == cursor: Drag + // events are the only path that moves `cursor` + // off `anchor`, so any drag at all falls through + // to the copy path. Drag-select inside an + // expanded block still works because the + // anchor/cursor pair diverges as soon as the + // first Drag event fires. + // Two gestures for toggling a collapsible / + // input group: + // + // 1. Shift-click on the row body — works on + // terminals that forward Shift+mouse to + // the app. Many (iTerm2, GNOME Terminal, + // macOS Terminal.app) reserve Shift+click + // for their OWN text-selection override + // and swallow the event before the app + // ever sees it — those events never reach + // here. + // + // 2. Plain click on the marker column (0-1) + // of the row that carries the ▶/▼ glyph + // — works everywhere, no modifier fights, + // matches the "click the arrow" gesture + // file explorers and Vivado's own GUI + // use. Guarded by `same` (no drag) so + // starting a text selection near the left + // edge doesn't accidentally toggle. + let same = sel.anchor == sel.cursor; + let shift_toggle = + mouse.modifiers.contains(KeyModifiers::SHIFT) + && same + && self.toggle_collapsible_at(sel.anchor.0); + let marker_toggle = !shift_toggle + && same + && sel.anchor.1 < 2 + && self.toggle_collapsible_at(sel.anchor.0); + if shift_toggle || marker_toggle { + return; + } + self.copy_selection_to_clipboard(sel); + } + } + _ => {} + } + } + + /// Map a wrapped-row index (0-based, spans all of `scrollback`) + /// to the entry that occupies it, and if that entry is a + /// collapsible NONE block, toggle its expand state. Returns + /// `true` when a toggle happened so the caller can suppress the + /// fallthrough "copy empty selection" path — a Shift-click on a + /// diagnostic line should still do nothing (not clobber the + /// clipboard with an empty string, not act on the diagnostic + /// entry), so a `false` here means "not our gesture, keep + /// falling through." + /// + /// Walks entries left-to-right, summing wrapped-row counts until + /// we find the entry the row lives in. O(N) per click — cheap at + /// scrollback sizes we care about. + fn toggle_collapsible_at(&mut self, wrapped_row: usize) -> bool { + let width = self.scrollback_area.map(|a| a.width).unwrap_or(0); + // First pass: figure out which entry index owns + // `wrapped_row`. Row math mirrors what + // `ui::compute_visible_counts` does — hidden children of + // a collapsed group contribute 0 rows, so a shift-click + // near the top of scrollback lands on the right entry + // regardless of what's collapsed above it. + let mut cursor: usize = 0; + let mut hit: Option = None; + for (idx, entry) in self.scrollback.iter().enumerate() { + let hidden = entry + .parent_input_idx + .and_then(|p| self.scrollback.get(p)) + .map(|p| p.group_collapsed) + .unwrap_or(false); + let rows = if hidden { + 0 + } else { + crate::render::count_wrapped_rows(entry, width) as usize + }; + let end = cursor.saturating_add(rows); + if (cursor..end).contains(&wrapped_row) { + hit = Some(idx); + break; + } + cursor = end; + } + let Some(idx) = hit else { return false }; + let target = &mut self.scrollback[idx]; + // Toggle behaviors: + // * Input rows whose OWN body is elided + // (`collapse_state == Some(true)` — auto-collapsed + // multi-line command) — expand the body first. The + // user hit the marker because the visible chevron + // sits next to a partial preview + `(N lines hidden)` + // suffix, and their intent is "show me the rest of + // the command I just typed". Only toggle + // `group_collapsed` on subsequent clicks (body is + // fully visible). + // * Input rows with a fully-visible body — flip the + // whole group's child visibility via + // `group_collapsed`. + // * Non-Input collapsible blocks — flip the entry's + // own multi-line body via `collapse_state`. + if matches!(target.kind, ScrollbackKind::Input) { + if matches!(target.collapse_state, Some(true)) { + target.collapse_state = Some(false); + } else { + target.group_collapsed = !target.group_collapsed; + } + return true; + } + match target.collapse_state { + Some(expanded) => { + target.collapse_state = Some(!expanded); + true + } + None => false, + } + } + + /// Translate a screen cell `(col, row)` inside the scrollback + /// `area` into a `(row, col)` index into the post-wrap line list. + /// The row index is `effective_scroll + (row - area.y)` so the + /// caller doesn't have to know about scroll state. + /// + /// Anchors against `last_rendered_scroll` rather than + /// `scrollback_scroll`. While tail-follow is on the renderer + /// computes the pinned offset on the fly and never writes it + /// back to `scrollback_scroll` — using the stale field here + /// would map mouse clicks to the wrong buffer rows once any + /// real volume of output has scrolled the viewport. + fn cell_to_buffer(&self, col: u16, row: u16, area: Rect) -> (usize, usize) { + let local_row = row.saturating_sub(area.y) as usize; + let local_col = col.saturating_sub(area.x) as usize; + let buf_row = self.last_rendered_scroll as usize + local_row; + (buf_row, local_col) + } + + /// Build the same post-wrap line list the UI renders, extract + /// the cells inside `sel`, and write the resulting plain text to + /// the OS clipboard. Failure (no clipboard backend / Wayland + /// permissions denied / …) surfaces as a Notice line so the + /// user knows the copy didn't go through. + fn copy_selection_to_clipboard(&mut self, sel: Selection) { + let Some(area) = self.scrollback_area else { + return; + }; + // Skip children of collapsed input groups when building + // the flat list — same visibility rule + // `ui::compute_visible_counts` uses. Selection row + // indices are in VISIBLE-row space (that's what the + // renderer draws and what mouse cell → row math + // produces), so if this build path included hidden + // entries the row indices would map to the wrong lines + // and the clipboard would get chunks of hidden output + // instead of what the user selected. + let mut flat: Vec> = Vec::new(); + for entry in &self.scrollback { + let hidden = entry + .parent_input_idx + .and_then(|p| self.scrollback.get(p)) + .map(|p| p.group_collapsed) + .unwrap_or(false); + if hidden { + continue; + } + for line in crate::render::entry_lines(entry, area.width) { + flat.push(line); + } + } + let wrapped = crate::render::wrap_lines(flat, area.width); + let (start, end) = sel.ordered(); + if start == end { + return; // pure click, nothing to copy + } + let mut out = String::new(); + let last_row = end.0.min(wrapped.len().saturating_sub(1)); + for (row_idx, line) in wrapped + .iter() + .enumerate() + .skip(start.0) + .take(last_row + 1 - start.0) + { + let plain = crate::render::line_plain_text(line); + let chars: Vec = plain.chars().collect(); + let row_start = if row_idx == start.0 { start.1 } else { 0 }; + let row_end = if row_idx == end.0 { end.1 } else { chars.len() }; + let row_end = row_end.min(chars.len()); + let row_start = row_start.min(row_end); + for c in &chars[row_start..row_end] { + out.push(*c); + } + if row_idx < end.0 { + out.push('\n'); + } + } + if out.is_empty() { + return; + } + // Primary path: OSC 52. The terminal itself puts the text on + // the system clipboard — no DISPLAY / Wayland socket / + // pbcopy dependency, and it works through SSH. Most modern + // terminals support it (kitty, ghostty, iTerm2, Alacritty, + // Wezterm, recent xterm). Some require an opt-in + // (`set -g set-clipboard on` in tmux, `Allow programs to use + // clipboard` in iTerm2's General → Selection prefs). + // + // Secondary path: arboard. When a real clipboard daemon is + // reachable, this also syncs into the X11/Wayland clipboard + // so other GUI apps see the text. Failures here are silent + // because OSC 52 above is already authoritative — the + // X11-unreachable / Wayland-without-perms case used to + // surface as a noisy "clipboard copy failed" Notice. + send_osc52(&out); + let _ = arboard::Clipboard::new().and_then(|mut c| c.set_text(out)); + } + + async fn handle_terminal_event(&mut self, ev: Event) { + if let Event::Mouse(mouse) = ev { + self.handle_mouse_event(mouse); + return; + } + if let Event::Paste(data) = ev { + self.handle_paste(data); + return; + } + let Event::Key(key) = ev else { return }; + if !matches!(key.kind, KeyEventKind::Press | KeyEventKind::Repeat) { + return; + } + + if self.reverse_search.is_some() { + self.handle_reverse_search_key(key).await; + return; + } + + // Popup navigation / dismissal takes precedence over both + // app-level chords AND the catch-all editor handoff, so + // Up/Down/Enter/Esc go to the popup when one is open. + if self.popup.is_some() && self.handle_popup_key(key) { + return; + } + + match (key.code, key.modifiers) { + (KeyCode::Char('d'), KeyModifiers::CONTROL) => { + // Exit unconditionally. The original behavior required + // the input to be empty (readline convention), but + // for a REPL it's strictly an annoyance — you can't + // exit a misformed in-progress command without + // clearing it first. Ctrl-C is the right key to + // discard the current input, and we already bind it + // to that. + self.push(ScrollbackKind::Notice, "exit".to_string()); + self.exit = true; + } + (KeyCode::Char('c'), KeyModifiers::CONTROL) => { + // Two modes for Ctrl-C: + // + // - **Eval in flight** (`WorkerState::Running`): + // send SIGINT to the Vivado child, which its Tcl + // runtime traps into `interp cancel` — the + // current eval aborts and returns a "interrupted" + // error through the shim protocol, without + // killing the Vivado session. Full mechanism + + // design assumptions on + // `vw_vivado::VivadoBackend::interrupt`. No fall- + // through to the input clear; the input line is + // probably empty during an eval anyway, and + // preserving whatever the user typed lets them + // resubmit / edit after the cancel lands. + // + // - **No eval running**: clear the current input + // (reedline / readline convention). + if matches!(self.worker_state, WorkerState::Running) { + if let Some(interrupt) = self.interrupt.clone() { + // Called directly rather than round-tripped + // through the worker channel, which is blocked + // on the very eval being cancelled. A local + // session signals Vivado's process group; a + // remote one asks the instance to do the same. + // Either way the eval aborts and the + // interpreter — with everything loaded into it + // — survives. + interrupt(); + self.push( + ScrollbackKind::Notice, + "interrupt sent — Vivado will abort the \ + current eval and return" + .into(), + ); + } else { + self.push( + ScrollbackKind::Warning, + "no pid cached; can't interrupt eval — \ + restart the REPL to recover" + .into(), + ); + } + } else { + self.input = TextArea::default(); + self.history_cursor = None; + self.history_draft.clear(); + } + } + (KeyCode::F(2), _) => { + // Flip terminal mouse-capture mode. OFF (the default) + // lets the terminal handle text-selection drags + // natively; ON routes wheel events into the app for + // scrollback navigation, at the cost of text + // selection requiring Shift-drag / Option-drag. + self.toggle_mouse_capture(); + } + (KeyCode::Char('r'), KeyModifiers::CONTROL) => { + self.reverse_search = Some(ReverseSearch { + query: String::new(), + match_index: None, + match_text: String::new(), + }); + } + (KeyCode::Tab, _) => { + self.trigger_completion(); + } + (KeyCode::Char('h'), KeyModifiers::CONTROL) => { + self.popup = Some(crate::popup::PopupState::Help( + crate::popup::HelpPopup, + )); + } + (KeyCode::Char('y'), KeyModifiers::CONTROL) => { + // Hover under cursor. We use Ctrl-Y (not Ctrl-K, + // which is already scrollback-up) — picked because + // Ctrl-K is also commonly conflated with + // "kill-line" elsewhere. Y for "your symbol's docs." + self.trigger_hover(); + } + (KeyCode::Char('s'), KeyModifiers::CONTROL) => { + // Fuzzy symbol picker over the session + pending + + // in-flight input. Opens centered; Tab toggles to + // the libraries view. (Ctrl-S over Ctrl-T because + // the latter often gets eaten by terminal + // multiplexers, and "S" for "search" is the more + // discoverable mnemonic.) + self.trigger_symbol_search(); + } + (KeyCode::Char('f'), KeyModifiers::CONTROL) => { + // Fuzzy diagnostics finder. Snapshots the current + // scrollback's Error/Warning/Notice entries. Enter + // jumps the viewport to the chosen entry and drops + // a persistent left-gutter marker (Alt-C clears). + self.trigger_diagnostic_search(); + } + (KeyCode::Char('c'), KeyModifiers::ALT) => { + // Clear the diagnostics-finder jump marker. No-op + // when nothing is marked. Picked Alt-C over Ctrl-L + // (which many terminals eat for "clear screen") and + // Ctrl-K (already bound to scroll-up). + self.clear_marker(); + } + (KeyCode::Char('p'), KeyModifiers::CONTROL) => { + self.history_step(-1); + } + (KeyCode::Char('n'), KeyModifiers::CONTROL) => { + self.history_step(1); + } + // Scrollback nav. PageUp/PageDown for keyboards that + // have them; Ctrl-K (up) / Ctrl-J (down) for compact + // keyboards (Mac laptops, 60% boards) where PageUp + // doesn't exist physically. Vim-style direction + // mapping — `k` is up, `j` is down. Picked over + // Ctrl-↑/↓ because macOS intercepts those (Mission + // Control / app-switching). + // + // Direction: see `handle_mouse_event` — `k`/PageUp + // moves the viewport UP toward older content, which + // means decreasing the y-scroll offset. + // Vim-style scroll: Alt+K up, Alt+J down. Alt (not + // Ctrl) because Ctrl+J is claimed by Shift+Enter on + // legacy-encoding terminals (see the Ctrl+J-as- + // newline arm below), and single-modifier consistency + // beats splitting the pair across two modifiers. + (KeyCode::PageUp, _) | (KeyCode::Char('k'), KeyModifiers::ALT) => { + self.scroll_by(-5); + } + (KeyCode::PageDown, _) + | (KeyCode::Char('j'), KeyModifiers::ALT) => { + self.scroll_by(5); + } + // Snap to bottom + re-engage tail-follow. Use End + // (when available) or Ctrl-G as the compact-keyboard + // alternative. After scrolling up to inspect old + // output the user explicitly requests "back to live" + // here; we no longer auto-re-engage on every scroll + // (that auto-engage was firing spuriously due to a + // raw-vs-wrapped line-count mismatch, making scroll + // appear dead on large outputs). + (KeyCode::End, _) | (KeyCode::Char('g'), KeyModifiers::CONTROL) => { + self.scrollback_follow = true; + } + (KeyCode::Enter, KeyModifiers::NONE) => { + self.on_submit().await; + } + (KeyCode::Enter, m) if !m.is_empty() => { + // Enter with ANY modifier is a "keep typing" + // escape hatch — Ctrl+Enter and Alt+Enter both + // arrive here as `Enter + CTRL/ALT` under the + // kitty protocol. + self.insert_newline_preserving_indent(); + } + // Shift+Enter on legacy-encoding terminals (macOS + // Terminal.app, GNOME Terminal without kitty + // protocol, and many tmux configurations) arrives + // as Ctrl+J — because ASCII 0x0A (linefeed) IS what + // the shifted-Enter physically produces, and raw + // mode disables the `\n` → Enter auto-translation. + // Bind it to newline directly so the user gets the + // expected behavior everywhere. + (KeyCode::Char('j'), m) if m.contains(KeyModifiers::CONTROL) => { + self.insert_newline_preserving_indent(); + } + _ => { + // Forward everything else to the text editor. + // Once the user starts editing, drop the history + // cursor so subsequent Ctrl-P starts at "newest" + // again — readline behavior: an edited recall is + // a new entry, not a continued walk. + self.history_cursor = None; + let input: Input = key.into(); + let _consumed = self.input.input(input); + // Auto-trigger signature help on every buffer- + // mutating keystroke. Costs one parse of the (small) + // in-flight input + a HashMap lookup; bails when the + // cursor isn't in a known call. Respects existing + // Completion / Help popups (won't displace them). + self.refresh_signature_help(); + } + } + } + + async fn handle_reverse_search_key(&mut self, key: KeyEvent) { + let Some(rs) = self.reverse_search.as_mut() else { + return; + }; + match (key.code, key.modifiers) { + (KeyCode::Esc, _) => { + self.reverse_search = None; + } + (KeyCode::Enter, _) => { + let text = std::mem::take(&mut rs.match_text); + self.reverse_search = None; + if !text.is_empty() { + self.set_input_to(&text); + } + } + (KeyCode::Char('r'), KeyModifiers::CONTROL) => { + let start = rs.match_index; + if let Some((idx, hit)) = + self.history.search_back(&rs.query, start) + { + rs.match_index = Some(idx); + rs.match_text = hit.to_string(); + } + } + (KeyCode::Backspace, _) => { + rs.query.pop(); + self.rerun_reverse_search(); + } + (KeyCode::Char(c), m) + if !m.contains(KeyModifiers::CONTROL) + && !m.contains(KeyModifiers::ALT) => + { + rs.query.push(c); + self.rerun_reverse_search(); + } + _ => {} + } + } + + fn rerun_reverse_search(&mut self) { + let Some(rs) = self.reverse_search.as_mut() else { + return; + }; + match self.history.search_back(&rs.query, None) { + Some((idx, hit)) => { + rs.match_index = Some(idx); + rs.match_text = hit.to_string(); + } + None => { + rs.match_index = None; + rs.match_text.clear(); + } + } + } + + fn set_input_to(&mut self, text: &str) { + let mut ta = TextArea::default(); + ta.set_cursor_line_style(ratatui::style::Style::default()); + for (i, line) in text.split('\n').enumerate() { + if i > 0 { + ta.insert_newline(); + } + ta.insert_str(line); + } + self.input = ta; + } + + /// Insert bracketed-paste content into the input area, one + /// line at a time. Embedded newlines become real newlines in + /// the buffer, NOT Enter events — that's the whole reason + /// bracketed paste exists: without it, a pasted multi-line + /// block delivers each `\n` as a submit trigger and every + /// intermediate line runs as its own command. + /// + /// Also drops the history walk (any typed edit — paste + /// included — starts a fresh history search on the next + /// Ctrl-P) and turns off scrollback follow so the user can + /// still scroll up during the paste render. + /// Insert a newline and re-emit the CURRENT line's leading + /// whitespace on the new line. Matches how editors (and + /// Claude Code's TUI) behave on Shift+Enter — hitting it + /// inside a `-flag`-continued command keeps you at the same + /// column so `-foo\n -bar` becomes `-foo\n -bar\n |cursor` + /// instead of `-foo\n -bar\n|cursor`. Only whitespace is + /// copied (spaces + tabs) — never the actual line content. + /// + /// Applies at the CURRENT cursor row, not the top row: if the + /// user is mid-line and hits Ctrl+J, they see indent-copied + /// behavior on the CURRENT line's indent, matching every + /// other editor's rule. + fn insert_newline_preserving_indent(&mut self) { + let (row, _) = self.input.cursor(); + let indent: String = self.input.lines()[row] + .chars() + .take_while(|c| *c == ' ' || *c == '\t') + .collect(); + self.input.insert_newline(); + for ch in indent.chars() { + self.input.insert_char(ch); + } + } + + fn handle_paste(&mut self, data: String) { + self.history_cursor = None; + let mut first = true; + for line in data.split('\n') { + if !first { + self.input.insert_newline(); + } + first = false; + // Strip carriage returns some terminals prepend (CRLF + // sources on Windows / some remote sessions). + for ch in line.chars().filter(|&c| c != '\r') { + self.input.insert_char(ch); + } + } + } + + async fn on_submit(&mut self) { + let text = self.current_input_text(); + let trimmed = text.trim(); + if trimmed.is_empty() { + return; + } + if !is_buffer_complete( + &text, + &self.session.read().unwrap().signature_table(), + ) { + self.input.insert_newline(); + return; + } + + self.history.append(&text); + self.push(ScrollbackKind::Input, text.clone()); + // User-typed inputs stay expanded by default — auto-collapse + // is a "wall of script output" convenience, not what the + // user wants right after they just typed a command and are + // waiting to see its output. Both dimensions apply: + // * `group_collapsed = false` keeps subsequent output + // visible under this input header. + // * `collapse_state = Some(false)` when the input is + // multi-line, so the input body itself stays fully + // visible instead of eliding to first-line + + // "(N lines hidden)". Preserves toggleability via the + // marker click — `Some(false)` still marks the entry + // as a collapsible block, just one that starts + // expanded. + if let Some(entry) = self.scrollback.last_mut() { + entry.group_collapsed = false; + if entry.collapse_state.is_some() { + entry.collapse_state = Some(false); + } + } + + if let Some(cmd) = trimmed.strip_prefix(':') { + self.run_meta_command(cmd).await; + } else { + self.dispatch_eval(text).await; + } + + self.input = TextArea::default(); + // Reset history walk: the next Ctrl-P should start from the + // newest entry, not pick up where the previous walk left + // off across submits. + self.history_cursor = None; + self.history_draft.clear(); + // Re-engage tail-follow: every new submit shows the input + // echo + its output at the bottom of the viewport, even + // if the user had scrolled up to inspect earlier results. + // The actual pin happens in the renderer next frame. + self.scrollback_follow = true; + } + + fn resolve_stack_frames(&self, msg: &str) -> String { + let session_guard = self.session.read().unwrap(); + resolve_stack_frames( + msg, + &session_guard, + self.pending_batch.as_ref(), + self.input_file_for_resolve(), + ) + } + + /// Append a synthetic ` at :` frame to streamed + /// warnings/errors that already arrived without a stack. Vivado + /// emits some message classes (notably `[IP_Flow 19-7090]` + /// "Invalid parameter" warnings during `set_property`) from + /// C++ code paths that bypass the Tcl-level `send_msg_id` + /// override — so the shim never sees them and can't attach a + /// real Tcl call stack. The fallback is "which user command + /// was the worker chewing on when this byte stream arrived," + /// which `pending_origins[pending_eval_index]` gives us. Won't + /// add a frame if the message already has one (the + /// `\n at …` shape from `attach_stack_if_message`) or if it + /// isn't a warning/error severity. + fn tag_streamed_message( + &self, + kind: ScrollbackKind, + msg: String, + ) -> String { + if !matches!(kind, ScrollbackKind::Warning | ScrollbackKind::Error) { + return msg; + } + if msg.contains("\n at ") { + return msg; + } + let Some(origin) = self.pending_origins.get(self.pending_eval_index) + else { + return msg; + }; + let path = match origin.file.as_deref() { + Some(p) => display_path(p), + None => match self.input_file_for_resolve() { + Some(p) => display_path(p), + None => "".into(), + }, + }; + format!("{msg}\n at {path}:{}", origin.line) + } + + /// File to substitute for `` frames in stack traces. + /// Comes from `--load ` for the auto-loaded program — its + /// content was copied verbatim into the lowering scratch, so + /// scratch line N corresponds to load-file line N. For + /// REPL-typed input there's no source file, so callers leave + /// `` as-is. + fn input_file_for_resolve(&self) -> Option<&std::path::Path> { + self.opts.initial_load.as_ref().map(|p| p.as_std_path()) + } + + async fn dispatch_eval(&mut self, text: String) { + self.dispatch_eval_with_echo(text, false).await; + } + + /// Same as [`dispatch_eval`] but echoes each lowered top-level + /// statement as an Input entry first. Used by the `--load` + /// auto-run path so the user can see *which* commands ran when + /// reading the trace, the same way manual REPL input shows up + /// as `› ` for each submit. + async fn dispatch_eval_with_echo(&mut self, text: String, echo: bool) { + if matches!(self.worker_state, WorkerState::Down) { + self.push( + ScrollbackKind::Error, + "vivado worker is down — try :restart".into(), + ); + return; + } + if matches!(self.worker_state, WorkerState::Starting) { + self.push( + ScrollbackKind::Notice, + "queued — vivado still starting".into(), + ); + } + + // Lower htcl → Tcl on a blocking thread so the event + // loop can render the input echo + tick the per-input + // timer during the (potentially minute-scale) parse + + // validate + lower work. `prepare` is fully synchronous + // CPU + I/O; spawn_blocking is the right primitive. + // + // The main task returns immediately after spawning — + // `handle_worker_event` picks up `WorkerEvent::PrepareDone` + // when the background thread finishes, and continues the + // dispatch pipeline from there. + // Show a "preparing…" sentinel only when the input will + // actually pull new files through the loader — i.e., the + // top-level parse of `text` contains a `src` command. + // A one-liner like `bd::clobber -name txr0` doesn't touch + // disk and prepare finishes in low double-digit ms; a + // notice per submit for that case is just noise. + // + // The parse itself is cheap (input is usually a single + // line); prepare will re-parse the flat post-load source + // internally either way. False negatives are impossible + // — `src` is the only mechanism that adds files — and + // false positives are bounded to "user typed `src` + // without triggering big work" (already-preloaded + // target), where a brief notice does no harm. + let will_load = { + let parsed = vw_htcl::parse(&text); + parsed.document.stmts.iter().any(|s| { + matches!( + s, + vw_htcl::Stmt::Command(c) + if matches!(c.kind, vw_htcl::CommandKind::Src(_)) + ) + }) + }; + if will_load { + self.push( + ScrollbackKind::Notice, + "preparing… (parsing + validating imports)".into(), + ); + } + + let cwd = std::env::current_dir().unwrap_or_else(|_| ".".into()); + let session = std::sync::Arc::clone(&self.session); + let event_tx = self.event_tx.clone(); + let text_moved = text; + tokio::task::spawn_blocking(move || { + // Catch panics on the blocking thread so a bug in + // prepare surfaces as an ERROR row rather than a + // silent stall. Without this the JoinHandle we drop + // absorbs the panic and the UI sits waiting forever + // for a PrepareDone that will never come. + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let session_guard = session.read().unwrap(); + crate::lower::prepare(&text_moved, &cwd, &session_guard) + })); + match result { + Ok(r) => { + let _ = event_tx.send(WorkerEvent::PrepareDone { + text: text_moved, + echo, + result: r, + }); + } + Err(payload) => { + // Turn the panic message into a LowerError so + // the existing PrepareDone/handle_prepare_done + // path renders it as an ERROR row. + let msg = if let Some(s) = payload.downcast_ref::<&str>() { + (*s).to_string() + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + "unknown panic in prepare".to_string() + }; + let _ = event_tx.send(WorkerEvent::PrepareDone { + text: text_moved, + echo, + result: Err(crate::lower::LowerError::Parse(format!( + "prepare panicked: {msg}", + ))), + }); + } + } + }); + } + + /// Continuation of `dispatch_eval_with_echo` — receives the + /// completed `Prepared` (or error) from the background prepare + /// task and finishes the dispatch pipeline: surfaces warnings, + /// commits pure-`src` imports directly, otherwise builds the + /// per-input timer boundaries and ships commands to the worker. + async fn handle_prepare_done( + &mut self, + _text: String, + echo: bool, + result: Result, + ) { + let lowered = match result { + Ok(l) => l, + Err(e) => { + self.push(ScrollbackKind::Error, format!("ERROR: {e}")); + // Prepare failed — no eval will run, so the Input + // entry's timer would otherwise tick forever. Freeze + // it at the "prepare failed" wall time. Same freeze + // path the empty-batch case (line ~2234) uses. + self.mark_inputs_completed(); + return; + } + }; + + // Surface any pre-flight warnings *before* shipping. If the + // eval then fails, the user already has the context they + // need to interpret the Vivado error. + for w in &lowered.warnings { + let where_ = + render_origin_path(w.origin.file.as_deref(), w.origin.line); + self.push( + ScrollbackKind::Warning, + format!("warning: {where_}: {}", w.message), + ); + } + if lowered.commands.is_empty() { + // Pure `src` import or comments-only input. Commit the + // parsed batch to the session anyway so future + // analyzer queries see the imported procs. + self.session.write().unwrap().commit(lowered.batch); + self.sync_preload_from_session(); + self.push(ScrollbackKind::Notice, "(no Tcl to evaluate)".into()); + // The per-input timer was ticking through prepare; + // freeze it now — this batch has nothing to eval. + self.mark_inputs_completed(); + return; + } + + // Build per-Input-entry timer boundaries. Empty when not + // in echo mode (the non-echo single-Input case uses the + // existing `mark_inputs_completed` end-of-batch path). + // + // Echo model: strictly linear. The batch's FIRST + // statement is echoed to scrollback now; every subsequent + // statement is registered as a boundary with + // `scrollback_idx: None` and echoed lazily by + // `advance_input_timers` when the prior boundary closes. + // That way a `:load prime.htcl` run reads like an + // interactive session — each command appears, its output + // and messages follow, then the next command appears. + let mut input_boundaries: Vec = Vec::new(); + if echo { + for origin in &lowered.entry_top_level { + input_boundaries.push(InputBoundary { + scrollback_idx: None, + snippet: origin.snippet.clone(), + last_command_idx: None, // filled below + completed: false, + }); + } + // For each entry-top-level Origin, find the LAST + // lowered command whose ultimate entry-file line + // matches it. A command's "entry line" is the line + // in the entry file it came from: directly when + // `origin.via` is empty (the command lives in the + // entry), or the bottom of the `via` chain (which + // lower.rs documents as "the last frame is the + // entry file / user input"). + for (cmd_idx, cmd) in lowered.commands.iter().enumerate() { + let entry_line = match cmd.origin.via.last() { + Some(f) => f.line, + None => cmd.origin.line, + }; + // Find which entry_top_level Origin this matches + // (linear scan — at most a handful of top-level + // statements per batch). + for (j, top) in lowered.entry_top_level.iter().enumerate() { + if top.line == entry_line { + if let Some(b) = input_boundaries.get_mut(j) { + b.last_command_idx = Some(cmd_idx); + } + break; + } + } + } + } + self.pending_input_boundaries = input_boundaries; + if echo { + // Push the first non-empty boundary's echo NOW so the + // user sees `› ` before the batch + // starts producing output. `activate_next_boundary` + // also handles the edge case of a leading empty + // boundary (a `src` whose target file lowered to zero + // commands): it echoes, freezes, and cascades. + self.activate_next_boundary(0); + } + + // Snapshot per-command origins + types for the stream- + // tagging + result-display paths. EvalBatch consumes + // `lowered.commands` below, so we grab both first. + self.pending_origins = + lowered.commands.iter().map(|c| c.origin.clone()).collect(); + self.pending_return_types = lowered + .commands + .iter() + .map(|c| c.expected_return_type.clone()) + .collect(); + self.pending_is_set_binding = + lowered.commands.iter().map(|c| c.is_set_binding).collect(); + self.pending_eval_index = 0; + + // Seed the shared preload map with this batch's file + // list BEFORE dispatching. Commands ship in document + // order, so any RPC that fires from a command MID-batch + // (currently only `compile_htcl_module` from + // `vw::configure_ip`) can trust that every proc from + // every file preceding it in the same batch is already + // installed in Vivado. Without this, the batch's own + // `src @vw` recursion doesn't reach the preload until + // AFTER the whole batch completes — which means the + // first `vw::configure_ip` call re-parses + re-lowers + // + re-ships @vw + @vivado-cmd (~10MB of Tcl, + // multiple minutes). Preload-then-dispatch turns that + // into "just workspace-local files" and shrinks the + // compile output by ~100×. + // + // Safety: the invariant on `SharedPreload` is "files + // whose Tcl has been eval'd by Vivado". Populating from + // this batch's file list before eval TECHNICALLY breaks + // the letter of that rule for the window between + // dispatch and completion — but for the specific caller + // that uses the preload (`compile_htcl_module`, invoked + // from mid-batch procs), the OR pending commands run + // strictly BEFORE the invocation, so the corresponding + // procs ARE installed by the time the RPC fires. + if let Ok(mut g) = self.preload.write() { + for f in &lowered.batch.program.files { + if let Some(t) = f.mtime { + g.insert(f.path.clone(), t); + } + } + } + // Commit to the session only after every command in the + // batch succeeds (see `handle_worker_event`); a failure + // mid-batch shouldn't pollute the analyzer's view. + let _ = self + .worker_tx + .send(WorkerCmd::EvalBatch(lowered.commands)) + .await; + self.pending_batch = Some(lowered.batch); + self.worker_state = WorkerState::Running; + } + + async fn run_meta_command(&mut self, cmd: &str) { + // Note for completion: keep [`META_COMMANDS`] in sync with + // the arms of this `match`. + let mut parts = cmd.splitn(2, char::is_whitespace); + let name = parts.next().unwrap_or(""); + let arg = parts.next().unwrap_or("").trim(); + match name { + "quit" | "q" | "exit" => { + self.exit = true; + } + "restart" => { + self.push( + ScrollbackKind::Notice, + "restart not yet implemented (stubbed for v1)".into(), + ); + } + "libs" => { + // List every library the session knows about + its + // symbol count, sorted by descending count. Built + // from the same SymbolIndex the Ctrl-T picker uses, + // so the totals stay consistent across surfaces. + let parsed_input = + vw_htcl::parser::parse(&self.current_input_text()); + let session = std::sync::Arc::clone(&self.session); + let session_guard = session.read().unwrap(); + let index = crate::symbol_index::SymbolIndex::build( + &session_guard, + self.pending_batch.as_ref(), + Some(&parsed_input.document), + ); + let libs = index.libraries(); + if libs.is_empty() { + self.push( + ScrollbackKind::Notice, + "no libraries loaded".to_string(), + ); + } else { + // Column widths: count gets 5 cells, library + // name takes the max of its actual lengths. + let max_name = libs + .iter() + .map(|l| l.library.display().chars().count()) + .max() + .unwrap_or(8); + let mut out = String::new(); + out.push_str(&format!( + "{:>5} {: { + "".to_string() + } + crate::symbol_index::LibraryRef::Import { + path, + .. + } => path.display().to_string(), + }; + out.push_str(&format!( + "{:>5} {: { + if arg.is_empty() { + self.push( + ScrollbackKind::Error, + ":load needs a path".into(), + ); + return; + } + match std::fs::read_to_string(arg) { + Ok(content) => { + self.push( + ScrollbackKind::Notice, + format!("loading {arg}"), + ); + self.dispatch_eval(content).await; + } + Err(e) => { + self.push( + ScrollbackKind::Error, + format!("could not read {arg}: {e}"), + ); + } + } + } + other => { + self.push( + ScrollbackKind::Error, + format!("unknown meta-command :{other}"), + ); + } + } + } + + async fn handle_worker_event(&mut self, event: WorkerEvent) { + match event { + WorkerEvent::Started { interrupt } => { + self.worker_state = WorkerState::Ready; + self.interrupt = Some(interrupt); + self.push(ScrollbackKind::Notice, "vivado ready".into()); + // `initial_source` (e.g. from + // `--from-*-checkpoint`) is a literal htcl snippet + // resolved by the CLI. It takes precedence over + // `initial_load` because it represents an + // explicit user request to skip `design.htcl` and + // pick up mid-flow from a persisted checkpoint. + if let Some(source) = self.opts.initial_source.clone() { + self.push( + ScrollbackKind::Notice, + "auto-dispatching --from-*-checkpoint snippet".into(), + ); + self.dispatch_eval_with_echo(source, true).await; + } else if let Some(path) = self.opts.initial_load.clone() { + match std::fs::read_to_string(path.as_std_path()) { + Ok(content) => { + self.push( + ScrollbackKind::Notice, + format!("auto-loading {path}"), + ); + self.dispatch_eval_with_echo(content, true).await; + } + Err(e) => { + self.push( + ScrollbackKind::Error, + format!("could not read {path}: {e}"), + ); + } + } + } + } + WorkerEvent::StartFailed(e) => { + self.worker_state = WorkerState::Down; + self.push( + ScrollbackKind::Error, + format!("vivado failed to start: {e}"), + ); + } + WorkerEvent::PrepareDone { text, echo, result } => { + self.handle_prepare_done(text, echo, result).await; + } + WorkerEvent::Stream { kind, data } => { + // Feed the chunk into the block accumulator: NONE + // blocks (Vivado tables / banners / license chatter) + // group into one collapsible scrollback entry per + // run of Stdout chunks, while Diagnostic chunks + // (INFO / WARNING / CRITICAL / ERROR) flush any + // pending NONE and emit themselves at full fidelity. + for block in self.block_acc.push(kind, &data) { + match block { + vw_vivado::Block::None { lines } => { + self.push_none_block(lines); + } + vw_vivado::Block::Diagnostic { severity, lines } => { + let is_critical = matches!( + severity, + vw_vivado::Severity::CriticalWarning + ); + let scrollback_kind = match severity { + vw_vivado::Severity::Info => { + ScrollbackKind::Notice + } + vw_vivado::Severity::Warning => { + ScrollbackKind::Warning + } + // Critical warnings render with the + // same red ✗ treatment as ERROR in + // scrollback — the block classifier + // keeps them semantically distinct + // for log-level filtering, but the + // visual severity is the same. + // `is_critical` above carries the + // discriminator through to the + // scrollback entry for the finder. + vw_vivado::Severity::CriticalWarning + | vw_vivado::Severity::Error => { + ScrollbackKind::Error + } + // Segmenter guarantees Diagnostic + // blocks are never Severity::None, + // but the arm has to exist for the + // match to be exhaustive. + vw_vivado::Severity::None => { + ScrollbackKind::Stdout + } + }; + let joined = lines.join("\n"); + let resolved = self.resolve_stack_frames(&joined); + let tagged = self.tag_streamed_message( + scrollback_kind, + resolved, + ); + let (body, stack) = split_body_and_stack(&tagged); + self.push_diag(scrollback_kind, body, is_critical); + if let Some(stack) = stack { + self.push_stack_trace(stack); + } + } + } + } + } + WorkerEvent::EvalDone { + origin, + result, + last_in_batch, + } => { + // Flush any pending NONE-block content the segmenter + // has been holding. Pure `puts` output arrives as + // Stdout chunks (Severity::None) and never emits from + // the accumulator on its own — the accumulator only + // flushes when a *classified* chunk (INFO/WARNING/ + // ERROR) arrives after it. Without this flush, plain + // `puts "muffins"` output sits invisible in + // `pending_none` until the next diagnostic — which + // for a quiet REPL never comes. Flushing at + // EvalDone is the right boundary: every eval's own + // output should surface before the next input echo. + for block in self.block_acc.flush() { + match block { + vw_vivado::Block::None { lines } => { + self.push_none_block(lines); + } + vw_vivado::Block::Diagnostic { severity, lines } => { + // Diagnostic-in-flush is unexpected — + // the accumulator emits diagnostics + // immediately on push, never holds them + // pending. Route through the same + // scrollback-kind mapping as the + // Stream-event path so future accumulator + // changes don't silently lose messages. + let is_critical = matches!( + severity, + vw_vivado::Severity::CriticalWarning + ); + let scrollback_kind = match severity { + vw_vivado::Severity::Info => { + ScrollbackKind::Notice + } + vw_vivado::Severity::Warning => { + ScrollbackKind::Warning + } + vw_vivado::Severity::CriticalWarning + | vw_vivado::Severity::Error => { + ScrollbackKind::Error + } + vw_vivado::Severity::None => { + ScrollbackKind::Stdout + } + }; + let joined = lines.join("\n"); + let resolved = self.resolve_stack_frames(&joined); + let tagged = self.tag_streamed_message( + scrollback_kind, + resolved, + ); + let (body, stack) = split_body_and_stack(&tagged); + self.push_diag(scrollback_kind, body, is_critical); + if let Some(stack) = stack { + self.push_stack_trace(stack); + } + } + } + } + // Grab the return type + set-binding flag for + // THIS command (the one that just finished) + // before we advance the index and possibly clear + // the buffer. + let finished_is_set_binding = self + .pending_is_set_binding + .get(self.pending_eval_index) + .copied() + .unwrap_or(false); + let finished_return_type = self + .pending_return_types + .get(self.pending_eval_index) + .cloned() + .flatten(); + // Capture the just-finished command's eval-index + // before we advance — used to freeze any Input + // entry whose last-command boundary matches it. + let just_finished_idx = self.pending_eval_index; + // Advance past the command that just finished — the + // stream-tagging path uses `pending_origins[index]` + // to label warnings emitted by the *currently* + // executing command, so the index should always + // point at "in-flight," not "just done." + self.pending_eval_index = + self.pending_eval_index.saturating_add(1); + // Per-statement timer freezing: if any echoed + // Input entry's `last_command_idx` matches the + // just-finished command, stamp its + // `completed_at`. On success we cascade — activate + // (echo + stamp `started_at` for) the next + // uncompleted boundary. On failure we do NOT + // cascade: the batch aborts here, and echoing the + // NEXT statement's `› …` line before the error + // trace we're about to render would make the trace + // look like it belonged to that statement. The + // ordering guarantee we're preserving is: an + // eval's output — including its error trace when + // the eval fails — appears between its own echo + // and the next one, if any next one appears at all. + match &result { + Ok(_) => self.advance_input_timers(just_finished_idx), + Err(_) => { + self.freeze_input_boundary(just_finished_idx); + } + } + if last_in_batch { + self.pending_origins.clear(); + self.pending_return_types.clear(); + self.pending_input_boundaries.clear(); + self.pending_eval_index = 0; + } + match result { + Ok(out) => { + // Drop the per-statement chatter — only the + // last item's value lands in scrollback so a + // `src @vivado-cmd` that runs 851 wrappers + // doesn't drown the user in "ok" lines. The + // intermediate procs etc. are silent unless + // they `puts` something (already streamed). + if last_in_batch { + if !out.stdout.is_empty() { + self.push( + ScrollbackKind::Stdout, + out.stdout + .trim_end_matches('\n') + .to_string(), + ); + } + // Result-rendering policy: + // - `unit`-typed expressions push nothing + // (the value is meaningless by design). + // - Other typed expressions push verbatim + // — the wrapped Tcl already ran the + // type's `repr` proc, so `out.value` + // is the formatted display string. + // - Untyped expressions we can't repr + // through the type's proc: skip the + // push. `out.value` in this case + // is the raw Tcl representation + // (`{Scalar x}` tagged-list form + // for our nested Properties trees) + // — displaying that leaks the + // internal encoding rather than + // the compiler-emitted + // `Variant(payload)` repr shape. + // If a caller wants to inspect an + // untyped value, `puts` is the + // explicit form. + // - `set VAR ` is a binding, + // not a display. The value the + // user asked to name is now bound; + // showing it isn't part of what + // they wrote. Same suppression + // applies for consistency with + // the "no unrepr'd values leak" + // rule above. + let suppress_unit = matches!( + finished_return_type.as_ref(), + Some(vw_htcl::TypeExpr::Named { name, .. }) + if name == "unit" + ); + let suppress_untyped = + finished_return_type.is_none(); + let suppress = suppress_unit + || suppress_untyped + || finished_is_set_binding; + if !suppress && !out.value.is_empty() { + // finished_return_type is Some + // here (untyped is suppressed + // above), so the wrap_with_repr + // path has already rendered + // through the type's `repr` + // proc. Push verbatim. + let text = out.value.clone(); + self.push(ScrollbackKind::Result, text); + } + if let Some(batch) = self.pending_batch.take() { + self.session.write().unwrap().commit(batch); + self.sync_preload_from_session(); + } + self.worker_state = WorkerState::Ready; + // Freeze per-input timers at their + // final duration now that the batch + // has finished evaluating. + self.mark_inputs_completed(); + } + } + Err(err) => { + self.worker_state = WorkerState::Ready; + // Hold the pending batch for the renderer + // — drill-down lookups need its proc map. + // Cleared below once the trace is emitted. + render_eval_error(self, &origin, err); + // Commit the parsed batch to the session + // even though the eval failed. The batch's + // `Document` carries every proc, type, and + // enum decl that the parser saw — including + // whatever `src @vivado-cmd`, `src project`, + // `src ip/cips`, etc. brought in *before* + // the failing user command ran. Tab + // completion, hover, and signature help all + // query session-wide symbols, and dropping + // the batch strands them with an empty + // symbol table until the next successful + // eval. The runtime state in Vivado may be + // partial or wrong; that's a separate + // concern from what the analyzer sees. + if let Some(batch) = self.pending_batch.take() { + self.session.write().unwrap().commit(batch); + self.sync_preload_from_session(); + } + // Failed evals also freeze their per-input + // timer — otherwise the live counter would + // tick forever on an error result. + self.mark_inputs_completed(); + } + } + } + } + } + + pub(crate) fn push(&mut self, kind: ScrollbackKind, text: String) { + // O(1). The tail-follow pin happens in the renderer (which + // already knows the wrapped-row total for free), not here — + // doing it per-push was O(N) per call, making a long burst + // of Vivado stream chunks O(N²) and freezing the REPL for + // minutes during `src @vivado-cmd` style fan-outs. + // + // Input entries get a start timestamp so the renderer can + // show a per-input timer (live while running, frozen on + // batch completion). Other kinds leave timing unset. + let started_at = if matches!(kind, ScrollbackKind::Input) { + Some(std::time::Instant::now()) + } else { + None + }; + // Expand tabs to spaces at push time so BOTH `entry_lines_ + // windowed` (which builds rendered spans) and `count_wrapped_ + // rows` (which does the outer viewport math) see the same + // character content. If a `\t` survives into the render + // pipeline, ratatui's Buffer writes a `\t` cell that + // iTerm2 (and any real terminal) interprets as "move + // cursor to next tab stop" — a control code, not a + // printable glyph. Cells between the cursor's start + // column and the tab stop stay whatever they were in the + // previous frame, producing the `728_` / `FO:` / `.v:` + // leftover fragments Vivado's parameter-dump output was + // showing. Four spaces per tab matches typical editor + // defaults and keeps the expansion fixed-count so + // downstream width math (`chars().count()`) stays honest. + let text = if text.contains('\t') { + text.replace('\t', " ") + } else { + text + }; + // Uniform Mathematica-notebook-style collapsibility: every + // multi-line entry is toggleable (Shift+click), and + // anything larger than COLLAPSE_AUTO_THRESHOLD lines starts + // collapsed so a wall of text doesn't dominate the + // scrollback. Single-line entries get `None` — a placeholder + // for something that fits in one row is worse UX than just + // showing the row itself. + let collapse_state = compute_collapse_state(&text, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind, + text, + started_at, + completed_at: None, + collapse_state, + is_critical_warning: false, + parent_input_idx: if matches!(kind, ScrollbackKind::Input) { + None + } else { + self.current_input_idx + }, + group_collapsed: matches!(kind, ScrollbackKind::Input), + error_child_count: 0, + warning_child_count: 0, + }); + if matches!(kind, ScrollbackKind::Input) { + self.current_input_idx = Some(self.scrollback.len() - 1); + } + // Bump the parent input's severity tally so the + // collapsed header can render ✗ / ⚠ badges without + // expanding. Only the two "actionable" kinds count — + // Notices, Stdout, Result, Chatter don't warrant a + // header badge. + match kind { + ScrollbackKind::Error => { + if let Some(pidx) = self.current_input_idx { + if let Some(p) = self.scrollback.get_mut(pidx) { + p.error_child_count = + p.error_child_count.saturating_add(1); + } + } + } + ScrollbackKind::Warning => { + if let Some(pidx) = self.current_input_idx { + if let Some(p) = self.scrollback.get_mut(pidx) { + p.warning_child_count = + p.warning_child_count.saturating_add(1); + } + } + } + _ => {} + } + } + + /// Push a diagnostic that came from the Vivado stream, tagging + /// it as a CRITICAL WARNING when the source severity was + /// `Severity::CriticalWarning`. CW entries share + /// [`ScrollbackKind::Error`]'s red gutter but carry the + /// [`ScrollbackEntry::is_critical_warning`] flag so the + /// diagnostics-finder popup can offer a `Critical` filter + /// checkbox that surfaces just them. + pub(crate) fn push_diag( + &mut self, + kind: ScrollbackKind, + text: String, + is_critical_warning: bool, + ) { + // Split the tagged first line off. For any diagnostic — + // ERROR / CRITICAL WARNING / WARNING — the leading + // `: [] ` line MUST stay + // full-brightness and never dim-collapse, or a real + // problem in the middle of dozens of INFO lines + // disappears. Trailing content (Resolution hints, + // continuations) is fine to auto-collapse — that's what + // `split_leading_diagnostic` gives us. Notice / Stdout + // kinds don't tag their leading line, so the split just + // returns `(text, None)` and the entry pushes as before. + let (leading, trailing) = split_leading_diagnostic(&text); + let leading = if leading.contains('\t') { + leading.replace('\t', " ") + } else { + leading + }; + // The leading line pushes as its own entry with + // `collapse_state = None` (via the <2-lines branch of + // `compute_collapse_state`) so it renders at full + // brightness. Only CW entries carry the flag — this is + // what feeds the diagnostics-finder's `Critical` filter. + let leading_collapse = + compute_collapse_state(&leading, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind, + text: leading, + started_at: None, + completed_at: None, + collapse_state: leading_collapse, + is_critical_warning, + // Diagnostics always belong to the currently + // executing input group. `kind` here is never + // `Input` — the classifier routes Input echoes + // through the plain `push` path. + parent_input_idx: self.current_input_idx, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + }); + // Bump the parent input's severity tally. push_diag is + // the only path that flags is_critical_warning, so the + // Error / CW check has to live here alongside the + // `push`-side check (the two deliberately don't share + // code — push_diag has its own leading/trailing split). + // CRITICAL WARNINGs count as errors — same red gutter, + // same ✗ badge — matching how the diagnostics finder + // buckets them. + match kind { + ScrollbackKind::Error => { + if let Some(pidx) = self.current_input_idx { + if let Some(p) = self.scrollback.get_mut(pidx) { + p.error_child_count = + p.error_child_count.saturating_add(1); + } + } + } + ScrollbackKind::Warning => { + if let Some(pidx) = self.current_input_idx { + if let Some(p) = self.scrollback.get_mut(pidx) { + p.warning_child_count = + p.warning_child_count.saturating_add(1); + } + } + } + _ => {} + } + if is_critical_warning && !matches!(kind, ScrollbackKind::Error) { + // Belt-and-suspenders: CriticalWarning severity + // always classifies to `Error` kind at the stream + // level, but keep the flag path independent so a + // future rerouting doesn't silently drop the badge. + if let Some(pidx) = self.current_input_idx { + if let Some(p) = self.scrollback.get_mut(pidx) { + p.error_child_count = p.error_child_count.saturating_add(1); + } + } + } + if let Some(trailing) = trailing { + let trailing = if trailing.contains('\t') { + trailing.replace('\t', " ") + } else { + trailing + }; + let collapse_state = + compute_collapse_state(&trailing, self.collapse_mode); + self.scrollback.push(ScrollbackEntry { + kind: ScrollbackKind::Chatter, + text: trailing, + started_at: None, + completed_at: None, + collapse_state, + is_critical_warning: false, + // Trailing content attaches to the diagnostic + // just pushed above, which is a child of the + // current input group — so this belongs to the + // same group. + parent_input_idx: self.current_input_idx, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + }); + } + } + + /// Push a NONE-severity block: the accumulated non-diagnostic + /// chatter between two classified messages (Vivado tables, + /// section headers, banners, `VHDL Output written to …` lines). + /// Always uses [`ScrollbackKind::Chatter`] — the "background + /// noise" bucket that carries the dim dark-gray body style so + /// non-diagnostic output visually reads as elidable. Whether + /// the entry is collapsed / expanded / not-collapsible is + /// decided by [`Self::push`]'s threshold logic (multi-line + /// entries over the auto-collapse threshold start collapsed; + /// smaller ones expand). + pub(crate) fn push_none_block(&mut self, lines: Vec) { + if lines.is_empty() { + return; + } + self.push(ScrollbackKind::Chatter, lines.join("\n")); + } + + /// Push a stack trace as its own entry, always Chatter-styled + /// and always start-collapsed when it has 2+ lines. Used by the + /// diagnostic stream path to split the `at :` tail + /// off a WARNING/ERROR body so the human-readable message + /// stays fully visible while the stack becomes a + /// `▶ at ` placeholder — Shift+click to expand. + /// Bypasses [`Self::push`]'s auto-threshold: even a 3-frame + /// stack collapses, which is the whole point of the split. + /// Single-line stacks (one frame) fit in a row and stay + /// non-collapsible — a placeholder for a 40-char line reads + /// as noise. + pub(crate) fn push_stack_trace(&mut self, stack: String) { + let text = if stack.contains('\t') { + stack.replace('\t', " ") + } else { + stack + }; + let collapse_state = if text.lines().count() < 2 { + None + } else { + Some(true) + }; + self.scrollback.push(ScrollbackEntry { + kind: ScrollbackKind::Chatter, + text, + started_at: None, + completed_at: None, + collapse_state, + is_critical_warning: false, + // Stack traces always attach to the just-emitted + // diagnostic; that diagnostic is a child of the + // current input group, so the trace belongs there too. + parent_input_idx: self.current_input_idx, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + }); + } + + /// Per-Input-entry timer advance triggered by an EvalDone. + /// If `just_finished_idx` matches any uncompleted boundary's + /// `last_command_idx`, freeze its scrollback entry's + /// `completed_at` and anchor the next uncompleted boundary's + /// `started_at` to NOW so its timer begins fresh rather than + /// inheriting the elapsed time from earlier statements' + /// commands. + fn advance_input_timers(&mut self, just_finished_idx: usize) { + if let Some(hit) = self.freeze_input_boundary(just_finished_idx) { + // Activate the next uncompleted boundary — pushes its + // echo and stamps `started_at`. Empty boundaries (no + // lowered commands attributed) get echoed + frozen + // instantly and we cascade to the next one; otherwise + // no EvalDone would ever close them and the batch would + // stall at that point in the visual trace. + self.activate_next_boundary(hit + 1); + } + } + + /// Freeze the boundary whose `last_command_idx` matches + /// `just_finished_idx` without cascading to the next + /// statement. Used by the failing-eval path so an error trace + /// isn't visually preceded by the next boundary's echo — the + /// batch is aborting, the next echo would be misleading, and + /// (worse) it would appear BEFORE the failure explanation the + /// user needs to read. Returns the boundary's index in + /// `pending_input_boundaries` when found, or `None` when this + /// EvalDone doesn't close any boundary (e.g. a synthetic + /// prelude command). + fn freeze_input_boundary( + &mut self, + just_finished_idx: usize, + ) -> Option { + let now = std::time::Instant::now(); + // Find the first uncompleted boundary whose + // last_command_idx matches. Multi-statement load + // batches process commands in order, so the matching + // boundary is always at the head of the uncompleted + // run. + let mut hit_position: Option = None; + for (i, b) in self.pending_input_boundaries.iter().enumerate() { + if b.completed { + continue; + } + if b.last_command_idx == Some(just_finished_idx) { + hit_position = Some(i); + } + break; + } + let hit = hit_position?; + self.pending_input_boundaries[hit].completed = true; + if let Some(idx) = self.pending_input_boundaries[hit].scrollback_idx { + if let Some(entry) = self.scrollback.get_mut(idx) { + if entry.completed_at.is_none() { + entry.completed_at = Some(now); + } + } + } + Some(hit) + } + + /// Push the echo for the first uncompleted boundary starting + /// at `start`, stamp its `started_at`, and freeze-and-cascade + /// past any empty boundaries encountered along the way. Used + /// both at batch dispatch (starting from index 0) and by + /// `advance_input_timers` when the prior boundary closes. + fn activate_next_boundary(&mut self, start: usize) { + let now = std::time::Instant::now(); + let mut i = start; + while i < self.pending_input_boundaries.len() { + if self.pending_input_boundaries[i].completed { + i += 1; + continue; + } + let has_commands = + self.pending_input_boundaries[i].last_command_idx.is_some(); + // Push echo lazily if we haven't already (the first + // boundary at batch dispatch may already have a + // scrollback_idx assigned). + let idx = match self.pending_input_boundaries[i].scrollback_idx { + Some(idx) => idx, + None => { + let snippet = + self.pending_input_boundaries[i].snippet.clone(); + let idx = self.scrollback.len(); + self.push(ScrollbackKind::Input, snippet); + self.pending_input_boundaries[i].scrollback_idx = Some(idx); + idx + } + }; + if let Some(entry) = self.scrollback.get_mut(idx) { + entry.started_at = Some(now); + } + if has_commands { + // Real boundary — wait for its EvalDone to close it. + return; + } + // Empty boundary: no lowered commands, so no EvalDone + // will match. Freeze it now with a zero-second timer + // and cascade to the next. + self.pending_input_boundaries[i].completed = true; + if let Some(entry) = self.scrollback.get_mut(idx) { + entry.completed_at = Some(now); + } + i += 1; + } + } + + /// Stamp `completed_at` on every still-running Input entry + /// from the most recent batch. Called from the `EvalDone` + /// handler on `last_in_batch` so the per-input timers freeze + /// at their final duration once the batch has finished + /// evaluating. For `--load` echoed batches with multiple + /// Input entries (one per top-level statement) all entries + /// freeze at the same wall time — finer-grained per-statement + /// timing would require carrying the eval-to-input mapping + /// through the worker round-trip, which is more plumbing + /// than the v1 timer needs. + fn mark_inputs_completed(&mut self) { + let now = std::time::Instant::now(); + for entry in self.scrollback.iter_mut().rev() { + if matches!(entry.kind, ScrollbackKind::Input) + && entry.completed_at.is_none() + { + entry.completed_at = Some(now); + } else if entry.completed_at.is_some() + && matches!(entry.kind, ScrollbackKind::Input) + { + // Already-completed Input from a prior batch — + // we've walked past the current batch's inputs. + break; + } + } + } + + /// Apply a signed scroll delta (positive = down toward newer + /// content, negative = up toward older content). Disengages + /// tail-follow when the user scrolls up; re-engages when + /// they scroll down past the bottom — same semantics as a + /// scroll-wheel in a terminal emulator. + /// + /// Anchors the new offset against `last_rendered_scroll` (set + /// by the renderer each frame) rather than `scrollback_scroll`, + /// because while tail-follow is on `scrollback_scroll` is stale + /// — the renderer computes the effective bottom-aligned offset + /// without writing it back to that field. Starting the manual + /// delta from the rendered offset is what lets Ctrl-K from + /// tail-follow mode actually move up by 5 instead of jumping + /// to position 5. + fn scroll_by(&mut self, delta: i32) { + let base = self.last_rendered_scroll as i32; + let new = base.saturating_add(delta).max(0) as u16; + if delta < 0 { + self.scrollback_follow = false; + } else if delta > 0 && new >= self.last_max_scroll { + // Downward scroll that reaches (or passes) the bottom + // re-engages tail-follow — standard terminal-emulator + // behavior. Safe now that `last_max_scroll` is the + // renderer's exact wrapped-row math, not the old + // raw-line-count heuristic that misfired on wrapped + // multi-MB entries (spuriously snapping back to bottom + // on any scroll-up and making scroll appear dead). + self.scrollback_follow = true; + } + self.scrollback_scroll = new; + // Predictively mirror the new offset into + // `last_rendered_scroll`. Without this, drag-to-select + // auto-scrolls but the subsequent `cell_to_buffer` call + // in the same event still uses the previously-rendered + // value — so the selection cursor lags one drag event + // behind the scroll. The renderer will write the + // actually-rendered offset back next frame (which may + // clamp to max_scroll), so this is at worst a one-frame + // optimistic preview. + self.last_rendered_scroll = new; + } +} + +// --------------------------------------------------------------------- +// Worker task: owns the Vivado backend, serializes evals. +// --------------------------------------------------------------------- + +#[allow(clippy::too_many_arguments)] +async fn worker_task( + mut rx: mpsc::Receiver, + tx: mpsc::UnboundedSender, + verbose: bool, + verbose_log: Option, + info_with_stack: bool, + part: Option, + variant: Option, + rpc_workspace_root: Option, + preload: vw_vivado::SharedPreload, + worker: crate::Worker, +) { + // Auto-project bootstrap: same rule as `vw run`. If the + // enclosing workspace declares `[[target-parts]]` or + // `[[workspace.variants]]`, create an in-memory project up + // front so `ip::check`, `get_ipdefs`, and every other + // project-scoped call have a project to read at the first + // user eval. `part` (part-mode workspaces) and `variant` + // (variant-mode workspaces) come from `--part` / `--variant` + // respectively; the workspace shape decides which applies. + let ws_utf8 = rpc_workspace_root + .as_deref() + .and_then(camino::Utf8Path::from_path) + .map(|p| p.to_path_buf()); + let (auto_project, active_variant) = match ws_utf8.as_deref() { + Some(ws) => match resolve_worker_selection( + ws, + part.as_deref(), + variant.as_deref(), + ) { + Ok(pair) => pair, + Err(e) => { + let _ = tx.send(WorkerEvent::StartFailed( + vw_eda::BackendError::Worker(e), + )); + return; + } + }, + None => (None, None), + }; + // RPC handler — mirrors `vw run`'s. `vw::workspace_root` + // answers with the entry / cwd's nearest `vw.toml` parent; + // unknown methods fail loudly so future htcl calls surface + // a clear "unknown method" instead of hanging. The + // session-scoped `active_variant` is the fallback the + // `vhdl_design_sources` filter uses when no per-call kwarg + // overrides it. + // Raw byte-log for the session: /target/logs/vivado-.log. + // Under the REPL's alternate screen we can't safely eprintln! the + // path (it'd race the TUI), so we swallow errors silently and note + // the path once the terminal is restored via `info!`. + let raw_log = rpc_workspace_root.as_deref().and_then(|ws| { + match vw_vivado::raw_log_path_for_workspace(ws) { + Ok(p) => { + tracing::info!(path = %p.display(), "raw vivado log"); + Some(p) + } + Err(e) => { + tracing::warn!(error = %e, "raw log unavailable"); + None + } + } + }); + // RPC handler with the shared preload map — App owns the + // Arc's other end and updates the map after every + // `session.commit()` from `session.loaded_paths()`. See the + // `SharedPreload` docs in vw-vivado. + // + // Also carries a session-scoped CW counter. The sink below + // bumps it on every `Severity::CriticalWarning` chunk, and + // `vw::synth` / `vw::place` read it via + // `vw::critical_warning_count` to gate their checkpoint + // writes on a CW-clean phase. Same wiring as `vw run` — the + // REPL must not diverge or the exact same htcl session would + // persist a checkpoint here that `vw run` would refuse. + let cw_count: vw_vivado::SharedCriticalWarningCount = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let rpc_handler = vw_vivado::make_handler_full( + rpc_workspace_root, + active_variant, + preload, + cw_count.clone(), + ); + // Everything assembled above — the RPC handler, the project, the raw log — + // is for a vivado on this machine. A remote session's instance built its + // own from the tree it is holding, because every one of those answers is + // about where the files are, and the files are there. + let mut backend: Box = match worker { + crate::Worker::Local => { + let spawned = + vw_vivado::VivadoBackend::spawn(vw_vivado::VivadoConfig { + verbose, + verbose_log, + info_with_stack, + rpc_handler: Some(rpc_handler), + auto_project, + raw_log, + ..Default::default() + }) + .await; + match spawned { + Ok(b) => { + // Vivado's on-disk binary forks its real process without + // `exec`, so cancelling means signalling the group. The + // reasoning lives in `vw_vivado::interrupt_process_group`. + let interrupt: crate::Interrupt = match b.child_pid() { + Some(pid) => std::sync::Arc::new(move || { + vw_vivado::interrupt_process_group(pid) + }), + None => std::sync::Arc::new(|| {}), + }; + let _ = tx.send(WorkerEvent::Started { interrupt }); + Box::new(b) + } + Err(e) => { + let _ = tx.send(WorkerEvent::StartFailed(e)); + return; + } + } + } + crate::Worker::Remote { backend, interrupt } => { + let _ = tx.send(WorkerEvent::Started { interrupt }); + backend + } + }; + + // Stream chunks to the UI as they arrive. The closure + // captures the unbounded sender so it can fire without + // awaiting. The kind tag (`StreamKind::Stdout` for user `puts` + // output, `Warning`/`Error`/`Info` for Vivado's own message + // lines harvested from the PTY) flows through unchanged so + // the UI can colour them appropriately. + let stdout_tx = tx.clone(); + let cw = cw_count.clone(); + backend.set_stdout_sink(Box::new(move |kind, chunk: &str| { + // Bump the CW counter exposed via the + // `critical_warning_count` RPC. Only exact + // CriticalWarning — errors already halt eval before any + // htcl checkpoint-write branch runs, so counting them + // here would double-report. + if matches!(kind, vw_vivado::StreamKind::CriticalWarning) { + cw.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + let _ = stdout_tx.send(WorkerEvent::Stream { + kind, + data: chunk.to_string(), + }); + })); + + while let Some(cmd) = rx.recv().await { + match cmd { + WorkerCmd::EvalBatch(items) => { + let total = items.len(); + for (i, item) in items.into_iter().enumerate() { + // Wrap each command's Tcl body with a + // shim-level origin marker so any traceless + // warning it emits stays tagged with THIS + // command's origin even when its PTY bytes lag + // the protocol response into the next eval's + // window. Without this the origin fallback in + // `tag_streamed_message` uses whatever + // `pending_eval_index` points at — which for a + // batch's synthetic prelude commands means + // `line=0` / `line=1` no-op tags. + let wrapped = crate::wrap_tcl_with_origin_marker( + &item.tcl, + &item.origin, + ); + let result = backend.eval(&wrapped).await; + let failed = result.is_err(); + let last_in_batch = i + 1 == total || failed; + let _ = tx.send(WorkerEvent::EvalDone { + origin: item.origin, + result, + last_in_batch, + }); + // Stop the batch at the first failure — running + // the rest of a script after an error confuses + // the user and risks side effects nobody + // intended. + if failed { + break; + } + } + } + WorkerCmd::Shutdown => break, + } + } + let _ = backend.shutdown().await; +} + +// --------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------- + +/// Render a Vivado error as a clean Python-style stack trace — +/// one frame per `file:line` from the outermost `src` the user +/// typed, down through any nested `src` imports, the leaf +/// statement we shipped, and any `(procedure X line N)` frames the +/// Tcl interpreter reported. The error message itself comes last. +/// +/// ```text +/// ip/cips.htcl:1 +/// src @cips +/// ~/src/htcl/amd/cips/module.htcl:3 +/// ip::check -name "xilinx.com:ip:versal_cips:3.4" +/// ~/src/htcl/amd/vivado-cmd/ip.htcl:18 +/// set ip_obj [get_ipdefs -all "$name"] +/// ERROR: [Common 17-53] No open project. ... +/// ``` +fn render_eval_error( + app: &mut App, + origin: &crate::lower::Origin, + err: vw_eda::BackendError, +) { + let mut frames: Vec = Vec::new(); + + // Outermost first: walk the `via` chain in reverse so the + // entry `src` lands at the top. + for f in origin.via.iter().rev() { + frames.push(Frame { + file: f.file.clone(), + line: f.line, + snippet: f.snippet.clone(), + }); + } + // Leaf htcl statement — the actual call site that triggered + // the Tcl evaluation. + frames.push(Frame { + file: origin.file.clone(), + line: origin.line, + snippet: origin.snippet.clone(), + }); + + // If Vivado gave us a Tcl trace, drill into any + // `(procedure "X" line N)` frames whose proc we recognize, so + // the user sees the actual failing line inside the proc body + // — not just the call to it. + let (message, code, info, stdout) = match err { + vw_eda::BackendError::Tcl { + message, + code, + info, + stdout, + } => (message, code, info, stdout), + other => { + for frame in &frames { + push_frame(app, frame); + } + app.push(ScrollbackKind::Error, format!("{other}")); + return; + } + }; + if let Some(info) = info.as_deref() { + let session_guard = app.session.read().unwrap(); + for tcl_frame in parse_tcl_proc_frames(info) { + // Check the in-flight batch first (the lowering that + // just ran), then fall back to prior session batches. + // This is what gives wrappers declared in earlier + // inputs a real `.htcl` path in the drill-down trace + // instead of an `(input):N` line in a vanished scratch. + let loc = app + .pending_batch + .as_ref() + .and_then(|b| b.procs.get(&tcl_frame.proc)) + .or_else(|| session_guard.lookup_proc(&tcl_frame.proc)); + let Some(loc) = loc else { continue }; + let Some((abs_line, content)) = + loc.resolve_body_line(tcl_frame.line) + else { + continue; + }; + frames.push(Frame { + file: loc.file.clone(), + line: abs_line, + snippet: content.trim().to_string(), + }); + } + } + + if !stdout.is_empty() { + app.push( + ScrollbackKind::Stdout, + stdout.trim_end_matches('\n').to_string(), + ); + } + + for frame in &frames { + push_frame(app, frame); + } + // Split `ERROR: [X] Msg\n Resolution: …` so the tagged + // first line pushes as its own single-line entry (full + // brightness, no `▼` dimming) — see `split_leading_diagnostic`. + let (leading, trailing) = split_leading_diagnostic(message.trim()); + app.push(ScrollbackKind::Error, leading); + if let Some(trailing) = trailing { + app.push(ScrollbackKind::Chatter, trailing); + } + if let Some(code) = code.filter(|s| !s.is_empty() && s != "NONE") { + app.push(ScrollbackKind::Notice, format!("({code})")); + } +} + +struct Frame { + file: Option, + line: u32, + snippet: String, +} + +fn push_frame(app: &mut App, frame: &Frame) { + let where_ = render_origin_path(frame.file.as_deref(), frame.line); + app.push(ScrollbackKind::Notice, where_); + if frame.snippet.is_empty() { + return; + } + // Indent every line of the snippet — for a multi-line + // command (`set proj [\n create_project\n -name x\n]`) + // this preserves the user's relative indentation so the + // structure is readable, while the gutter prefix added by + // `entry_lines` distinguishes the first line from the + // continuations. + let body = frame + .snippet + .lines() + .map(|line| format!(" {line}")) + .collect::>() + .join("\n"); + app.push(ScrollbackKind::Notice, body); +} + +/// Parse `(procedure "NAME" line N)` annotations out of Tcl's +/// `$errorInfo`. Returned in the order they appear in `info`, +/// which is innermost-first per Tcl convention — but the renderer +/// wants OUTERMOST-first (we already have the outer leaf frame from +/// the htcl side), so we reverse here and yield the inner frames +/// in execution order. +fn parse_tcl_proc_frames(info: &str) -> Vec { + let mut out = Vec::new(); + for line in info.lines() { + let trimmed = line.trim(); + // Expected shape: `(procedure "NAME" line N)` + let Some(rest) = trimmed.strip_prefix("(procedure \"") else { + continue; + }; + let Some((name, rest)) = rest.split_once("\" line ") else { + continue; + }; + let Some(num) = rest.strip_suffix(')') else { + continue; + }; + let Ok(n) = num.parse::() else { continue }; + out.push(TclProcFrame { + proc: name.to_string(), + line: n, + }); + } + // errorInfo lists innermost first; we want execution order + // (outermost first) so reverse. + out.reverse(); + out +} + +struct TclProcFrame { + proc: String, + line: u32, +} + +/// Rewrite `:N in ::procname` frames in a Vivado message to +/// point at the actual htcl source file and line. Delegates the +/// per-line parsing + dedup to [`crate::trace`], which is shared +/// with the `vw run` CLI driver so both surfaces render the same. +/// This wrapper closes over the REPL's session+pending proc lookup. +fn resolve_stack_frames( + msg: &str, + session: &Session, + pending: Option<&SessionBatch>, + input_file: Option<&std::path::Path>, +) -> String { + crate::trace::resolve_stack_frames_with( + msg, + |name| { + pending + .and_then(|b| b.procs.get(name)) + .or_else(|| session.lookup_proc(name)) + .cloned() + }, + input_file, + ) +} + +fn render_origin_path(file: Option<&std::path::Path>, line: u32) -> String { + match file { + Some(p) => format!("{}:{line}", display_path(p)), + None => format!("(input):{line}"), + } +} + +/// Shorten a path for display: drop the cwd prefix when it lines +/// up, leave it absolute otherwise. Saves screen real estate when +/// reporting errors from a dep cached deep under `~/.vw/deps/...`. +fn display_path(path: &std::path::Path) -> String { + if let Ok(cwd) = std::env::current_dir() { + if let Ok(rel) = path.strip_prefix(&cwd) { + return rel.display().to_string(); + } + if let Some(home) = dirs::home_dir() { + if let Ok(rel) = path.strip_prefix(&home) { + return format!("~/{}", rel.display()); + } + } + } + path.display().to_string() +} + +/// Decide whether the input buffer parses cleanly enough to ship to +/// Write an OSC 52 set-clipboard escape to stdout, base64-encoding +/// `text` per the protocol. The terminal puts the decoded text on +/// the system clipboard — no DISPLAY/Wayland-socket/pbcopy +/// dependency, and the same code path works over SSH. +/// +/// Some terminals cap the payload size at ~74KB (the original xterm +/// limit) or somewhere similar; selections larger than that may be +/// truncated by the terminal. Encoding/IO errors are swallowed — +/// the caller has nowhere useful to surface them, since OSC 52 is +/// fire-and-forget (the terminal doesn't ack). +fn send_osc52(text: &str) { + use base64::engine::general_purpose::STANDARD; + use base64::Engine; + use std::io::Write; + let encoded = STANDARD.encode(text.as_bytes()); + let payload = format!("\x1b]52;c;{encoded}\x07"); + let mut stdout = std::io::stdout(); + let _ = stdout.write_all(payload.as_bytes()); + let _ = stdout.flush(); +} + +/// Vivado, or whether the user is still in the middle of typing +/// (unterminated brace, etc.). We re-use the htcl parser since +/// it already understands every multi-line construct (procs, +/// `[ … ]` substitutions, braced groups). +/// Walk a Document looking for `proc ` (recursing into +/// `namespace eval` blocks). Returns the `Command.doc_comments` +/// slice when found. Used by the signature-help lookup path to +/// surface the proc's `##` docs alongside its argument list. +/// +/// Matches the recursion shape that `vw_htcl::signature_table` +/// uses — qualified names like `util::props` resolve to a proc +/// declared inside `namespace eval util { … }`. +/// Same rule as `vw-cli::resolve_workspace_selection`, mirrored +/// here so the REPL worker doesn't take a cross-crate dep on the +/// CLI. Returns `(auto_project, active_variant)` when the +/// workspace is happy with the flags, `Err(String)` for +/// mode-mismatch or bad selectors. +fn resolve_worker_selection( + ws: &camino::Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result<(Option, Option), String> { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok((None, None)); + }; + let ws_info = &cfg.workspace; + if variant.is_some() && ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} has no `[[workspace.variants]]`; \ + remove `--variant` or add variants to vw.toml", + )); + } + if part.is_some() && !ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} is variant-mode; use `--variant ` \ + instead of `--part`", + )); + } + if !ws_info.variants.is_empty() { + let Some(v) = + ws_info.select_variant(variant).map_err(|e| e.to_string())? + else { + return Ok((None, None)); + }; + let persist_dir = prepare_repl_persist_dir(ws, &ws_info.name); + return Ok(( + Some(vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: v.part.clone(), + persist_dir, + }), + Some(v.name.clone()), + )); + } + let selected = ws_info + .select_target_part(part) + .map_err(|e| e.to_string())?; + let persist_dir = prepare_repl_persist_dir(ws, &ws_info.name); + Ok(( + selected.map(|p| vw_vivado::AutoProject { + name: ws_info.name.clone(), + part: p.to_string(), + persist_dir: persist_dir.clone(), + }), + None, + )) +} + +/// REPL sibling of `vw-cli::prepare_persist_dir`. Same behavior: +/// runs Phase-6 legacy IP-cache cleanup + Phase-3 staleness wipe, +/// returns `Some(/target/vw-project)` for the worker to +/// `open_project`/`create_project -dir` into, or `None` if the +/// bootstrap fails (falling back to in-memory). +/// +/// Kept mirrored (not shared through a common crate) for the same +/// reason `resolve_worker_selection` is duplicated: `vw-repl` +/// doesn't take a cross-crate dep on `vw-cli`. +fn prepare_repl_persist_dir( + ws: &camino::Utf8Path, + name: &str, +) -> Option { + match vw_lib::prepare_vw_project_dir(ws, name) { + Ok(prep) => { + if prep.legacy_cache_removed > 0 { + tracing::info!( + "removed {} legacy IP cache entr{y} under \ + {ws}/target/ip — replaced by on-disk Vivado project", + prep.legacy_cache_removed, + y = if prep.legacy_cache_removed == 1 { + "y" + } else { + "ies" + }, + ); + } + if let Some(wiped) = &prep.wiped_project { + tracing::info!( + "wiped stale Vivado project at {wiped} \ + (source fingerprint changed or manifest missing)" + ); + } + Some(prep.project_dir.into_std_path_buf()) + } + Err(e) => { + tracing::warn!( + "failed to prepare on-disk Vivado project dir under \ + {ws}/target/vw-project ({e}); falling back to in-memory \ + project (state won't persist across sessions)" + ); + None + } + } +} + +fn lookup_proc_doc_comments<'a>( + doc: &'a vw_htcl::Document, + qualified_name: &str, +) -> Option<&'a [String]> { + lookup_in_stmts(&doc.stmts, "", qualified_name) +} + +fn lookup_in_stmts<'a>( + stmts: &'a [vw_htcl::Stmt], + prefix: &str, + qualified_name: &str, +) -> Option<&'a [String]> { + use vw_htcl::CommandKind; + for stmt in stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + if let Some(name) = proc.name.as_deref() { + let qualified = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + if qualified == qualified_name { + return Some(&cmd.doc_comments); + } + } + } + CommandKind::NamespaceEval(ns) => { + if let Some(name) = ns.name.as_deref() { + let nested = if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + }; + if let Some(d) = + lookup_in_stmts(&ns.body, &nested, qualified_name) + { + return Some(d); + } + } + } + _ => {} + } + } + None +} + +/// Index of the parameter the cursor is on within `sig.args`. +/// Mirrors the logic in `vw_htcl::signature_help::active_parameter`: +/// the active arg is whichever `-flag` was most recently completed +/// on the line, or — when the partial is still being typed — the +/// first arg whose name has the partial as a prefix. +fn compute_active_parameter( + sig: &vw_htcl::ProcSignature, + line: &vw_htcl::cmdline::CmdLine<'_>, +) -> Option { + let mut active = None; + for word in line.words.iter().skip(1) { + if let Some(flag) = word.strip_prefix('-') { + if let Some(i) = sig.args.iter().position(|a| a.name == flag) { + active = Some(i as u32); + } + } + } + if let Some(flag) = line.partial.strip_prefix('-') { + if !flag.is_empty() { + if let Some(i) = + sig.args.iter().position(|a| a.name.starts_with(flag)) + { + return Some(i as u32); + } + } + } + active +} + +/// Identifier under the cursor — bare-word-shape including the `::` +/// namespace separator so `util::props` resolves as one symbol. +/// Used by the hover lookup path to find what the user is pointing +/// at when [`vw_htcl::hover_at`] can't see the proc (because it's +/// defined in a session batch, not the in-flight input). +fn ident_under_cursor(text: &str, offset: u32) -> Option<&str> { + let bytes = text.as_bytes(); + let o = (offset as usize).min(bytes.len()); + let is_word_byte = + |b: u8| -> bool { b.is_ascii_alphanumeric() || b == b'_' || b == b':' }; + let mut start = o; + while start > 0 && is_word_byte(bytes[start - 1]) { + start -= 1; + } + let mut end = o; + while end < bytes.len() && is_word_byte(bytes[end]) { + end += 1; + } + if start < end { + Some(&text[start..end]) + } else { + None + } +} + +/// Build the title line for a hover popup that points at a proc — +/// shows the proc's name plus its signature in compact one-line form +/// (`name -arg1: type -arg2: type → ret`). The body of the popup is +/// the reflowed doc-comment block. +fn render_proc_title(name: &str, sig: &vw_htcl::ProcSignature) -> String { + let mut out = name.to_string(); + for arg in &sig.args { + out.push_str(" -"); + out.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + out.push_str(": "); + out.push_str(&render_type(ty)); + } + if let Some(default) = format_default_value(arg) { + out.push_str(" = "); + out.push_str(&default); + } + } + if let Some(ret) = sig.return_type.as_ref() { + out.push_str(" → "); + out.push_str(&render_type(ret)); + } + out +} + +/// Convert a [`vw_htcl::hover::HoverTarget`] into our owned +/// [`crate::popup::HoverPopup`]. Returns `None` when the target +/// can't be rendered usefully (anonymous procs, missing signatures). +fn hover_target_to_popup( + target: vw_htcl::HoverTarget<'_>, + anchor: (u16, u16), +) -> Option { + use vw_htcl::HoverTarget; + let (title, body) = match target { + HoverTarget::ProcDef { proc, .. } => { + let name = proc.name.clone()?; + let sig = proc.signature.as_ref()?; + // Doc comments live on the enclosing Command. The hover + // module doesn't surface them here — accept that we + // show only the signature for in-buffer proc decls; + // the user can re-hover the call site for full docs. + (render_proc_title(&name, sig), String::new()) + } + HoverTarget::ProcArgDef { arg, .. } => { + let mut title = format!("-{}", arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + title.push_str(": "); + title.push_str(&render_type(ty)); + } + let body = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + (title, body) + } + HoverTarget::CallSite { + proc_name, + signature, + .. + } => (render_proc_title(&proc_name, signature), String::new()), + HoverTarget::CallArg { arg, .. } => { + let mut title = format!("-{}", arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + title.push_str(": "); + title.push_str(&render_type(ty)); + } + let body = vw_htcl::doc::reflow_doc_comments(&arg.doc_comments); + (title, body) + } + HoverTarget::LocalVar { name, .. } => { + (format!("${name}"), String::from("local variable")) + } + HoverTarget::EnumDef { decl, .. } => { + let name = decl.name.clone()?; + let variants: Vec = decl + .variants + .iter() + .map(|v| { + if let Some(ty) = v.payload.as_ref() { + format!("{}: {}", v.name, render_type(ty)) + } else { + v.name.clone() + } + }) + .collect(); + let title = format!("enum {name}"); + let body = format!("{{ {} }}", variants.join("; ")); + (title, body) + } + HoverTarget::TypeDef { decl, .. } => { + let name = decl.name.clone()?; + let title = format!("type {name}"); + let body = match decl.underlying.as_ref() { + Some(ty) => format!("= {}", render_type(ty)), + None => String::new(), + }; + (title, body) + } + }; + Some(crate::popup::HoverPopup { + title, + body, + anchor, + }) +} + +/// Return the indices of `sig.args` in completion-popup / +/// signature-help **display order**: required arguments (no +/// `@default(...)` attribute) first, then optional ones, both +/// groups sorted alphabetically within. Source declaration order +/// often follows IP-XACT or generator conventions that don't match +/// what users want to scan visually — surfacing required args at +/// the top makes "what MUST I supply?" answerable at a glance. +/// +/// The display order is just a permutation of `sig.args` indices; +/// callers map the `active_parameter` (which is computed in +/// declaration-order space) into display space via `.position()`. +fn sorted_arg_indices(sig: &vw_htcl::ProcSignature) -> Vec { + let mut indices: Vec = (0..sig.args.len()).collect(); + indices.sort_by(|&a, &b| { + let a_arg = &sig.args[a]; + let b_arg = &sig.args[b]; + let a_has_default = a_arg.attribute("default").is_some(); + let b_has_default = b_arg.attribute("default").is_some(); + // false < true → no-default (required) sorts before has-default. + a_has_default + .cmp(&b_has_default) + .then_with(|| a_arg.name.cmp(&b_arg.name)) + }); + indices +} + +/// Detail string shown next to a `-flag` row in the completion popup. +/// Combines the arg's type annotation (when present) with its +/// `@default(...)` value (when present). Returns `None` when the +/// arg has neither so the popup row stays compact for untyped +/// undefaulted args. +fn build_flag_detail(arg: &vw_htcl::ast::ProcArg) -> Option { + let ty = arg.type_annotation.as_ref().map(render_type); + let default = format_default_value(arg); + match (ty, default) { + (None, None) => None, + (Some(t), None) => Some(t), + (None, Some(d)) => Some(format!("= {d}")), + (Some(t), Some(d)) => Some(format!("{t} = {d}")), + } +} + +/// Extract a proc arg's `@default(...)` value, formatted as a short +/// display string. Returns `None` when the arg has no default. Long +/// values (multi-KB paired-dict literals from IP-XACT generators) +/// are truncated to the first ~32 chars with an ellipsis so the +/// signature-help / hover / completion popups don't blow wide. +pub fn format_default_value(arg: &vw_htcl::ast::ProcArg) -> Option { + let attr = arg.attribute("default")?; + let first = attr.values.first()?; + let raw = first.to_tcl_literal(); + const MAX: usize = 32; + if raw.chars().count() > MAX { + let truncated: String = raw.chars().take(MAX - 1).collect(); + Some(format!("{truncated}…")) + } else { + Some(raw) + } +} + +/// One-line rendering of a `TypeExpr` — `string`, `dict`, etc. +/// Used by the signature-help popup to show arg + return types +/// alongside the names. +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + use vw_htcl::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(", ")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +/// Decide whether the current input buffer is ready to submit. +/// +/// Two hard "no": unterminated brace/bracket errors from the parser, +/// or a "missing required argument" diagnostic from the validator. +/// The latter is the key ergonomic hook: when the user types +/// `vivado_cmd::assign_bd_address` alone and hits Enter, the +/// validator sees a call to a proc with required args that haven't +/// been supplied. Treat as incomplete → the REPL appends a newline +/// so the user can continue with `-offset …` on the next line. +/// Once every required arg is supplied, the diagnostic clears and +/// Enter submits. +/// +/// The signature table comes from the app's session so procs +/// defined earlier in the same REPL run are visible; passing an +/// empty map degrades gracefully to "unterminated-only" behavior +/// (Slice 5's compositional constructors have required args but +/// aren't in the session table for unit tests). +fn is_buffer_complete( + text: &str, + sig_table: &std::collections::HashMap, +) -> bool { + let parsed = vw_htcl::parse(text); + if parsed + .errors + .iter() + .any(|e| e.message.contains("unterminated")) + { + return false; + } + // Ask the validator whether required-arg gaps remain. Any + // other kind of diagnostic (unknown proc, type error, etc.) is + // a real error the user should see; incomplete is reserved + // for the specific "waiting for more flags" state. + let diags = + vw_htcl::validate_with_signatures(&parsed.document, text, sig_table); + let waiting = diags.iter().any(|d| { + matches!(d.severity, vw_htcl::Severity::Error) + && d.message.starts_with("missing required argument") + }); + !waiting +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_sigs( + ) -> std::collections::HashMap + { + std::collections::HashMap::new() + } + + #[test] + fn buffer_complete_for_simple_statement() { + assert!(is_buffer_complete("set x 1", &empty_sigs())); + assert!(is_buffer_complete("puts \"hi\"", &empty_sigs())); + } + + #[test] + fn buffer_incomplete_with_unterminated_brace() { + assert!(!is_buffer_complete( + "set x [\n create_cpm5\n -name cpm5", + &empty_sigs() + )); + assert!(!is_buffer_complete("proc foo {", &empty_sigs())); + } + + #[test] + fn buffer_complete_for_multiline_well_formed_proc() { + assert!(is_buffer_complete( + "proc foo {\n @default(1) x\n} {\n puts $x\n}", + &empty_sigs() + )); + } + + /// The core UX fix: when a proc call is missing required args, + /// the buffer is considered incomplete so Enter appends a + /// newline instead of submitting — the user can continue with + /// `-flag value` continuations. + #[test] + fn buffer_incomplete_when_required_args_missing() { + use vw_htcl::{parse, signature_table}; + let src = "\ +proc greet { + name + msg +} unit { + puts \"$msg $name\" +} +"; + let parsed = parse(src); + let sigs = signature_table(&parsed.document); + // Bare call — both required args missing. + assert!(!is_buffer_complete("greet", &sigs)); + // One required arg missing — still incomplete. + assert!(!is_buffer_complete("greet -name there", &sigs)); + // Both provided — complete. + assert!(is_buffer_complete("greet -name there -msg hi", &sigs)); + } + + // --- stack-frame resolution --------------------------------- + + use crate::lower::ProcLocation; + use crate::session::SessionBatch; + use std::collections::HashMap; + use std::path::PathBuf; + use vw_htcl::{parse, LoadedProgram}; + + fn session_with_proc( + proc: &str, + file: PathBuf, + body_start_line: u32, + body_lines: Vec, + ) -> Session { + // Session stores proc names without the leading `::` — + // see `lower::qualify`. + let key = proc.strip_prefix("::").unwrap_or(proc); + let src = format!("proc {key} {{}} {{}}\n"); + let parsed = parse(&src); + let mut procs = HashMap::new(); + procs.insert( + key.to_string(), + ProcLocation { + file: Some(file), + body_start_line, + body_lines, + }, + ); + let batch = SessionBatch { + program: LoadedProgram { + source: src, + files: Vec::new(), + regions: Vec::new(), + }, + document: parsed.document, + procs, + }; + let mut s = Session::new(); + s.commit(batch); + s + } + + #[test] + fn rewrite_resolves_input_line_to_absolute_file_line() { + let session = session_with_proc( + "::configure_cips", + "ip/cips.htcl".into(), + 95, + (0..30).map(|i| format!("body line {i}")).collect(), + ); + let frame = crate::trace::rewrite_stack_line( + " at :14 in ::configure_cips", + |name| session.lookup_proc(name).cloned(), + None, + ) + .expect("should resolve"); + // body line 14 = body_start_line (95) + (14 - 1) = 108 + assert!( + frame.formatted.contains("ip/cips.htcl:108"), + "got {:?}", + frame.formatted + ); + assert_eq!(frame.line, 108); + } + + #[test] + fn rewrite_resolves_namespaced_proc() { + // Tcl reports `::port::plumb_if_pin` (with leading `::`) + // but the session indexes it as `port::plumb_if_pin`. + let session = session_with_proc( + "::port::plumb_if_pin", + "vivado-cmd/port.htcl".into(), + 70, + (0..10).map(|i| format!("line {i}")).collect(), + ); + let frame = crate::trace::rewrite_stack_line( + " at :5 in ::port::plumb_if_pin", + |name| session.lookup_proc(name).cloned(), + None, + ) + .expect("should resolve namespaced proc"); + assert!( + frame.formatted.contains("vivado-cmd/port.htcl:74"), + "got {:?}", + frame.formatted + ); + } + + #[test] + fn rewrite_passes_unknown_proc_through() { + let session = Session::new(); + assert!(crate::trace::rewrite_stack_line( + " at :14 in ::vivado_builtin_thing", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + } + + #[test] + fn rewrite_skips_non_frame_lines() { + let session = Session::new(); + assert!(crate::trace::rewrite_stack_line( + "WARNING: [Common 17-1] something", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + assert!(crate::trace::rewrite_stack_line( + "", + |name| session.lookup_proc(name).cloned(), + None, + ) + .is_none()); + } + + #[test] + fn resolve_dedupes_adjacent_same_proc_frames() { + // Two consecutive `:N in ::port::plumb_if_pin` frames + // resolving to the same absolute line should collapse to one. + let session = session_with_proc( + "::port::plumb_if_pin", + "vivado-cmd/port.htcl".into(), + 70, + (0..10).map(|i| format!("line {i}")).collect(), + ); + let msg = "\ +WARNING: [port::plumb_if_pin-1] skipping foo + at :5 in ::port::plumb_if_pin + at :5 in ::port::plumb_if_pin"; + let out = resolve_stack_frames(msg, &session, None, None); + // Only one resolved frame line should remain. + let count = out + .lines() + .filter(|l| l.contains("port::plumb_if_pin")) + .count(); + assert_eq!(count, 2, "got:\n{out}"); // header + 1 frame + } + + // --- request/response ordering --------------------------------- + + /// Regression: when a batch's Nth command fails, the (N+1)th + /// boundary's echo must NOT be pushed to scrollback ahead of the + /// error trace. The auto-load repro looked like this: `set cips + /// [configure_cips]` failed, then `› set clk [configure_clock]` + /// appeared, then the trace + error for `set cips`. The error + /// belongs to `set cips` and has to land BEFORE the next + /// statement's echo (or preferably: the next echo shouldn't + /// appear at all, since the batch aborts on failure). + #[tokio::test] + async fn failed_eval_does_not_activate_next_boundary_before_error() { + let (worker_tx, _worker_rx) = + tokio::sync::mpsc::channel::(8); + let (event_tx, event_rx) = + tokio::sync::mpsc::unbounded_channel::(); + let mut app = App::new( + ReplOptions::default(), + worker_tx, + event_rx, + event_tx, + CollapseMode::Normal, + std::sync::Arc::new(std::sync::RwLock::new( + std::collections::HashMap::new(), + )), + ); + + // Two-boundary batch. Only boundary 0's echo lives in + // scrollback (deferred push means boundary 1 stays lazy + // until its predecessor closes). + app.push( + ScrollbackKind::Input, + "set cips [configure_cips]".to_string(), + ); + app.pending_input_boundaries = vec![ + InputBoundary { + scrollback_idx: Some(0), + snippet: "set cips [configure_cips]".to_string(), + last_command_idx: Some(0), + completed: false, + }, + InputBoundary { + scrollback_idx: None, + snippet: "set clk [configure_clock]".to_string(), + last_command_idx: Some(1), + completed: false, + }, + ]; + let origin0 = crate::lower::Origin { + file: None, + line: 14, + snippet: "set cips [configure_cips]".to_string(), + via: Vec::new(), + }; + app.pending_origins = vec![ + origin0.clone(), + crate::lower::Origin { + file: None, + line: 15, + snippet: "set clk [configure_clock]".to_string(), + via: Vec::new(), + }, + ]; + app.pending_return_types = vec![None, None]; + + // Failed EvalDone for the first command, last_in_batch=true + // (mirrors what worker_task sends when it breaks on error). + let err = vw_eda::BackendError::Tcl { + message: "[Common 17-163] Missing value for option 'objects'" + .into(), + code: None, + info: None, + stdout: String::new(), + }; + app.handle_worker_event(WorkerEvent::EvalDone { + origin: origin0, + result: Err(err), + last_in_batch: true, + }) + .await; + + // Walk scrollback and find the two positions we care about: + // the `set clk` Input entry (if any) and the Error entry + // carrying the failure message. + let clk_pos = app + .scrollback() + .iter() + .position(|e| e.text.contains("set clk")); + let err_pos = app.scrollback().iter().position(|e| { + matches!(e.kind, ScrollbackKind::Error) + && e.text.contains("Missing value") + }); + assert!( + err_pos.is_some(), + "expected an Error entry, got scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + if let Some(clk) = clk_pos { + let err = err_pos.unwrap(); + assert!( + err < clk, + "error at {err} must land before `set clk` echo at \ + {clk} — the trace belongs to `set cips` which came \ + first. scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + } + } + + /// Regression: a pure `puts` output (which classifies as + /// `StreamKind::Stdout` → `Severity::None`) has to reach + /// scrollback even when NO diagnostic follows it. The block + /// segmenter only flushes pending NONE content when a + /// classified chunk arrives; without an explicit flush on + /// `EvalDone`, `puts "muffins"` on a quiet REPL sits invisible + /// in `pending_none` until the next diagnostic (which for a + /// small interactive session may never come). The fix: flush + /// the accumulator at eval-done boundaries so every eval's own + /// output surfaces before the next input echo. + #[tokio::test] + async fn plain_puts_output_flushes_at_eval_done() { + let (worker_tx, _worker_rx) = + tokio::sync::mpsc::channel::(8); + let (event_tx, event_rx) = + tokio::sync::mpsc::unbounded_channel::(); + let mut app = App::new( + ReplOptions::default(), + worker_tx, + event_rx, + event_tx, + CollapseMode::Normal, + std::sync::Arc::new(std::sync::RwLock::new( + std::collections::HashMap::new(), + )), + ); + // Emulate the sequence a `puts "muffins"` eval produces: + // one Stream chunk carrying the output, then EvalDone. + // Both events land through `handle_worker_event`, same as + // the real worker task drives them. + app.handle_worker_event(WorkerEvent::Stream { + kind: vw_vivado::StreamKind::Stdout, + data: "muffins\n".to_string(), + }) + .await; + // Before EvalDone the accumulator is still holding the + // chunk — nothing in scrollback yet. + assert!( + !app.scrollback().iter().any(|e| e.text.contains("muffins")), + "muffins should still be in pending_none before EvalDone" + ); + app.handle_worker_event(WorkerEvent::EvalDone { + origin: crate::lower::Origin { + file: None, + line: 1, + snippet: "puts \"muffins\"".to_string(), + via: Vec::new(), + }, + result: Ok(vw_eda::EvalOutput::default()), + last_in_batch: true, + }) + .await; + // After EvalDone the pending NONE content flushes into + // scrollback — the user actually sees their output. + assert!( + app.scrollback().iter().any(|e| e.text.contains("muffins")), + "muffins should have flushed into scrollback on EvalDone. \ + scrollback: {:#?}", + app.scrollback() + .iter() + .map(|e| (e.kind, e.text.clone())) + .collect::>() + ); + } +} diff --git a/vw-repl/src/config.rs b/vw-repl/src/config.rs new file mode 100644 index 0000000..f43dfb5 --- /dev/null +++ b/vw-repl/src/config.rs @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Workspace-local REPL configuration loaded from `/.vw/repl.toml`. +//! +//! Optional file — the REPL runs fine without it. Present so a +//! project can pin per-workspace UI preferences (currently just the +//! auto-collapse policy for scrollback entries) without touching +//! `vw.toml`, which is the *build* manifest and shouldn't be +//! littered with editor-UX knobs. + +use std::path::Path; + +use serde::Deserialize; + +/// How aggressively the REPL auto-collapses multi-line scrollback +/// entries when they land. Single-line entries are never +/// collapsible regardless of mode — a `▶` around one row of text +/// is worse UX than the row itself. +#[derive(Copy, Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CollapseMode { + /// Auto-collapse only past [`crate::app::COLLAPSE_AUTO_THRESHOLD`] + /// lines. Smaller multi-line entries land expanded but stay + /// toggleable via Shift+click. The default — matches the + /// out-of-the-box behavior users see without a config file. + #[default] + Normal, + /// Every collapsible entry (>=2 lines) starts collapsed. + /// Turns the scrollback into a compact index of `▶`-marked + /// placeholders that expand on demand — useful when running + /// long batches where most output is chatter you scroll past. + Aggressive, +} + +/// Deserialized `/.vw/repl.toml`. All fields optional so a +/// stub `[ui]` section is legal — missing keys fall back to +/// [`Default`]. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct ReplConfig { + pub ui: UiConfig, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct UiConfig { + pub collapse: CollapseMode, +} + +/// Load `/.vw/repl.toml` if it exists. Absent file → default +/// config. Malformed file → default config + a `tracing::warn!` so +/// the user notices in the verbose log but the REPL still starts; +/// a config error shouldn't be a fatal boot condition for an +/// interactive tool. +pub fn load(workspace_root: Option<&Path>) -> ReplConfig { + let Some(ws) = workspace_root else { + return ReplConfig::default(); + }; + let path = ws.join(".vw").join("repl.toml"); + let raw = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return ReplConfig::default(); + } + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "failed to read repl.toml — using defaults", + ); + return ReplConfig::default(); + } + }; + match toml::from_str::(&raw) { + Ok(cfg) => cfg, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "malformed repl.toml — using defaults", + ); + ReplConfig::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_file_is_default() { + let tmp = tempfile::tempdir().unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn no_workspace_root_is_default() { + let cfg = load(None); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn parses_aggressive() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"aggressive\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Aggressive); + } + + #[test] + fn parses_normal_explicit() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"normal\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn empty_file_is_default() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write(vw_dir.join("repl.toml"), "").unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } + + #[test] + fn malformed_file_falls_back() { + let tmp = tempfile::tempdir().unwrap(); + let vw_dir = tmp.path().join(".vw"); + std::fs::create_dir_all(&vw_dir).unwrap(); + std::fs::write( + vw_dir.join("repl.toml"), + "[ui]\ncollapse = \"chaotic\"\n", + ) + .unwrap(); + let cfg = load(Some(tmp.path())); + assert_eq!(cfg.ui.collapse, CollapseMode::Normal); + } +} diff --git a/vw-repl/src/diag_search.rs b/vw-repl/src/diag_search.rs new file mode 100644 index 0000000..49ab109 --- /dev/null +++ b/vw-repl/src/diag_search.rs @@ -0,0 +1,765 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Diagnostics fuzzy-finder — Ctrl-F opens a centered modal listing +//! every diagnostic entry (Error / Warning / Notice) currently in the +//! scrollback. Typing filters via [`nucleo_matcher`]; ↑/↓ navigates; +//! Enter closes the modal and jumps the scrollback viewport to the +//! chosen entry, dropping a persistent left-gutter marker so the user +//! can spot it in a busy log. Alt-C clears the marker. +//! +//! Kind-filter checkboxes at the top let the user toggle inclusion of +//! each severity independently — Ctrl-E, Ctrl-W, Ctrl-N. Defaults +//! are Error+Warning on, Notice off (chatty INFO messages usually +//! aren't what someone reaches for `find diagnostic` to see). +//! +//! Modeled on [`crate::symbol_search`] — same nucleo-matcher engine, +//! same modal shape (70% × 75% of frame). + +use nucleo_matcher::pattern::{CaseMatching, Normalization, Pattern}; +use nucleo_matcher::{Config, Matcher, Utf32String}; +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, Scrollbar, + ScrollbarOrientation, ScrollbarState, +}; +use ratatui::Frame; + +use crate::app::{ScrollbackEntry, ScrollbackKind}; + +/// One scrollback entry surfaced in the picker. `scrollback_idx` is +/// stable relative to the App's `scrollback` Vec at open time — the +/// picker holds a snapshot, and while it's open new appends land at +/// higher indices without perturbing existing ones (scrollback is +/// append-only), so accepting an item still jumps to the right row. +#[derive(Clone, Debug)] +pub struct DiagItem { + pub scrollback_idx: usize, + pub kind: ScrollbackKind, + /// True when the source entry was a Vivado CRITICAL WARNING + /// (kind is still `Error` — CW and plain Error share the same + /// scrollback visual bucket). The picker uses this to answer + /// the `Critical` filter checkbox; a critical entry also + /// passes the `Error` filter, so turning on both is fine and + /// won't duplicate rows. + pub is_critical_warning: bool, + /// First non-empty line of the entry — what shows in the result + /// list. Truncated to a sensible width by the renderer. + pub preview: String, + /// Full text used as the fuzzy-match haystack. Multi-line + /// diagnostics (WARNING body + attached stack) match on any + /// content, so a query for a function name in the trace still + /// finds the top-level warning. + pub full: String, + /// Scrollback index of the [`ScrollbackKind::Input`] entry + /// this diagnostic was emitted under, or `None` when the + /// diagnostic predates any input (startup notices, e.g. + /// `vivado ready`). Used by the picker to group results by + /// the command that produced them. + pub parent_input_idx: Option, + /// Single-line preview of the parent input's text — + /// captured at snapshot time so the picker's header rows + /// don't need to walk back into the App's scrollback. `None` + /// when `parent_input_idx` is `None`. + pub parent_preview: Option, +} + +/// Scored entry after applying the current query + kind filters. +/// Sorted by descending `score` at the end of `recompute`. +#[derive(Clone, Copy, Debug)] +pub struct Scored { + pub item_idx: usize, + pub score: u32, +} + +/// Diagnostic-picker overlay state. +#[derive(Debug)] +pub struct DiagnosticPicker { + /// Snapshot of Error/Warning/Notice entries from the scrollback, + /// taken at open time. Fixed for the picker's lifetime — new + /// scrollback appends aren't reflected until the user reopens. + pub items: Vec, + pub query: String, + pub results: Vec, + pub selected: usize, + /// Which kinds pass the filter row's checkboxes. + pub filter_error: bool, + pub filter_warning: bool, + pub filter_notice: bool, + /// Critical-warning subset filter. CW entries carry + /// `kind == Error` (shared visual bucket) — this checkbox + /// lets the user surface JUST the criticals within that + /// bucket. An item passes when either its kind matches an + /// enabled kind filter OR its CW flag matches `filter_critical`. + pub filter_critical: bool, +} + +impl DiagnosticPicker { + /// Build the picker from an App scrollback slice. Filters the + /// slice down to diagnostic-kind entries; entries with empty + /// text are dropped (nothing to preview or match against). + pub fn from_scrollback(scrollback: &[ScrollbackEntry]) -> Self { + let items: Vec = scrollback + .iter() + .enumerate() + .filter(|(_, e)| is_diagnostic(e.kind)) + .filter_map(|(idx, e)| { + let preview = e.text.lines().next().unwrap_or("").to_string(); + if preview.is_empty() && e.text.is_empty() { + return None; + } + let parent_preview = e.parent_input_idx.and_then(|pidx| { + scrollback.get(pidx).map(|p| { + p.text.lines().next().unwrap_or("").to_string() + }) + }); + Some(DiagItem { + scrollback_idx: idx, + kind: e.kind, + is_critical_warning: e.is_critical_warning, + preview, + full: e.text.clone(), + parent_input_idx: e.parent_input_idx, + parent_preview, + }) + }) + .collect(); + let mut p = Self { + items, + query: String::new(), + results: Vec::new(), + selected: 0, + // Defaults: Error+Warning+Critical on, Notice off. + // Notice buckets INFO-severity Vivado messages, which + // are chatty and usually not what someone reaches for + // a diagnostics finder to see — but it's one keystroke + // (Ctrl-N) away. Critical is on by default because + // it's a subset of Error, so having it enabled changes + // nothing until the user turns Error off to isolate + // criticals. + filter_error: true, + filter_warning: true, + filter_notice: false, + filter_critical: true, + }; + p.recompute(); + p + } + + pub fn move_up(&mut self) { + if self.selected > 0 { + self.selected -= 1; + } + } + + pub fn move_down(&mut self) { + if self.selected + 1 < self.results.len() { + self.selected += 1; + } + } + + pub fn push_char(&mut self, c: char) { + self.query.push(c); + self.recompute(); + } + + pub fn pop_char(&mut self) { + self.query.pop(); + self.recompute(); + } + + /// Toggle inclusion of a specific kind. Ignores kinds outside + /// the picker's diagnostic set — passing `Chatter` is a no-op. + pub fn toggle_kind(&mut self, kind: ScrollbackKind) { + match kind { + ScrollbackKind::Error => self.filter_error = !self.filter_error, + ScrollbackKind::Warning => { + self.filter_warning = !self.filter_warning; + } + ScrollbackKind::Notice => self.filter_notice = !self.filter_notice, + _ => return, + } + self.recompute(); + } + + /// Toggle the Critical-warning subset filter. Independent of + /// the kind filters — with `filter_critical` on and every + /// kind off, the picker shows just critical warnings. + pub fn toggle_critical(&mut self) { + self.filter_critical = !self.filter_critical; + self.recompute(); + } + + /// Currently-selected item, if any. Returns the item struct + /// (contains `scrollback_idx` and `kind`) rather than just the + /// index so callers can pass kind into the marker-styling code + /// without a second lookup. + pub fn current(&self) -> Option<&DiagItem> { + let scored = self.results.get(self.selected)?; + self.items.get(scored.item_idx) + } + + /// Whether `item` passes the current filter row. An entry + /// passes when EITHER its kind's checkbox is on OR (for + /// critical warnings) the Critical checkbox is on — so a CW + /// entry surfaces under Error, under Critical, or both, + /// without appearing twice. + fn item_allowed(&self, item: &DiagItem) -> bool { + let kind_pass = match item.kind { + ScrollbackKind::Error => self.filter_error, + ScrollbackKind::Warning => self.filter_warning, + ScrollbackKind::Notice => self.filter_notice, + _ => false, + }; + let critical_pass = item.is_critical_warning && self.filter_critical; + kind_pass || critical_pass + } + + /// Rebuild `results` from current query + filter state. O(N) + /// over the snapshot. Called from `new`, `push_char`, + /// `pop_char`, `toggle_kind`, `toggle_critical`. + fn recompute(&mut self) { + let allowed: Vec = self + .items + .iter() + .enumerate() + .filter(|(_, it)| self.item_allowed(it)) + .map(|(i, _)| i) + .collect(); + + if self.query.is_empty() { + // No query: show all kind-allowed items in scrollback + // order (== descending scrollback_idx would be reverse- + // chronological; ascending == chronological, which is + // what appears in the log itself — pick the latter so + // the picker order matches what the user's eye scanned). + self.results = allowed + .into_iter() + .map(|item_idx| Scored { item_idx, score: 0 }) + .collect(); + self.clamp_selected(); + return; + } + + let mut matcher = Matcher::new(Config::DEFAULT); + let pattern = Pattern::parse( + &self.query, + CaseMatching::Smart, + Normalization::Smart, + ); + let mut scored: Vec = allowed + .iter() + .filter_map(|&item_idx| { + let it = &self.items[item_idx]; + let hay = Utf32String::from(it.full.as_str()); + let score = pattern.score(hay.slice(..), &mut matcher)?; + if score == 0 { + None + } else { + Some(Scored { item_idx, score }) + } + }) + .collect(); + scored.sort_by_key(|s| std::cmp::Reverse(s.score)); + scored.truncate(500); + self.results = scored; + self.clamp_selected(); + } + + fn clamp_selected(&mut self) { + if self.selected >= self.results.len() { + self.selected = self.results.len().saturating_sub(1); + } + } +} + +/// True when `kind` is one of the diagnostic kinds the picker +/// surfaces. Broken out so `from_scrollback` and `kind_allowed` +/// stay in sync — adding a new diagnostic kind means updating +/// this predicate alone. +fn is_diagnostic(kind: ScrollbackKind) -> bool { + matches!( + kind, + ScrollbackKind::Error + | ScrollbackKind::Warning + | ScrollbackKind::Notice + ) +} + +/// Render the diagnostic picker as a centered modal. Sized to +/// match [`crate::symbol_search::draw_symbol_picker`] so the two +/// finders feel like siblings. +pub fn draw_diagnostic_picker(f: &mut Frame, picker: &DiagnosticPicker) { + let frame = f.area(); + let width = (frame.width as f32 * 0.7) as u16; + let height = (frame.height as f32 * 0.75) as u16; + let x = frame.x + (frame.width.saturating_sub(width)) / 2; + let y = frame.y + (frame.height.saturating_sub(height)) / 2; + let area = Rect { + x, + y, + width: width.min(frame.width), + height: height.min(frame.height), + }; + + f.render_widget(Clear, area); + // Title shows a live match / snapshot-size count so the user + // knows how selective their filters + query are. `snapshot` + // is the total diagnostics captured when the picker opened; + // `matches` is the count after filter+query. When the two + // are equal the count reads as e.g. `(12/12)` = "everything + // in scrollback passes". + let title = format!( + " find diagnostic ({}/{}) — Esc to close ", + picker.results.len(), + picker.items.len(), + ); + let block = Block::default().borders(Borders::ALL).title(title); + let inner = block.inner(area); + f.render_widget(block, area); + if inner.width == 0 || inner.height < 4 { + return; + } + + // Vertical layout: filter row (1) + query row (1) + separator + // (1) + result list (remaining). + let filter_row = Rect { + x: inner.x, + y: inner.y, + width: inner.width, + height: 1, + }; + let query_row = Rect { + x: inner.x, + y: inner.y + 1, + width: inner.width, + height: 1, + }; + let sep_row = Rect { + x: inner.x, + y: inner.y + 2, + width: inner.width, + height: 1, + }; + let list_area = Rect { + x: inner.x, + y: inner.y + 3, + width: inner.width, + height: inner.height.saturating_sub(3), + }; + + draw_filter_row(f, picker, filter_row); + draw_query_row(f, picker, query_row); + f.render_widget( + Paragraph::new(Line::from(Span::styled( + "─".repeat(inner.width as usize), + Style::default().add_modifier(Modifier::DIM), + ))), + sep_row, + ); + draw_result_list(f, picker, list_area); +} + +fn draw_filter_row(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + let cell = |on: bool, glyph: &str, label: &str, key: &str, color: Color| { + let box_style = if on { + Style::default().fg(color).add_modifier(Modifier::BOLD) + } else { + Style::default().add_modifier(Modifier::DIM) + }; + vec![ + Span::styled(glyph.to_string(), box_style), + Span::raw(" "), + Span::styled(label.to_string(), box_style), + Span::raw(" "), + Span::styled( + format!("({key})"), + Style::default().add_modifier(Modifier::DIM), + ), + Span::raw(" "), + ] + }; + let mut spans: Vec> = Vec::new(); + spans.push(Span::styled( + " filter: ".to_string(), + Style::default().add_modifier(Modifier::DIM), + )); + spans.extend(cell( + picker.filter_error, + if picker.filter_error { "[x]" } else { "[ ]" }, + "Error", + "^E", + Color::Red, + )); + spans.extend(cell( + picker.filter_critical, + if picker.filter_critical { "[x]" } else { "[ ]" }, + "Critical", + "^K", + Color::Rgb(255, 90, 90), + )); + spans.extend(cell( + picker.filter_warning, + if picker.filter_warning { "[x]" } else { "[ ]" }, + "Warning", + "^W", + Color::Rgb(255, 140, 0), + )); + spans.extend(cell( + picker.filter_notice, + if picker.filter_notice { "[x]" } else { "[ ]" }, + "Info", + "^N", + Color::Gray, + )); + f.render_widget(Paragraph::new(Line::from(spans)), area); +} + +fn draw_query_row(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + let spans = vec![ + Span::styled( + " › ".to_string(), + Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD), + ), + Span::styled(picker.query.clone(), Style::default().fg(Color::White)), + // Cursor position — a solid block on the end of the query + // makes the input row read as an editable field even + // without a real terminal cursor there (ratatui's cursor + // is bound to the input editor below the popup). + Span::styled( + "▏".to_string(), + Style::default().fg(Color::Rgb(180, 130, 220)), + ), + ]; + f.render_widget(Paragraph::new(Line::from(spans)), area); +} + +fn draw_result_list(f: &mut Frame, picker: &DiagnosticPicker, area: Rect) { + if picker.results.is_empty() { + let msg = if picker.items.is_empty() { + "no diagnostics in scrollback" + } else if picker.query.is_empty() { + "no diagnostics match current filters" + } else { + "no diagnostics match query" + }; + f.render_widget( + Paragraph::new(Line::from(Span::styled( + format!(" {msg}"), + Style::default().add_modifier(Modifier::DIM), + ))), + area, + ); + return; + } + + // Reserve the rightmost column for a scrollbar when the + // result list overflows the visible area. Matches the main + // scrollback pane's auto-hide behavior — no wasted column + // when everything fits. + let needs_scrollbar = picker.results.len() > area.height as usize; + let (list_area, scrollbar_area) = if needs_scrollbar && area.width >= 2 { + let narrower = Rect { + width: area.width - 1, + ..area + }; + (narrower, Some(area)) + } else { + (area, None) + }; + + let max_preview_width = (list_area.width as usize) + .saturating_sub(6 /* kind badge + space */); + + // Results grouped by parent input. Walk results in order, + // emit a header row whenever the parent_input_idx changes, + // then emit the item row itself. Track selected_visual_row + // so ratatui's list highlight lands on the actual item (not + // a header) even though `picker.selected` indexes into + // `picker.results`. + let mut items: Vec = Vec::new(); + let mut selected_visual_row: usize = 0; + let mut prev_parent: Option> = None; + for (row_idx, s) in picker.results.iter().enumerate() { + let Some(it) = picker.items.get(s.item_idx) else { + continue; + }; + // Group header — emitted when the parent changes vs the + // previous item. `Some(None)` (rendered as "before any + // command") differs from an actual command's group, so + // we key on the Option> pair. + if prev_parent != Some(it.parent_input_idx) { + let header_text = match (&it.parent_preview, it.parent_input_idx) { + (Some(preview), _) => preview.clone(), + (None, _) => "(before any command)".to_string(), + }; + items.push(ListItem::new(Line::from(vec![ + Span::styled( + "▼ ".to_string(), + Style::default().fg(Color::Gray), + ), + Span::styled( + header_text, + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ), + ]))); + prev_parent = Some(it.parent_input_idx); + } + // Item row. Indented 2 cells to visually nest under the + // group header. + let (badge_str, badge_style) = match it.kind { + ScrollbackKind::Error if it.is_critical_warning => ( + "C ", + Style::default() + .fg(Color::Rgb(255, 90, 90)) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Error => ( + "E ", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Warning => ( + "W ", + Style::default() + .fg(Color::Rgb(255, 140, 0)) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Notice => ( + "i ", + Style::default() + .fg(Color::Gray) + .add_modifier(Modifier::BOLD), + ), + _ => ("? ", Style::default()), + }; + // 2-cell indent for nesting under the group header; + // room in the preview budget shrinks accordingly. + let preview_budget = max_preview_width.saturating_sub(2); + let preview = truncate_chars(&it.preview, preview_budget); + let is_selected = row_idx == picker.selected; + let preview_style = if is_selected { + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(Color::Gray) + }; + let row_style = if is_selected { + Style::default().bg(Color::Rgb(40, 40, 60)) + } else { + Style::default() + }; + if is_selected { + selected_visual_row = items.len(); + } + let spans = vec![ + Span::raw(" "), + Span::styled(badge_str.to_string(), badge_style), + Span::styled(preview, preview_style), + ]; + items.push(ListItem::new(Line::from(spans)).style(row_style)); + } + + let mut list_state = ListState::default(); + list_state.select(Some(selected_visual_row)); + let list = List::new(items) + .highlight_style(Style::default().bg(Color::Rgb(40, 40, 60))); + f.render_stateful_widget(list, list_area, &mut list_state); + + // Scrollbar reflects the selection's position within the + // full result set (not just the visible viewport) — that's + // what the user cares about when they've paged Down deep + // into hits. + if let Some(sb_area) = scrollbar_area { + let mut sb_state = ScrollbarState::new(picker.results.len()) + .position(picker.selected) + .viewport_content_length(list_area.height as usize); + let scrollbar = Scrollbar::default() + .orientation(ScrollbarOrientation::VerticalRight); + f.render_stateful_widget(scrollbar, sb_area, &mut sb_state); + } +} + +/// Truncate `s` to `max_chars` display cells, appending `…` when +/// clipped. Character-count based (not byte-based) so multi-byte +/// content — non-ASCII proc names, stack-frame arrows — clip +/// cleanly. +fn truncate_chars(s: &str, max_chars: usize) -> String { + if max_chars < 2 { + return String::new(); + } + let count = s.chars().count(); + if count <= max_chars { + return s.to_string(); + } + let take = max_chars.saturating_sub(1); + let mut out: String = s.chars().take(take).collect(); + out.push('…'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(kind: ScrollbackKind, text: &str) -> ScrollbackEntry { + ScrollbackEntry { + kind, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: false, + parent_input_idx: None, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + } + } + + fn critical_entry(text: &str) -> ScrollbackEntry { + ScrollbackEntry { + kind: ScrollbackKind::Error, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: true, + parent_input_idx: None, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + } + } + + #[test] + fn snapshot_filters_to_diagnostics() { + let sb = vec![ + entry(ScrollbackKind::Input, "some input"), + entry(ScrollbackKind::Error, "unresolved symbol foo"), + entry(ScrollbackKind::Stdout, "hello"), + entry(ScrollbackKind::Warning, "deprecated call"), + entry(ScrollbackKind::Chatter, "banner"), + entry(ScrollbackKind::Notice, "vivado: ready"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.items.len(), 3); + // scrollback indices preserved + assert_eq!(picker.items[0].scrollback_idx, 1); + assert_eq!(picker.items[1].scrollback_idx, 3); + assert_eq!(picker.items[2].scrollback_idx, 5); + } + + #[test] + fn default_filter_hides_notice() { + let sb = vec![ + entry(ScrollbackKind::Error, "e1"), + entry(ScrollbackKind::Warning, "w1"), + entry(ScrollbackKind::Notice, "n1"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + // 3 items in the snapshot but only 2 pass default filters. + assert_eq!(picker.items.len(), 3); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn toggle_notice_shows_it() { + let sb = vec![ + entry(ScrollbackKind::Error, "e1"), + entry(ScrollbackKind::Notice, "n1"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.results.len(), 1); + picker.toggle_kind(ScrollbackKind::Notice); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn critical_shows_under_error_and_critical() { + let sb = vec![ + entry(ScrollbackKind::Error, "plain error"), + critical_entry("critical warning body"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + // Both Error+Critical filters on by default → both entries. + assert_eq!(picker.results.len(), 2); + // Error off, Critical on → just the CW (it still passes + // via the critical predicate even though its kind is + // Error). + picker.toggle_kind(ScrollbackKind::Error); + assert_eq!(picker.results.len(), 1); + assert!(picker.current().unwrap().is_critical_warning); + // Critical off too → nothing (both filter paths off). + picker.toggle_critical(); + assert_eq!(picker.results.len(), 0); + // Error back on, Critical still off → BOTH entries show: + // CW passes via its kind (Error) even without the + // critical checkbox. This is the "if they show up when + // error is selected that's fine" behavior — the Critical + // filter is additive, not exclusive. + picker.toggle_kind(ScrollbackKind::Error); + assert_eq!(picker.results.len(), 2); + } + + #[test] + fn fuzzy_query_narrows() { + let sb = vec![ + entry(ScrollbackKind::Error, "unresolved symbol foo"), + entry(ScrollbackKind::Error, "cannot open file bar"), + entry(ScrollbackKind::Warning, "deprecated call foo"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.results.len(), 3); + picker.push_char('f'); + picker.push_char('o'); + picker.push_char('o'); + // Both `foo`-mentioning entries survive; the `bar` one doesn't. + assert_eq!(picker.results.len(), 2); + for r in &picker.results { + assert!(picker.items[r.item_idx].full.contains("foo")); + } + } + + #[test] + fn move_updown_clamps() { + let sb = vec![ + entry(ScrollbackKind::Error, "a"), + entry(ScrollbackKind::Error, "b"), + ]; + let mut picker = DiagnosticPicker::from_scrollback(&sb); + assert_eq!(picker.selected, 0); + picker.move_up(); + assert_eq!(picker.selected, 0); + picker.move_down(); + assert_eq!(picker.selected, 1); + picker.move_down(); + assert_eq!(picker.selected, 1); + } + + #[test] + fn current_maps_to_scrollback_idx() { + let sb = vec![ + entry(ScrollbackKind::Input, "ignored"), + entry(ScrollbackKind::Warning, "hit"), + ]; + let picker = DiagnosticPicker::from_scrollback(&sb); + let current = picker.current().unwrap(); + assert_eq!(current.scrollback_idx, 1); + assert_eq!(current.kind, ScrollbackKind::Warning); + } + + #[test] + fn empty_scrollback_no_panic() { + let picker = DiagnosticPicker::from_scrollback(&[]); + assert_eq!(picker.results.len(), 0); + assert!(picker.current().is_none()); + } +} diff --git a/vw-repl/src/highlight.rs b/vw-repl/src/highlight.rs new file mode 100644 index 0000000..01a5c00 --- /dev/null +++ b/vw-repl/src/highlight.rs @@ -0,0 +1,525 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Syntax highlighter for compiler-emitted enum reprs. +//! +//! Reprs follow a uniform shape regardless of which enum produced +//! them (the compiler emits `Variant`, `Variant(payload)`, or +//! `Variant(\n inner\n)` for any user-declared enum, and +//! dict/list reprs join entries with `\n`). This module recognizes +//! that shape line-by-line and emits styled +//! [`ratatui::text::Span`]s — keys in blue, variant names in teal, +//! punctuation in dim, scalar payloads in green. +//! +//! Shape-based, not name-based: the highlighter has no knowledge +//! of `Property` / `Properties` / any specific enum. It recognizes +//! the structural pattern (`IDENT '(' … ')'` for variant calls, +//! `KEY SP VARIANT …` for dict entries, bare `)` for multi-line +//! close), so adding a new enum to the htcl source automatically +//! gets the same highlighting on its repr output. +//! +//! Falls back to plain text when a line doesn't parse — non-repr +//! content (raw `puts` output, error messages, etc.) renders +//! normally. +//! +//! Color palette is exported so [`crate::render::entry_lines`] +//! can apply the same fallback `body_style` for unparsed runs. + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::Span; +use winnow::ascii::space0; +use winnow::combinator::repeat; +use winnow::error::ContextError; +use winnow::token::take_while; +use winnow::{ModalResult, Parser}; + +/// Semantic role of a highlighted piece of text. Backend-agnostic: +/// callers (the REPL's ratatui renderer, `vw run`'s ANSI stdout +/// renderer, or anything else consuming +/// [`highlight_line_pieces`]) map each variant to whatever styling +/// primitives their target supports. The RGB values in +/// [`key_style`] / [`variant_style`] / [`scalar_style`] and the +/// dim modifier in [`punct_style`] are the CANONICAL palette; every +/// backend should reproduce these colors so the two rendering +/// surfaces look identical. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StyleKind { + /// Uncolored spacing (indent, inter-token whitespace, + /// trailing runs). Backends emit the text verbatim. + Plain, + /// Dict key (`CONFIG`, `CPM_PCIE0_MODES`, …). + Key, + /// Enum variant name (`Scalar`, `Nested`, …). + Variant, + /// Structural punctuation (`(` / `)`). + Punct, + /// Scalar payload (the string inside `Scalar(…)`). + Scalar, +} + +/// One styled fragment of a highlighted repr line. +#[derive(Debug, Clone)] +pub struct Piece { + pub text: String, + pub kind: StyleKind, +} + +impl Piece { + fn plain(text: impl Into) -> Self { + Self { + text: text.into(), + kind: StyleKind::Plain, + } + } + fn styled(text: impl Into, kind: StyleKind) -> Self { + Self { + text: text.into(), + kind, + } + } +} + +/// Style for dict keys (`CONFIG`, `CPM_PCIE0_MODES`, …) — the +/// identifier immediately preceding a value. +pub fn key_style() -> Style { + Style::default().fg(Color::Rgb(80, 150, 255)) +} + +/// Style for enum variant names (`Scalar`, `Nested`, …) — the +/// identifier immediately preceding `(`. +pub fn variant_style() -> Style { + Style::default().fg(Color::Rgb(100, 200, 200)) +} + +/// Style for structural punctuation (`(` and `)`) — dimmed so the +/// nesting structure recedes visually next to keys and values. +pub fn punct_style() -> Style { + Style::default().add_modifier(Modifier::DIM) +} + +/// Style for scalar payloads — the string inside `Scalar(…)`. +pub fn scalar_style() -> Style { + Style::default().fg(Color::Rgb(120, 200, 120)) +} + +/// Convert a [`StyleKind`] to the ratatui [`Style`] the REPL's +/// scrollback uses. Kept as a single map so the REPL and any other +/// ratatui-based consumer stay in sync with the canonical palette. +pub fn ratatui_style(kind: StyleKind) -> Style { + match kind { + StyleKind::Plain => Style::default(), + StyleKind::Key => key_style(), + StyleKind::Variant => variant_style(), + StyleKind::Punct => punct_style(), + StyleKind::Scalar => scalar_style(), + } +} + +/// Try to recognize `line` as a compiler-emitted enum-repr line +/// and return a backend-agnostic piece sequence. Returns `None` +/// when the line doesn't match the repr grammar — callers fall +/// back to rendering the raw text with their default body style. +/// +/// Callers targeting a ratatui surface (the REPL scrollback) can +/// use [`highlight_line`] for a Span-typed shortcut; callers +/// targeting a plain terminal (`vw run`'s stdout stream) consume +/// pieces directly and produce ANSI escapes. +pub fn highlight_line_pieces(line: &str) -> Option> { + let mut input = line; + parse_line.parse_next(&mut input).ok().filter(|pieces| { + // Reject parses that didn't consume the whole line — a + // partial match means we'd silently style some tokens + // and drop the rest. Better to fall through to plain. + input.is_empty() && !pieces.is_empty() + }) +} + +/// Ratatui-flavored wrapper around [`highlight_line_pieces`]. The +/// REPL's scrollback renderer consumes ratatui `Span`s directly. +pub fn highlight_line(line: &str) -> Option>> { + highlight_line_pieces(line).map(|pieces| { + pieces + .into_iter() + .map(|p| match p.kind { + StyleKind::Plain => Span::raw(p.text), + _ => Span::styled(p.text, ratatui_style(p.kind)), + }) + .collect() + }) +} + +// Top-level line shapes: +// `INDENT? ')' [SP] EOL` — multi-line close +// `INDENT? KEY SP VALUE [SP]? EOL` — dict entry +fn parse_line(input: &mut &str) -> ModalResult> { + let mut pieces: Vec = Vec::new(); + let indent = space0::<_, ContextError>.parse_next(input)?; + if !indent.is_empty() { + pieces.push(Piece::plain(indent)); + } + // Multi-line-close line: bare `)`, optionally followed by trailing whitespace. + if input.starts_with(')') { + let close = ")"; + *input = &input[1..]; + pieces.push(Piece::styled(close, StyleKind::Punct)); + let trailing = space0::<_, ContextError>.parse_next(input)?; + if !trailing.is_empty() { + pieces.push(Piece::plain(trailing)); + } + return Ok(pieces); + } + // Dict-entry line: KEY SP VALUE + let key = parse_dict_key(input)?; + pieces.push(Piece::styled(key, StyleKind::Key)); + let sp = take_while(1.., |c: char| c == ' ').parse_next(input)?; + pieces.push(Piece::plain(sp)); + // Top-level: require the value to use parens to avoid false- + // positives on plain prose `puts "Word Other"` Stdout lines. + let value_pieces = parse_value(input, true)?; + pieces.extend(value_pieces); + Ok(pieces) +} + +// VALUE is one of: +// IDENT '(' INNER ')' — single-line variant call with payload +// IDENT '(' — multi-line open (line ends after `(`) +// IDENT — empty-payload variant +// +// `require_parens = true` rejects the bare-IDENT form. We use that +// at the TOP LEVEL of a Stdout/Result line, where "KEY VARIANT" +// without parens is far more often plain prose (e.g. +// `puts "Configuring CIPS"` → `Configuring CIPS`) than an actual +// repr — styling that prose as if it were a typed value produces +// distracting false-positives. Inside an inline payload (sub-entries +// like `Nested(K1 Empty K2 Other(x))`) we accept bare IDENT because +// we've already seen the surrounding `Nested(`, so the context is +// unambiguous. +fn parse_value( + input: &mut &str, + require_parens: bool, +) -> ModalResult> { + let variant = parse_ident(input)?; + let mut out = vec![Piece::styled(variant, StyleKind::Variant)]; + if !input.starts_with('(') { + if require_parens { + return Err(winnow::error::ErrMode::Backtrack(ContextError::new())); + } + return Ok(out); + } + *input = &input[1..]; + out.push(Piece::styled("(", StyleKind::Punct)); + if input.is_empty() { + // `Variant(` at end of line — multi-line open. The + // closing `)` will appear on a later line and be matched + // by the close-only branch in `parse_line`. + return Ok(out); + } + // Inline payload. The payload is everything up to the + // matching close paren, with `(`/`)` balanced. Could be: + // - a scalar string (no inner parens): color green + // - a sub-entry KEY VARIANT(...) [SP KEY VARIANT(...)]*: recurse + let payload_pieces = parse_inline_payload(input)?; + out.extend(payload_pieces); + if input.starts_with(')') { + *input = &input[1..]; + out.push(Piece::styled(")", StyleKind::Punct)); + } + Ok(out) +} + +// Inline payload between `(` and its matching `)`. Recognizes +// either a single scalar (text with no parens) or a sequence of +// inline dict-entry-shaped sub-values (`KEY VARIANT(...) ...`). +// Stops at the closing `)` of the surrounding call. +fn parse_inline_payload(input: &mut &str) -> ModalResult> { + // Look ahead: does the payload look like `IDENT SP IDENT (`? + // If so it's a sub-entry — recurse. Otherwise treat it as a + // scalar value. + if looks_like_sub_entry(input) { + let mut out = Vec::new(); + // First sub-entry. + let entry = parse_sub_entry(input)?; + out.extend(entry); + // Optional further sub-entries separated by space (for + // dicts with multiple inline children). + let more: Vec> = repeat( + 0.., + ( + take_while(1.., |c: char| c == ' ') + .map(|s: &str| Piece::plain(s)), + parse_sub_entry, + ) + .map(|(sp, mut e)| { + e.insert(0, sp); + e + }), + ) + .parse_next(input)?; + for chunk in more { + out.extend(chunk); + } + Ok(out) + } else { + // Scalar payload: take everything up to the matching close + // paren of the surrounding call. Track paren depth so + // scalar values containing their own `(…)` (e.g. Vivado's + // `RS(544) CL119` FEC-config string) don't get cut short at + // the FIRST `)` — that used to abort the parse, drop the + // trailing text onto the caller's leftover input, and + // fall the whole line back to gray. + let start = *input; + let mut depth: usize = 0; + let mut end = 0; + for (i, c) in start.char_indices() { + match c { + '(' => depth += 1, + ')' if depth == 0 => { + end = i; + break; + } + ')' => depth -= 1, + _ => {} + } + end = i + c.len_utf8(); + } + let scalar = &start[..end]; + *input = &start[end..]; + Ok(vec![Piece::styled(scalar, StyleKind::Scalar)]) + } +} + +// A sub-entry inside an inline payload: KEY SP VARIANT [( ... )]. +fn parse_sub_entry(input: &mut &str) -> ModalResult> { + let mut out = Vec::new(); + let key = parse_ident(input)?; + out.push(Piece::styled(key, StyleKind::Key)); + let sp = take_while(1.., |c: char| c == ' ').parse_next(input)?; + out.push(Piece::plain(sp)); + // Sub-entry: bare-ident variant allowed (we're already inside + // an established `Variant(...)` so the context is unambiguous). + let value_pieces = parse_value(input, false)?; + out.extend(value_pieces); + Ok(out) +} + +// Best-effort lookahead: does the input start with `IDENT SP IDENT` +// (which would indicate a KEY SP VARIANT sub-entry rather than a +// bare scalar payload)? Doesn't consume input. +fn looks_like_sub_entry(input: &&str) -> bool { + let s = input; + let mut it = s.chars(); + // First ident + let first_ok = matches!( + it.next(), + Some(c) if c.is_ascii_alphabetic() || c == '_', + ); + if !first_ok { + return false; + } + let mut saw_first = 1; + for c in it.by_ref() { + if c.is_ascii_alphanumeric() || c == '_' { + saw_first += 1; + } else if c == ' ' { + break; + } else { + return false; + } + } + if saw_first == 0 { + return false; + } + // Next must be IDENT (after the space we just consumed). + let mut second_count = 0; + for c in it { + if c.is_ascii_alphanumeric() || c == '_' { + second_count += 1; + } else { + // A sub-entry's second ident is the variant name — must + // be followed by `(` to count. + return second_count > 0 && c == '('; + } + } + false +} + +// `[A-Za-z_][A-Za-z0-9_]*` — take identifier-shaped chars then +// verify the leading character is letter/underscore (we can't +// match the leading-letter constraint and the run cleanly with a +// single `take_while`, but it's fine to take everything plausible +// then reject if the leading char would have made it digit-led). +fn parse_ident<'a>(input: &mut &'a str) -> ModalResult<&'a str> { + let ident = + take_while(1.., |c: char| c.is_ascii_alphanumeric() || c == '_') + .parse_next(input)?; + match ident.chars().next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => Ok(ident), + _ => Err(winnow::error::ErrMode::Backtrack(ContextError::new())), + } +} + +/// Same as [`parse_ident`] but also accepts `.` in the middle — +/// used for dict KEYS, not variant names. Vivado's property keys +/// are `CONFIG.` style (a dot-composed namespace), so a +/// `::Config`-style repr line like `CONFIG.CPM_PCIE0_MODES +/// Scalar(None)` needs to accept the `.` as part of the key or +/// the whole line fails to parse and falls back to plain rendering +/// — which is what "the highlighter isn't working" looked like in +/// practice. Variant names (`Scalar`, `Nested`, `Empty`, …) still +/// use [`parse_ident`] so this stays confined to KEYS only. +fn parse_dict_key<'a>(input: &mut &'a str) -> ModalResult<&'a str> { + let ident = take_while(1.., |c: char| { + c.is_ascii_alphanumeric() || c == '_' || c == '.' + }) + .parse_next(input)?; + match ident.chars().next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => { + // Reject leading dot / trailing dot / consecutive dots + // shape — those are structurally malformed keys, and + // accepting them would silently paint bogus prose. + if ident.starts_with('.') + || ident.ends_with('.') + || ident.contains("..") + { + Err(winnow::error::ErrMode::Backtrack(ContextError::new())) + } else { + Ok(ident) + } + } + _ => Err(winnow::error::ErrMode::Backtrack(ContextError::new())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn close_only_line() { + let spans = highlight_line(")").expect("parses"); + assert!(!spans.is_empty()); + // Last span content is ")" + let last = &spans[spans.len() - 1]; + assert_eq!(last.content.as_ref(), ")"); + } + + #[test] + fn indented_close_line() { + let spans = highlight_line(" )").expect("parses"); + // First span = " " (indent), last span = ")" + assert_eq!(spans[0].content.as_ref(), " "); + assert_eq!(spans.last().unwrap().content.as_ref(), ")"); + } + + #[test] + fn simple_scalar_entry() { + let spans = highlight_line("CONFIG Scalar(foo)").expect("parses"); + // Should have spans for CONFIG, " ", Scalar, "(", foo, ")" + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"CONFIG"), + "missing CONFIG span: {contents:?}" + ); + assert!( + contents.contains(&"Scalar"), + "missing Scalar span: {contents:?}" + ); + assert!(contents.contains(&"foo"), "missing foo span: {contents:?}"); + } + + #[test] + fn variant_open_multiline() { + let spans = highlight_line("CONFIG Nested(").expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!(contents.contains(&"CONFIG")); + assert!(contents.contains(&"Nested")); + assert_eq!(spans.last().unwrap().content.as_ref(), "("); + } + + /// The `::Config::repr` shape emits `CONFIG. + /// Scalar(value)` — a dot-composed Vivado property key. The + /// key parser has to accept dots or the whole line falls + /// through to unstyled plain text. + #[test] + fn dotted_config_key_parses() { + let spans = highlight_line("CONFIG.CPM_PCIE0_MODES Scalar(None)") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"CONFIG.CPM_PCIE0_MODES"), + "dotted key not captured whole: {contents:?}" + ); + assert!(contents.contains(&"Scalar"), "{contents:?}"); + assert!(contents.contains(&"None"), "{contents:?}"); + } + + /// Consecutive-dot / leading-dot / trailing-dot keys are + /// rejected so genuine prose (`. Alignment ...`) doesn't + /// silently get repainted as a repr. + #[test] + fn malformed_dotted_key_rejected() { + assert!(highlight_line(".leading Scalar(x)").is_none()); + assert!(highlight_line("trailing. Scalar(x)").is_none()); + assert!(highlight_line("dou..ble Scalar(x)").is_none()); + } + + #[test] + fn nested_inline_entry() { + let spans = + highlight_line("CONFIG Nested(CPM_PCIE0_MODES Scalar(None))") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!(contents.contains(&"CONFIG")); + assert!(contents.contains(&"Nested")); + assert!(contents.contains(&"CPM_PCIE0_MODES")); + assert!(contents.contains(&"Scalar")); + assert!(contents.contains(&"None")); + } + + #[test] + fn non_repr_line_returns_none() { + assert!(highlight_line("INFO: vivado started").is_none()); + assert!(highlight_line("just some random text").is_none()); + assert!(highlight_line("").is_none()); + } + + /// Regression: `Scalar(RS(544) CL119)` — Vivado's FEC-config + /// property value contains its own `(…)` pair. The scalar + /// payload parser must balance parens and stop only at the + /// matching outer `)`, otherwise the whole line falls back to + /// the gray no-repr styling. + #[test] + fn scalar_payload_with_inner_parens_still_highlights() { + let spans = highlight_line("FEC_SLICE0_CFG_C0 Scalar(RS(544) CL119)") + .expect("parses"); + let contents: Vec<&str> = + spans.iter().map(|s| s.content.as_ref()).collect(); + assert!( + contents.contains(&"FEC_SLICE0_CFG_C0"), + "missing key: {contents:?}" + ); + assert!( + contents.contains(&"Scalar"), + "missing variant: {contents:?}" + ); + assert!( + contents.contains(&"RS(544) CL119"), + "missing scalar payload: {contents:?}" + ); + } + + #[test] + fn plain_two_word_prose_not_styled_as_repr() { + // Regression: stdout text like `puts "Configuring CIPS"` used + // to match the `KEY EmptyVariant` shape and get colored. The + // top-level parser must require parens to avoid this. + assert!(highlight_line("Configuring CIPS").is_none()); + assert!(highlight_line("CPM 5 USER PROPS").is_none()); + assert!(highlight_line("Hello World").is_none()); + } +} diff --git a/vw-repl/src/highlight_htcl.rs b/vw-repl/src/highlight_htcl.rs new file mode 100644 index 0000000..1bdf6f9 --- /dev/null +++ b/vw-repl/src/highlight_htcl.rs @@ -0,0 +1,1049 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Syntax highlighter for htcl source text. +//! +//! Single-pass character scanner with a small context-state machine. +//! Walks the source one byte at a time, recognizing comments, strings, +//! `$variable` refs, command substitutions, attributes, numeric +//! literals, and bare-word identifiers. The state machine tracks +//! "what was the previous token" so an identifier after `proc` is +//! styled as a declaration name, an identifier after `set` is styled +//! as a variable, etc. +//! +//! Why not the AST? The user-facing requirement is **stable** +//! highlighting — tokens keep the same color whether or not the +//! source as a whole parses. An AST-based highlighter has to +//! choose between (a) emitting nothing for unparseable regions +//! (flickers off mid-edit) or (b) merging with a separate lexical +//! pass that produces different output than the AST pass +//! (flickers between two styles when the parse status changes). +//! Neither is acceptable. +//! +//! A scanner-with-state produces the same output for `proc foo` +//! whether followed by `{x} unit { ... }` (complete) or by `{x` +//! (incomplete) — every recognizable byte run gets its consistent +//! token kind in one pass. Tree-sitter's grammar-with-error-recovery +//! gives the same property; a stateful scanner is the lighter-weight +//! analog for our small language. +//! +//! Token kinds: +//! +//! - **Keyword** — `proc`, `set`, `type`, `enum`, `src`, `namespace`, +//! plus the control-flow / Tcl-builtin set. +//! - **Function (builtin)** — `puts`, `lappend`, `dict`, etc. +//! - **Declaration** — the identifier immediately following a +//! declaration keyword (`proc foo`, `type T`, `enum E`). +//! - **Variable** — `$name`, `${name}`, and the identifier after +//! `set`. +//! - **Type** — the identifier following `:` (proc arg type +//! annotation), and the bare word after a proc decl's args +//! brace (the return-type slot). +//! - **Parameter** — bare identifiers inside the first `{...}` of +//! a proc decl (the args block). +//! - **Attribute** — `@name` on proc args. +//! - **String** — `"..."` quoted, and `{...}` braced when not a +//! script body. +//! - **Comment / DocComment** — `#` / `##` to end of line, only +//! when in command-start position. +//! - **Numeric** — integer literals. +//! - **Punctuation** — `[`, `]`, `(`, `)`, `:`. Brackets recurse: +//! the scanner re-enters at `[` with fresh state for the inner +//! command, then resumes the outer state after the matching `]`. + +use std::ops::Range; + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::Span as RatatuiSpan; + +/// Built-in command / control-flow words. +const BUILTIN_KEYWORDS: &[&str] = &[ + "if", "elseif", "else", "while", "for", "foreach", "switch", "catch", + "return", "break", "continue", "expr", "eval", "uplevel", "upvar", + "global", "variable", "try", "throw", "finally", +]; + +/// Built-in commands; styled distinct from user-proc calls. +/// `putr` is compile-time-rewritten to `puts [T::repr -v $x]` by +/// `vw_htcl::putr::rewrite` before any code reaches Tcl, but the +/// user writes it exactly like they write `puts` — same color for +/// visual consistency. +const BUILTIN_FUNCS: &[&str] = &[ + "puts", "putr", "lappend", "lindex", "llength", "lrange", "lsearch", + "lset", "lsort", "dict", "list", "string", "incr", "format", "scan", + "regexp", "regsub", "join", "split", "concat", "subst", "info", +]; + +/// Declaration keywords that name the next identifier as a decl. +const DECL_KEYWORDS: &[&str] = &["proc", "type", "enum", "namespace"]; + +/// One styled byte range. Used by both the scrollback renderer +/// (slice 2) and the input editor (slice 3) to apply per-token +/// styles on top of the entry's default body color. +#[derive(Clone, Debug, PartialEq)] +pub struct TokenSpan { + pub range: Range, + pub style: Style, +} + +// --- color palette -------------------------------------------------- + +pub fn keyword_style() -> Style { + Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD) +} +pub fn builtin_func_style() -> Style { + Style::default().fg(Color::Rgb(180, 130, 220)) +} +pub fn function_style() -> Style { + Style::default().fg(Color::Rgb(230, 200, 120)) +} +pub fn declaration_style() -> Style { + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD) +} +pub fn parameter_style() -> Style { + Style::default().fg(Color::Rgb(220, 220, 220)) +} +pub fn variable_style() -> Style { + Style::default().fg(Color::Rgb(130, 200, 230)) +} +pub fn string_style() -> Style { + Style::default().fg(Color::Rgb(140, 200, 130)) +} +pub fn comment_style() -> Style { + Style::default() + .fg(Color::Rgb(110, 110, 110)) + .add_modifier(Modifier::DIM) +} +pub fn doc_comment_style() -> Style { + Style::default() + .fg(Color::Rgb(130, 160, 170)) + .add_modifier(Modifier::ITALIC) +} +pub fn type_style() -> Style { + Style::default().fg(Color::Rgb(100, 200, 200)) +} +pub fn attribute_style() -> Style { + Style::default().fg(Color::Rgb(200, 130, 180)) +} +pub fn numeric_style() -> Style { + Style::default().fg(Color::Rgb(180, 200, 130)) +} +pub fn punct_style() -> Style { + Style::default().add_modifier(Modifier::DIM) +} + +// --- public entry points -------------------------------------------- + +/// Scan `source` and return a sorted, non-overlapping list of +/// [`TokenSpan`]s. Output is stable across edits — the same byte +/// sequence produces the same tokens regardless of whether the +/// surrounding source parses successfully. +pub fn highlight_source(source: &str) -> Vec { + let mut s = Scanner::new(source); + s.scan_script(usize::MAX); + s.out +} + +/// Slice `text` per-line using the same token classifier as +/// [`highlight_source`], returning one styled-`Span` Vec per line. +/// Lines are split on `\n`. Bytes outside any token use `body_style`. +pub fn highlight_per_line( + text: &str, + body_style: Style, +) -> Vec>> { + let tokens = highlight_source(text); + let mut out = Vec::new(); + let mut line_start: usize = 0; + for line in text.split('\n') { + let line_end = line_start + line.len(); + let mut spans: Vec> = Vec::new(); + let mut cursor = line_start; + for ts in tokens + .iter() + .filter(|t| t.range.start < line_end && t.range.end > line_start) + { + let start = ts.range.start.max(line_start); + let end = ts.range.end.min(line_end); + if cursor < start { + spans.push(RatatuiSpan::styled( + text[cursor..start].to_string(), + body_style, + )); + } + if start < end { + spans.push(RatatuiSpan::styled( + text[start..end].to_string(), + ts.style, + )); + cursor = end; + } + } + if cursor < line_end { + spans.push(RatatuiSpan::styled( + text[cursor..line_end].to_string(), + body_style, + )); + } + out.push(spans); + line_start = line_end + 1; + } + out +} + +// --- scanner -------------------------------------------------------- + +struct Scanner<'a> { + source: &'a str, + bytes: &'a [u8], + pos: usize, + out: Vec, +} + +/// Per-command classification state. Reset at every command +/// boundary (`\n`, `;`, or the start of a `[...]` interior). +#[derive(Clone, Copy, Default)] +struct CmdState { + /// What the previous bare-word token was. Used to classify the + /// NEXT bare word in context (e.g. "after `proc`" → decl). + prev: PrevToken, + /// Word index within the current command (0 = command name, + /// 1 = first arg, ...). Used to detect the `proc NAME ARGS RET` + /// shape: at word 2 inside a proc decl, the args brace is + /// expected; at word 3, the return-type ident. + word_idx: usize, + /// True when this command's word-0 was `proc`. Drives the + /// "args-block braces hold parameters" + "word-3 is return + /// type" behaviour. + is_proc_decl: bool, + /// True when word-0 was `dict`. Drives the `dict for` / + /// `dict get` / etc. compound-head recognition — word-1 is a + /// sub-command keyword rather than a plain arg. + is_dict: bool, +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +enum PrevToken { + #[default] + None, + Keyword, // proc / type / enum / namespace + SetKeyword, // `set` +} + +impl<'a> Scanner<'a> { + fn new(source: &'a str) -> Self { + Self { + source, + bytes: source.as_bytes(), + pos: 0, + out: Vec::new(), + } + } + + fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + fn peek_at(&self, off: usize) -> Option { + self.bytes.get(self.pos + off).copied() + } + + fn push(&mut self, range: Range, style: Style) { + if range.start < range.end { + self.out.push(TokenSpan { range, style }); + } + } + + /// Scan a script: a sequence of commands until either end-of- + /// source or a closing-bracket terminator (when we're inside a + /// `[...]` substitution). `limit` is a byte ceiling on the scan + /// (usize::MAX for top-level). + fn scan_script(&mut self, limit: usize) { + while self.pos < self.bytes.len() && self.pos < limit { + self.skip_horizontal_ws(); + if self.pos >= limit { + break; + } + match self.peek() { + None => break, + Some(b']') => return, // end of `[...]` interior + Some(b'\n') | Some(b';') => { + self.pos += 1; + continue; + } + Some(b'#') => { + // Comment in command position. + self.scan_comment(); + continue; + } + _ => {} + } + self.scan_command(limit); + } + } + + /// Skip spaces and tabs. NOT newlines (which are command + /// terminators). + fn skip_horizontal_ws(&mut self) { + while let Some(c) = self.peek() { + if c == b' ' || c == b'\t' { + self.pos += 1; + } else if c == b'\\' && self.peek_at(1) == Some(b'\n') { + // Line continuation. + self.pos += 2; + } else { + break; + } + } + } + + fn scan_command(&mut self, limit: usize) { + let mut state = CmdState::default(); + loop { + self.skip_horizontal_ws(); + if self.pos >= limit { + return; + } + match self.peek() { + None => return, + Some(b'\n') | Some(b';') | Some(b']') => return, + _ => {} + } + self.scan_word(&mut state); + state.word_idx += 1; + // Guard against an inner scan (bare-word / braced / + // bracketed) overshooting the parent's byte limit. The + // narrow case that used to bite: `{lib srcs}` in + // command-arg position — scan_braced_as_script hands the + // interior to scan_script(close_pos), scan_command then + // consumes past the closing `}` because it only looked + // for `\n`/`;`/`]` as terminators. After the outer scan + // resumed, every span from `$deps { … }` was emitted a + // second time, and the per-line renderer duplicated the + // corresponding text on screen. + if self.pos >= limit { + return; + } + } + } + + fn scan_word(&mut self, state: &mut CmdState) { + match self.peek() { + Some(b'"') => { + self.scan_quoted(); + state.prev = PrevToken::None; + } + Some(b'{') => { + if state.is_proc_decl && state.word_idx == 2 { + // Proc args block: scan interior for params/types. + self.scan_proc_args(); + } else { + // All other braces — proc body (word 3 with no + // return type, or word 4 with one), control-flow + // condition / body, generic braced word — get + // scanned as a script. Tcl-convention braces are + // scripts; we always treat them that way so + // identifiers inside `if {…}`, `while {…}`, proc + // bodies, etc. get live highlighting. The narrow + // loss: braced return-type annotations + // (`proc foo {} {dict} { … }`) + // get their interior scanned as a script rather + // than as a type — acceptable since the bare-word + // form is far more common. + self.scan_braced_as_script(); + } + state.prev = PrevToken::None; + } + Some(b'[') => { + self.scan_bracket_subst(); + state.prev = PrevToken::None; + } + Some(b'$') => { + self.scan_var_ref(); + // After a $var, the next ident isn't a decl/var/type. + state.prev = PrevToken::None; + } + Some(b'@') if state.word_idx > 0 => { + self.scan_attribute(); + state.prev = PrevToken::None; + } + Some(c) if c.is_ascii_digit() => { + self.scan_numeric(); + state.prev = PrevToken::None; + } + Some(b'-') => { + // Could be a flag (`-name`) or a negative number. + let next = self.peek_at(1); + if matches!(next, Some(c) if c.is_ascii_digit()) { + self.scan_numeric(); + } else { + self.scan_bare_word(state); + } + state.prev = PrevToken::None; + } + Some(_) => { + self.scan_bare_word(state); + } + None => (), + } + } + + /// Scan a `#` or `##` comment to end of line. Only called when + /// `#` appears in command position. + fn scan_comment(&mut self) { + let start = self.pos; + let is_doc = self.peek_at(1) == Some(b'#'); + while self.pos < self.bytes.len() && self.bytes[self.pos] != b'\n' { + self.pos += 1; + } + let style = if is_doc { + doc_comment_style() + } else { + comment_style() + }; + self.push(start..self.pos, style); + } + + fn scan_quoted(&mut self) { + let start = self.pos; + self.pos += 1; // consume opening `"` + while let Some(c) = self.peek() { + if c == b'\\' && self.peek_at(1).is_some() { + self.pos += 2; + continue; + } + if c == b'"' { + self.pos += 1; + break; + } + self.pos += 1; + } + self.push(start..self.pos, string_style()); + } + + /// Scan a `{...}` proc-args block: emit braces as punct, walk + /// the interior recognizing parameter names, `@attributes`, + /// `:type` annotations, and `;` separators. + fn scan_proc_args(&mut self) { + let open = self.pos; + self.pos += 1; + self.push(open..open + 1, punct_style()); + let mut depth = 1usize; + // Walk the interior emitting param-style for bare idents, + // colon-then-type for annotations, attribute style for `@`. + let mut expect_type = false; + while let Some(c) = self.peek() { + if c == b'{' { + depth += 1; + self.pos += 1; + continue; + } + if c == b'}' { + if depth == 1 { + let close = self.pos; + self.pos += 1; + self.push(close..close + 1, punct_style()); + return; + } + depth -= 1; + self.pos += 1; + continue; + } + if c == b' ' || c == b'\t' || c == b'\n' || c == b';' { + self.pos += 1; + continue; + } + if c == b'#' { + self.scan_comment(); + continue; + } + if c == b':' { + let p = self.pos; + self.pos += 1; + self.push(p..p + 1, punct_style()); + expect_type = true; + continue; + } + if c == b'@' { + self.scan_attribute(); + continue; + } + if c.is_ascii_alphabetic() || c == b'_' { + let start = self.pos; + while let Some(d) = self.peek() { + if d.is_ascii_alphanumeric() + || d == b'_' + || d == b'<' + || d == b'>' + || d == b',' + { + self.pos += 1; + } else if d == b':' && self.peek_at(1) == Some(b':') { + self.pos += 2; + } else { + break; + } + } + let style = if expect_type { + type_style() + } else { + parameter_style() + }; + self.push(start..self.pos, style); + expect_type = false; + continue; + } + // Unknown byte — skip. + self.pos += 1; + } + } + + /// Scan `{...}` as a proc body: recurse with full script + /// classification on the interior. + fn scan_braced_as_script(&mut self) { + let open = self.pos; + self.pos += 1; + self.push(open..open + 1, punct_style()); + let mut depth = 1usize; + // Find the matching close brace position (best-effort). + let mut p = self.pos; + while p < self.bytes.len() { + let c = self.bytes[p]; + if c == b'\\' && p + 1 < self.bytes.len() { + p += 2; + continue; + } + if c == b'{' { + depth += 1; + } else if c == b'}' { + depth -= 1; + if depth == 0 { + break; + } + } + p += 1; + } + // Recurse on the interior [self.pos .. p). + let close_pos = p; + self.scan_script(close_pos); + // Skip past the matching close brace if we found one. + if close_pos < self.bytes.len() && self.bytes[close_pos] == b'}' { + self.pos = close_pos; + self.push(close_pos..close_pos + 1, punct_style()); + self.pos += 1; + } else { + // EOF without close — leave the cursor at end. + self.pos = close_pos.max(self.pos); + } + } + + /// `[command-substitution]` — recurse on the interior as a + /// fresh script. + fn scan_bracket_subst(&mut self) { + let open = self.pos; + self.push(open..open + 1, punct_style()); + self.pos += 1; + self.scan_script(usize::MAX); + if let Some(b']') = self.peek() { + let close = self.pos; + self.push(close..close + 1, punct_style()); + self.pos += 1; + } + } + + fn scan_var_ref(&mut self) { + let start = self.pos; + self.pos += 1; // $ + if self.peek() == Some(b'{') { + self.pos += 1; + while let Some(c) = self.peek() { + self.pos += 1; + if c == b'}' { + break; + } + } + } else { + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() + || c == b'_' + || c == b':' && self.peek_at(1) == Some(b':') + { + self.pos += 1; + } else { + break; + } + } + } + if self.pos > start + 1 { + self.push(start..self.pos, variable_style()); + } + } + + fn scan_attribute(&mut self) { + let start = self.pos; + self.pos += 1; // `@` + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() || c == b'_' { + self.pos += 1; + } else { + break; + } + } + if self.pos > start + 1 { + self.push(start..self.pos, attribute_style()); + } + // Optional `(...)` payload — consume as a balanced group. + if self.peek() == Some(b'(') { + let p = self.pos; + self.pos += 1; + self.push(p..p + 1, punct_style()); + let mut depth = 1usize; + while let Some(c) = self.peek() { + if c == b'(' { + depth += 1; + self.pos += 1; + } else if c == b')' { + depth -= 1; + self.pos += 1; + if depth == 0 { + self.push(self.pos - 1..self.pos, punct_style()); + return; + } + } else if c == b'"' { + self.scan_quoted(); + } else if c.is_ascii_digit() { + self.scan_numeric(); + } else if c.is_ascii_alphabetic() || c == b'_' { + let s = self.pos; + while let Some(d) = self.peek() { + if d.is_ascii_alphanumeric() || d == b'_' { + self.pos += 1; + } else { + break; + } + } + self.push(s..self.pos, string_style()); + } else { + self.pos += 1; + } + } + } + } + + fn scan_numeric(&mut self) { + let start = self.pos; + if self.peek() == Some(b'-') { + self.pos += 1; + } + while let Some(c) = self.peek() { + if c.is_ascii_digit() { + self.pos += 1; + } else { + break; + } + } + if self.pos > start { + self.push(start..self.pos, numeric_style()); + } + } + + fn scan_bare_word(&mut self, state: &mut CmdState) { + let start = self.pos; + while let Some(c) = self.peek() { + if c.is_ascii_alphanumeric() + || c == b'_' + || c == b'-' + || c == b'.' + || c == b'/' + { + self.pos += 1; + } else if c == b':' && self.peek_at(1) == Some(b':') { + self.pos += 2; + } else { + break; + } + } + if self.pos == start { + // No bare-word chars (probably punctuation we don't know). + self.pos += 1; + return; + } + let word = &self.source[start..self.pos]; + let style = match state.prev { + PrevToken::Keyword => { + // Declaration name follows a decl keyword. + state.prev = PrevToken::None; + Some(declaration_style()) + } + PrevToken::SetKeyword => { + state.prev = PrevToken::None; + Some(variable_style()) + } + PrevToken::None => { + if state.word_idx == 0 { + // Command position: keyword / builtin / function. + if DECL_KEYWORDS.contains(&word) { + state.prev = PrevToken::Keyword; + if word == "proc" { + state.is_proc_decl = true; + } + Some(keyword_style()) + } else if word == "set" { + state.prev = PrevToken::SetKeyword; + Some(keyword_style()) + } else if BUILTIN_KEYWORDS.contains(&word) { + Some(keyword_style()) + } else if BUILTIN_FUNCS.contains(&word) { + if word == "dict" { + state.is_dict = true; + } + Some(builtin_func_style()) + } else { + Some(function_style()) + } + } else if state.is_proc_decl && state.word_idx == 3 { + // Return-type slot (bare-ident form). + Some(type_style()) + } else if state.is_dict && state.word_idx == 1 { + // `dict for`, `dict get`, `dict set`, … + // Compound-head sub-command; color like the + // top-level keyword it stands in for. + Some(keyword_style()) + } else if word == "true" || word == "false" { + // Boolean literals in arg position — same + // treatment as top-level bool sniffing so + // `-flag true` reads consistently. + Some(keyword_style()) + } else if word.starts_with('-') && word.len() > 1 { + // Flag-style word. + Some(attribute_style()) + } else { + // Argument to a call — leave to body color, with + // the exception that if the next char is `:` we + // still treat this as a parameter name (rare in + // call args but harmless to leave unstyled). + None + } + } + }; + if let Some(s) = style { + self.push(start..self.pos, s); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn highlights(source: &str) -> Vec<(String, &'static str)> { + highlight_source(source) + .into_iter() + .map(|s| { + let text = source[s.range.clone()].to_string(); + let tag = style_tag(&s.style); + (text, tag) + }) + .collect() + } + + fn style_tag(style: &Style) -> &'static str { + if style == &keyword_style() { + "keyword" + } else if style == &builtin_func_style() { + "builtin_func" + } else if style == &function_style() { + "function" + } else if style == &declaration_style() { + "decl" + } else if style == ¶meter_style() { + "param" + } else if style == &variable_style() { + "var" + } else if style == &string_style() { + "string" + } else if style == &comment_style() { + "comment" + } else if style == &doc_comment_style() { + "doc" + } else if style == &type_style() { + "type" + } else if style == &attribute_style() { + "attr" + } else if style == &numeric_style() { + "num" + } else if style == &punct_style() { + "punct" + } else { + "?" + } + } + + fn has(h: &[(String, &str)], text: &str, tag: &str) -> bool { + h.iter().any(|(t, g)| t == text && *g == tag) + } + + #[test] + fn empty_input() { + assert!(highlight_source("").is_empty()); + } + + #[test] + fn set_keyword_var_value() { + let h = highlights("set foo 42"); + assert!(has(&h, "set", "keyword"), "{h:?}"); + assert!(has(&h, "foo", "var"), "{h:?}"); + assert!(has(&h, "42", "num"), "{h:?}"); + } + + #[test] + fn generic_call_first_word_function() { + let h = highlights("foo bar baz"); + assert!(has(&h, "foo", "function"), "{h:?}"); + } + + #[test] + fn builtin_control_flow_keyword() { + let h = highlights("if {$x > 0} { puts hi }"); + assert!(has(&h, "if", "keyword"), "{h:?}"); + // `puts` inside the body braces should still highlight as a + // builtin func because the body is scanned as a script. + assert!(has(&h, "puts", "builtin_func"), "{h:?}"); + // $x inside the condition braces gets var styling via the + // braced-as-string scanner... but the condition braces are + // treated as a string today. v1 accepts that — the cost of + // braced-string consistency is occasional loss of $var + // styling inside conditions. Acceptable trade. + let _ = h; + } + + #[test] + fn proc_decl_keywords_and_names() { + let h = highlights("proc foo {x: int} bool { return $x }"); + assert!(has(&h, "proc", "keyword")); + assert!(has(&h, "foo", "decl")); + assert!(has(&h, "x", "param")); + assert!(has(&h, "int", "type")); + assert!(has(&h, "bool", "type")); + // Body scanned as script. + assert!(has(&h, "return", "keyword")); + assert!(has(&h, "$x", "var")); + } + + #[test] + fn proc_with_attribute() { + let h = highlights("proc foo {@default(0) x: int} unit {}"); + assert!(has(&h, "@default", "attr")); + assert!(has(&h, "0", "num")); + assert!(has(&h, "x", "param")); + assert!(has(&h, "int", "type")); + assert!(has(&h, "unit", "type")); + } + + #[test] + fn type_decl() { + let h = highlights("type Properties = {dict}"); + assert!(has(&h, "type", "keyword")); + assert!(has(&h, "Properties", "decl")); + } + + #[test] + fn enum_decl() { + let h = highlights("enum Direction = {North South East West}"); + assert!(has(&h, "enum", "keyword")); + assert!(has(&h, "Direction", "decl")); + } + + #[test] + fn doc_comment_vs_regular() { + let h = highlights("# regular\n## doc\nset x 1"); + assert!(h + .iter() + .any(|(t, tag)| t.contains("regular") && *tag == "comment")); + assert!(h.iter().any(|(t, tag)| t.contains("doc") && *tag == "doc")); + } + + #[test] + fn cmd_substitution_recurses() { + let h = highlights("set y [foo $x]"); + assert!(has(&h, "set", "keyword")); + assert!(has(&h, "foo", "function")); + assert!(has(&h, "$x", "var")); + assert!(has(&h, "[", "punct")); + assert!(has(&h, "]", "punct")); + } + + #[test] + fn quoted_string() { + let h = highlights("puts \"hello world\""); + assert!(has(&h, "puts", "builtin_func")); + assert!(has(&h, "\"hello world\"", "string")); + } + + // The crucial stability tests: incomplete input must produce + // the SAME classifications for the tokens that ARE present. + #[test] + fn stable_on_unclosed_proc_brace() { + let complete = highlights("proc foo {x: int} bool { return 0 }"); + let incomplete = highlights("proc foo {x: int} bool { return 0"); + // The tokens shared between the two must classify the same. + // Specifically: proc/foo/x/int/bool/return all match. + for (text, tag) in &[ + ("proc", "keyword"), + ("foo", "decl"), + ("x", "param"), + ("int", "type"), + ("bool", "type"), + ("return", "keyword"), + ] { + assert!( + has(&complete, text, tag), + "complete missing ({text}, {tag}): {complete:?}" + ); + assert!( + has(&incomplete, text, tag), + "incomplete missing ({text}, {tag}): {incomplete:?}" + ); + } + } + + #[test] + fn stable_on_unclosed_outer_brace() { + let complete = highlights("proc foo {}"); + let incomplete = highlights("proc foo {"); + for (text, tag) in &[("proc", "keyword"), ("foo", "decl")] { + assert!(has(&complete, text, tag), "complete missing"); + assert!(has(&incomplete, text, tag), "incomplete missing"); + } + } + + #[test] + fn stable_on_unclosed_bracket() { + let complete = highlights("set y [foo $x]"); + let incomplete = highlights("set y [foo $x"); + for (text, tag) in + &[("set", "keyword"), ("foo", "function"), ("$x", "var")] + { + assert!( + has(&complete, text, tag), + "complete missing ({text}, {tag})" + ); + assert!( + has(&incomplete, text, tag), + "incomplete missing ({text}, {tag}): {incomplete:?}" + ); + } + } + + #[test] + fn no_panic_on_garbage() { + let _ = highlight_source("[[["); + let _ = highlight_source("{{{"); + let _ = highlight_source("$$$"); + let _ = highlight_source(""); + let _ = highlight_source("# unterminated\nproc \"weird name\""); + } + + #[test] + fn plain_prose_not_styled_as_decl() { + // Top-level plain words shouldn't get function styling for + // their non-first words (i.e. just the first word styles + // as function-call; the rest are left to body color). + let h = highlights("just some words"); + assert!(has(&h, "just", "function")); + // "some" and "words" are call args — no special styling. + assert!(!has(&h, "some", "decl")); + assert!(!has(&h, "words", "decl")); + } + + #[test] + fn proc_body_contents_scan_as_script() { + // Regression: the body of a proc decl (without a return-type + // annotation) used to be styled as a single type span, + // painting the whole interior teal. Body must scan as a + // normal script so `set`, `[...]`, `$vars`, etc. all + // classify normally. + let h = + highlights("proc foo { x: int } { set foo [reticulate -value x] }"); + // Inside the body: + assert!(has(&h, "set", "keyword"), "{h:?}"); + // `foo` after `set` is the variable being assigned. + assert!( + h.iter().any(|(t, tag)| t == "foo" && *tag == "var"), + "expected `foo` styled as var inside body: {h:?}" + ); + assert!(has(&h, "reticulate", "function"), "{h:?}"); + assert!(has(&h, "[", "punct")); + assert!(has(&h, "]", "punct")); + } + + #[test] + fn body_with_and_without_return_type_scan_same() { + // Identical body content should classify the same whether + // the proc has a return-type annotation or not. + let with = highlights("proc f {} unit { set x 1 }"); + let without = highlights("proc f {} { set x 1 }"); + for (text, tag) in &[("set", "keyword"), ("x", "var")] { + assert!(has(&with, text, tag), "with: missing ({text}, {tag})"); + assert!( + has(&without, text, tag), + "without: missing ({text}, {tag}): {without:?}" + ); + } + } + + #[test] + fn per_line_preserves_content() { + let src = "set a 1\nproc f {} unit {}\nputs hi"; + let lines = highlight_per_line(src, Style::default()); + let mut reconstructed = String::new(); + for (i, line) in lines.iter().enumerate() { + if i > 0 { + reconstructed.push('\n'); + } + for span in line { + reconstructed.push_str(span.content.as_ref()); + } + } + assert_eq!(reconstructed, src); + } + + #[test] + fn braced_word_scan_does_not_overshoot_close() { + // Regression: `{lib srcs}` as an arg to `dict for` was + // recursed into as a script, but `scan_command` inside + // that recursion consumed past the closing `}` because + // it only checked for `\n`/`;`/`]`. The outer scan then + // re-tokenized everything from `$deps` onward, and the + // renderer duplicated the corresponding text on screen. + // + // Assert token spans are non-overlapping and monotonic. + let src = "dict for {lib srcs} $deps {\n puts $lib\n}"; + let tokens = highlight_source(src); + let mut prev_end = 0usize; + for t in &tokens { + assert!( + t.range.start >= prev_end, + "overlap at {:?} after prev end {prev_end} — full: {tokens:?}", + t.range, + ); + prev_end = t.range.end; + } + // And reconstructing per-line output round-trips. + let lines = highlight_per_line(src, Style::default()); + let mut reconstructed = String::new(); + for (i, line) in lines.iter().enumerate() { + if i > 0 { + reconstructed.push('\n'); + } + for span in line { + reconstructed.push_str(span.content.as_ref()); + } + } + assert_eq!(reconstructed, src); + } +} diff --git a/vw-repl/src/history.rs b/vw-repl/src/history.rs new file mode 100644 index 0000000..ed6ad72 --- /dev/null +++ b/vw-repl/src/history.rs @@ -0,0 +1,236 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Persistent input history with incremental search. +//! +//! Entries are appended to a newline-delimited file (one entry per +//! line, with embedded newlines escaped) under the platform's state +//! dir — typically `~/.local/state/vw/repl-history` on Linux. The +//! file is loaded once at startup; new entries are appended both to +//! memory and to the file as soon as they're recorded so a crashed +//! session doesn't lose history. +//! +//! Ctrl-R triggers an *incremental* search: as the user types, we +//! find the most recent entry whose text contains the query as a +//! substring (case-insensitive). Repeated Ctrl-R steps to the next- +//! older match. Esc cancels; Enter accepts the match into the input +//! buffer. + +use std::fs::{create_dir_all, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; + +const ESCAPED_NEWLINE: &str = "\\n"; +const ESCAPED_BACKSLASH: &str = "\\\\"; + +/// In-memory history, backed by an on-disk file. Indexed +/// most-recent-last; `entries[entries.len() - 1]` is the freshest +/// record, matching how Readline / Reedline order things. +#[derive(Debug)] +pub struct History { + file_path: PathBuf, + entries: Vec, +} + +impl History { + /// Load history from the default location. Returns an empty + /// store (and skips disk writes) when no state dir is available + /// — the REPL still runs, just without persistence. + pub fn load_default() -> Self { + let path = default_history_path(); + match path { + Some(p) => Self::load_from(p), + None => Self { + file_path: PathBuf::new(), + entries: Vec::new(), + }, + } + } + + /// Load history from a specific file. Missing file → empty + /// history (the file gets created on first append). + pub fn load_from(file_path: PathBuf) -> Self { + let entries = read_entries(&file_path).unwrap_or_default(); + Self { file_path, entries } + } + + #[allow(dead_code)] // public API for the in-progress completion slice + pub fn entries(&self) -> &[String] { + &self.entries + } + + /// Append `entry` to the in-memory log and persist it. Empty or + /// whitespace-only entries are ignored. An entry identical to + /// the most recent one is also ignored (the common case of + /// re-running the same command shouldn't bloat the file). + pub fn append(&mut self, entry: &str) { + let trimmed = entry.trim(); + if trimmed.is_empty() { + return; + } + if self.entries.last().map(String::as_str) == Some(entry) { + return; + } + self.entries.push(entry.to_string()); + if self.file_path.as_os_str().is_empty() { + return; + } + if let Some(parent) = self.file_path.parent() { + let _ = create_dir_all(parent); + } + if let Ok(mut f) = OpenOptions::new() + .create(true) + .append(true) + .open(&self.file_path) + { + let _ = writeln!(f, "{}", encode_line(entry)); + } + } + + /// Find the most recent entry whose text contains `query` as a + /// substring (case-insensitive). Returns the index in + /// [`Self::entries`] plus the entry itself. `start_before` is an + /// exclusive upper bound — passing `Some(prev_idx)` resumes the + /// search at the next-older entry, which is how repeated + /// `Ctrl-R` steps backward. + pub fn search_back( + &self, + query: &str, + start_before: Option, + ) -> Option<(usize, &str)> { + if query.is_empty() { + return None; + } + let upper = start_before.unwrap_or(self.entries.len()); + let needle = query.to_lowercase(); + for i in (0..upper).rev() { + if self.entries[i].to_lowercase().contains(&needle) { + return Some((i, self.entries[i].as_str())); + } + } + None + } +} + +fn default_history_path() -> Option { + let state = dirs::state_dir().or_else(dirs::data_local_dir)?; + Some(state.join("vw").join("repl-history")) +} + +fn read_entries(path: &PathBuf) -> Option> { + let f = std::fs::File::open(path).ok()?; + let mut entries = Vec::new(); + for line in BufReader::new(f).lines().map_while(Result::ok) { + entries.push(decode_line(&line)); + } + Some(entries) +} + +fn encode_line(s: &str) -> String { + // Single-line newline-delimited file format: backslashes and + // embedded newlines get a literal escape so a multi-line htcl + // buffer round-trips cleanly. + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str(ESCAPED_BACKSLASH), + '\n' => out.push_str(ESCAPED_NEWLINE), + other => out.push(other), + } + } + out +} + +fn decode_line(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('n') => out.push('\n'), + Some('\\') => out.push('\\'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + #[test] + fn append_persists_and_dedupes_consecutive_duplicates() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("h"); + let mut h = History::load_from(p.clone()); + h.append("foo"); + h.append("foo"); + h.append("bar"); + h.append(""); + h.append(" "); + assert_eq!(h.entries(), &["foo".to_string(), "bar".to_string()]); + // File round-trips. + let mut buf = String::new(); + std::fs::File::open(&p) + .unwrap() + .read_to_string(&mut buf) + .unwrap(); + assert_eq!(buf, "foo\nbar\n"); + } + + #[test] + fn multiline_entries_round_trip_with_escapes() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("h"); + { + let mut h = History::load_from(p.clone()); + h.append("set x [\n create_cpm5 -name cpm5\n]"); + h.append("with \\ backslash"); + } + let h2 = History::load_from(p); + assert_eq!( + h2.entries(), + &[ + "set x [\n create_cpm5 -name cpm5\n]".to_string(), + "with \\ backslash".to_string(), + ] + ); + } + + #[test] + fn search_back_finds_most_recent_match() { + let dir = tempfile::tempdir().unwrap(); + let mut h = History::load_from(dir.path().join("h")); + h.append("set x 1"); + h.append("create_project foo"); + h.append("set y 2"); + let (idx, hit) = h.search_back("set", None).unwrap(); + assert_eq!(hit, "set y 2"); + assert_eq!(idx, 2); + // Step to the next older. + let (idx2, hit2) = h.search_back("set", Some(idx)).unwrap(); + assert_eq!(hit2, "set x 1"); + assert_eq!(idx2, 0); + // Nothing older. + assert!(h.search_back("set", Some(idx2)).is_none()); + } + + #[test] + fn search_is_case_insensitive() { + let dir = tempfile::tempdir().unwrap(); + let mut h = History::load_from(dir.path().join("h")); + h.append("Create_Project foo"); + let (_, hit) = h.search_back("create", None).unwrap(); + assert_eq!(hit, "Create_Project foo"); + } +} diff --git a/vw-repl/src/lib.rs b/vw-repl/src/lib.rs new file mode 100644 index 0000000..d33c68b --- /dev/null +++ b/vw-repl/src/lib.rs @@ -0,0 +1,166 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Interactive REPL for htcl scripts. +//! +//! A ratatui-driven shell that talks to a long-lived Vivado worker +//! via [`vw_vivado::VivadoBackend`]. The session document model +//! (every successful eval appended to an in-memory script + the +//! current input as its tail) lets the analyzer power the same +//! features the LSP gives editors — completion, hover, signature +//! help — without any REPL-specific machinery. +//! +//! v1 (this slice) ships the foundation: screen layout, multi-line +//! input with Readline-quality editing, persistent history with +//! Ctrl-R search, a long-lived Vivado worker, and `:load `. +//! Tab completion, signature help, hover overlay, command palette, +//! and structured-result rendering layer on top in subsequent +//! slices. + +mod app; +pub mod config; +pub mod diag_search; +pub mod highlight; +pub mod highlight_htcl; +mod history; +pub mod lower; +mod popup; +mod render; +mod session; +mod symbol_index; +mod symbol_search; +pub mod trace; +mod ui; + +use camino::Utf8PathBuf; +use thiserror::Error; + +pub use app::App; +pub use lower::{build_proc_locations, Origin, OriginFrame, ProcLocation}; +pub use session::Session; +pub use trace::{ + display_path, resolve_stack_frames_with, rewrite_stack_line, RewrittenFrame, +}; + +/// Wrap a Tcl body with shim-side origin markers so any traceless +/// warning/error emitted during the eval is tagged with `origin` +/// via the marker stack — not with whatever the REPL / CLI happens +/// to have as `pending_eval_index` when the message eventually +/// arrives. +/// +/// The race this fixes: Vivado's C++ writes warning bytes to the +/// PTY, then Tcl sends the eval response over the protocol socket. +/// The pump thread's latency can put the response ahead of the +/// warning at the receiver, so the warning lands during the *next* +/// eval and inherits its origin — often a synthetic prelude +/// command with `line=0`. Wrapping the body means the origin frame +/// sits on the shim's marker stack from `emit_pty_ctx_begin` (just +/// before the body runs) until `emit_pty_ctx_end` (just after), +/// and every straggler in that window tags off the top of the +/// stack. +/// +/// The wrapped body preserves rc/result/errorcode/-errorinfo via +/// `return -options`, so callers see identical behavior to the +/// unwrapped `tcl`. +pub fn wrap_tcl_with_origin_marker(tcl: &str, origin: &Origin) -> String { + // Build a single frame string in the same ":" + // shape `capture_stack` emits, so the downstream renderer's + // stack-frame regex handles both uniformly. No proc part — + // this frame is the *statement's* file/line, not a Tcl + // proc-body line. + let file_repr = origin + .file + .as_deref() + .map(display_path) + .unwrap_or_else(|| "".to_string()); + let frame = format!("{file_repr}:{}", origin.line); + // Tcl list-quote via braces. The frame content is a file path + // + integer, so braces alone are sufficient — no metachars to + // escape. + format!( + "::vw::emit_pty_ctx_begin [list {{{frame}}}]\n\ + set _vw_wrap_rc [catch {{\n{tcl}\n}} _vw_wrap_r _vw_wrap_o]\n\ + ::vw::emit_pty_ctx_end\n\ + return -options $_vw_wrap_o $_vw_wrap_r" + ) +} + +#[derive(Debug, Error)] +pub enum ReplError { + #[error("terminal I/O: {0}")] + Io(#[from] std::io::Error), + #[error("backend: {0}")] + Backend(#[from] vw_eda::BackendError), +} + +/// Tunable knobs supplied by the CLI invocation. +#[derive(Clone, Debug, Default)] +pub struct ReplOptions { + /// Minimum severity that renders in scrollback. `Debug` shows + /// every block raw (including Vivado's banners, tables, and + /// other non-diagnostic noise); higher levels collapse + /// non-diagnostic content into a toggleable placeholder and + /// hide diagnostics below the threshold. + pub log_level: vw_vivado::LogLevel, + /// If set, source this file into the session immediately after + /// the Vivado worker comes up. Equivalent to typing `:load + /// ` as the first input. + pub initial_load: Option, + /// If set, dispatch this literal htcl snippet as the first + /// input after the Vivado worker comes up. Takes precedence + /// over `initial_load`. Used by `vw repl --from-*-checkpoint` + /// to open a pre-existing DCP instead of running `design.htcl`. + pub initial_source: Option, + /// If true, INFO-severity Vivado messages carry their full Tcl + /// stack frames into the scrollback. Off by default — INFO is + /// noisy enough without stack traces — but useful when diagnosing + /// where a particular INFO is emitted from. + pub info_with_stack: bool, + /// Optional `--part ` selector — picks a non-default + /// `[[target-parts]]` entry to drive the auto-project. `None` + /// uses the workspace default. Mutually exclusive with + /// `variant`; the CLI enforces this via clap. + pub part: Option, + /// Optional `--variant ` selector — picks a + /// `[[workspace.variants]]` entry to drive the auto-project + /// AND to filter design sources via the session-scoped + /// active-variant fallback in the RPC handler. + pub variant: Option, +} + +/// Where the vivado backing a REPL session runs. +/// +/// Passed alongside [`ReplOptions`] rather than inside it: the options are +/// cloned and debug-printed, and a live backend is neither. Everything above +/// this — the editor, the scrollback, the symbol index, the diagnostics — is +/// the same either way, because all of it is about the source on this machine. +#[derive(Default)] +pub enum Worker { + /// Spawn one on this machine, as `vw repl` always has. + #[default] + Local, + /// Drive one already running on an instance. + Remote { + /// The session, already opened. + backend: Box, + /// How to cut short a running command. + /// + /// Carried separately because the backend is borrowed for as long as + /// a command is in flight, and Ctrl-C has to work precisely then. + interrupt: Interrupt, + }, +} + +/// How to stop whatever the worker is running. +/// +/// A local session signals vivado's process group; a remote one asks the +/// instance to. The REPL does not need to know which, only that Ctrl-C during +/// an eval means calling this. +pub type Interrupt = std::sync::Arc; + +/// Run the REPL until the user exits. Owns the terminal alternate +/// screen for the duration; restores it on every exit path. +pub async fn run(opts: ReplOptions, worker: Worker) -> Result<(), ReplError> { + app::run(opts, worker).await +} diff --git a/vw-repl/src/lower.rs b/vw-repl/src/lower.rs new file mode 100644 index 0000000..b01a7ef --- /dev/null +++ b/vw-repl/src/lower.rs @@ -0,0 +1,1735 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lower a REPL input buffer to a sequence of `(htcl-origin, Tcl)` +//! commands the Vivado worker can evaluate one at a time. +//! +//! Shipping one statement per `eval` (rather than a single +//! concatenated script) is what lets us render Vivado errors against +//! htcl source. The loader's [`vw_htcl::LoadedProgram::locate_span`] +//! tells us which `.htcl` file each top-level statement came from; +//! we keep that mapping alongside the lowered Tcl so the REPL can +//! report `× : ` instead of a Tcl stack +//! trace pointing into our shim. + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use camino::Utf8Path; +use vw_htcl::{LineIndex, Resolver}; + +use crate::session::{Session, SessionBatch}; + +struct NoopObserver; +impl vw_htcl::LoadObserver for NoopObserver {} + +#[derive(Debug, thiserror::Error)] +pub enum LowerError { + #[error("writing scratch input file: {0}")] + Io(#[from] std::io::Error), + #[error("loading htcl: {0}")] + Load(#[from] vw_htcl::LoadError), + #[error("{0}")] + Parse(String), +} + +/// Where in the loaded htcl tree a particular command came from. +/// Drives the error renderer in the App. +#[derive(Clone, Debug)] +pub struct Origin { + /// `.htcl` file the command was declared in, when known. `None` + /// only when the input itself wasn't backed by a real file + /// (e.g. interactive REPL input lowered before any imports). + pub file: Option, + /// 1-based line number in `file` (or in the input buffer when + /// `file` is `None`). + pub line: u32, + /// First line of the command as written by the user — used as + /// the "what was running" line in the error renderer. + pub snippet: String, + /// The chain of `src` imports that brought this command's file + /// into scope, ordered nearest-first (so the last frame is the + /// entry file / user input). Empty when the command lives + /// directly in the entry, since there's nothing to chain. + pub via: Vec, +} + +/// One frame in the `via` chain: a `src` statement in some +/// importing file, captured as that importer's path, the line the +/// `src` lives on, and the snippet of that line (so the user sees +/// `src ip/cips` and not just `src`). +#[derive(Clone, Debug)] +pub struct OriginFrame { + pub file: Option, + pub line: u32, + pub snippet: String, +} + +#[derive(Clone, Debug)] +pub struct PreparedCommand { + pub tcl: String, + pub origin: Origin, + /// Declared return type of the expression this command + /// evaluates, when knowable from static analysis. `None` for + /// expressions whose head we couldn't resolve to a known proc + /// (untyped calls, control flow, raw Tcl, etc.). The App uses + /// this to suppress the Result push entirely on `unit` and to + /// skip the heuristic fallback formatter on every other typed + /// case (since the wrapped `tcl` already returns a formatted + /// string from the type's `repr` proc). + pub expected_return_type: Option, + /// True when the top-level command is `set VAR ` — a + /// binding operation. The App suppresses the Result echo for + /// these: the user picked a name to hold the value; they + /// didn't ask to see it. `puts $VAR` is the explicit + /// "show me" form when they want the value displayed. + /// + /// Applies to the LITERAL `set` command only. `[set x y]` + /// inside a bracketed expression doesn't count — that's a + /// nested Tcl call, not a top-level binding. + pub is_set_binding: bool, +} + +#[derive(Debug)] +pub struct Prepared { + /// Each top-level statement in the loaded program, in source + /// order. The worker fires `eval` once per item and stops at + /// the first failure. + pub commands: Vec, + /// The parsed program + proc map for this batch. Stays out of + /// the session document until every command in [`commands`] + /// succeeds — at which point the App calls + /// [`Session::commit`](crate::session::Session::commit) to + /// fold it into the running session. On failure the batch is + /// dropped, which is what keeps a half-applied state from + /// polluting the analyzer. + pub batch: SessionBatch, + /// Pre-flight findings worth surfacing to the user *before* we + /// ship anything to Vivado. The most common one is "this call + /// uses `-flag` keyword args but the proc isn't a loaded htcl + /// wrapper" — Vivado's underlying builtin almost always parses + /// the arguments differently, and the resulting error message + /// makes no sense without that context. + pub warnings: Vec, + /// Top-level statements that lived directly in the entry file + /// (the user's `--load` target, or the typed REPL input), + /// regardless of whether they lowered to any Tcl. Captured so + /// the `--load` echo path can show `src` directives next to + /// the calls that produce Tcl — without this, `src @vivado-cmd` + /// would never get its `›` echo because its lowering is empty + /// (consumed at load time by the loader). + pub entry_top_level: Vec, +} + +#[derive(Clone, Debug)] +pub struct PrepareWarning { + pub origin: Origin, + pub message: String, +} + +/// Where a proc's body lives in htcl source. `body_start_line` is +/// the 1-based absolute line of the first body line in `file`; line +/// N of the proc's body is `body_start_line + N - 1` in `file` and +/// `body_lines[N - 1]` carries that line's text. +#[derive(Clone, Debug)] +pub struct ProcLocation { + pub file: Option, + pub body_start_line: u32, + pub body_lines: Vec, +} + +impl ProcLocation { + /// Resolve a 1-based body line into a renderable + /// (absolute_line, content) pair. Returns `None` when the + /// reported line is past the end of the body — happens when + /// Tcl points at a line we can't account for (synthesized + /// content, off-by-one in some wrapper, etc.); the caller + /// gracefully skips the frame. + pub fn resolve_body_line(&self, n: u32) -> Option<(u32, String)> { + let idx = n.checked_sub(1)? as usize; + let content = self.body_lines.get(idx).cloned()?; + Some((self.body_start_line + idx as u32, content)) + } +} + +pub fn prepare( + input: &str, + cwd: &Path, + session: &Session, +) -> Result { + let mut noop = NoopObserver; + prepare_with_observer(input, cwd, session, &mut noop) +} + +/// Same as [`prepare`], with an extra hook the loader fires per +/// parsed file. Used by the perf regression test to assert that a +/// new batch only parses its own content (plus any transitive +/// `src` imports), never the entire prior-session prelude. +pub fn prepare_with_observer( + input: &str, + cwd: &Path, + session: &Session, + observer: &mut dyn vw_htcl::LoadObserver, +) -> Result { + let workspace_dir = vw_lib::find_workspace_dir(cwd); + let resolver = build_resolver(workspace_dir.as_deref()); + + let scratch_dir = workspace_dir + .as_deref() + .map(Utf8Path::as_std_path) + .unwrap_or(cwd); + + // The scratch contains ONLY the new input — never a prepended + // prelude. Prior batches contribute parsed signatures and proc + // locations directly via `session`, so we never re-parse the + // entire session on each keystroke. This is what keeps the + // REPL responsive after several `src @lib` imports have built + // up hundreds of thousands of lines of wrapper declarations. + let scratch = ScratchFile::new(scratch_dir, input)?; + + // Seed the loader's already-loaded set from prior batches so + // an incoming `src ip/gtm` (already sourced by an earlier + // batch) short-circuits at each preloaded file instead of + // re-parsing thousands of transitive imports. Every REPL + // submit prior to this change re-parsed the full transitive + // tree from scratch — 879 vivado-cmd files can dominate + // wall-clock for tens of seconds even though the analyzer + // already has all of it in prior session batches. + let preloaded = session.loaded_paths(); + let program = vw_htcl::load_program_with_preloaded( + &scratch.path, + &resolver, + observer, + &preloaded, + )?; + let parsed = vw_htcl::parse(&program.source); + + if let Some(err) = parsed.errors.first() { + let idx = LineIndex::new(&program.source); + let (start, _) = idx.range(err.span); + let where_ = + render_location(&program, err.span, start.line + 1, &scratch.path); + return Err(LowerError::Parse(format!("{where_}: {}", err.message))); + } + + // Validator runs first so unknown-keyword-call errors land + // before we ship anything. Prior-batch signatures + types + + // enums + top-level var names are merged in so calls, type + // refs, and `$var` references to prior-batch state all + // resolve. These are hard errors (not pre-flight warnings); + // routing them back as `LowerError` keeps the App's existing + // error-rendering path unchanged. + let prior_sigs = session.signature_table(); + let prior_types = session.type_decl_table(); + let prior_vars = session.top_level_var_names(); + // Prior-batch variable TYPES for the putr rewrite. Without + // this seed, `putr $prior_var` at a fresh prompt would fall + // through to plain `puts` (the var came from a previous + // batch's `set`, invisible to the current parse alone) and + // dump the raw tagged Tcl list instead of dispatching + // through the type's `repr`. + let prior_var_types = session.top_level_var_types(); + + // Build the `putr` rewrite map: for every `putr ` + // command in the document, the value's replacement Tcl. The + // lowering consults this map per-command via + // `vw_htcl::lower_command_with_putr`. Empty when the source + // contained no `putr` calls; safe (and cheap) to build + // unconditionally. + let putr_map = vw_htcl::putr::rewrite_with_extras( + &program.source, + &parsed.document, + &prior_sigs, + &prior_var_types, + ); + // Names of every dep the workspace resolver knows about. + // Passed to the validator so `src @` where `` + // isn't in vw.toml fires a spanned Error diagnostic. + let dep_names: std::collections::HashSet = + resolver.deps().map(|(name, _)| name.to_string()).collect(); + let validator_diags = vw_htcl::validate_with_all_extras_and_vars( + &parsed.document, + &program.source, + &prior_sigs, + &prior_types, + &std::collections::HashMap::new(), + &prior_vars, + &dep_names, + ); + if let Some(first_err) = validator_diags + .iter() + .find(|d| matches!(d.severity, vw_htcl::Severity::Error)) + { + let idx = LineIndex::new(&program.source); + let (start, _) = idx.range(first_err.span); + let where_ = render_location( + &program, + first_err.span, + start.line + 1, + &scratch.path, + ); + return Err(LowerError::Parse(format!( + "{where_}: {}", + first_err.message + ))); + } + + // Build the lowering table by merging prior-batch signatures + // with the new doc's own. The new doc's entries shadow prior + // ones (Tcl's "second `proc` redefines" semantics) — done by + // starting from the prior table and `extend`-ing with the new + // doc's table, since `extend` overwrites on key collision. + let mut table = prior_sigs; + table.extend(vw_htcl::signature_table(&parsed.document)); + let line_index = LineIndex::new(&program.source); + // Parse the *raw* input (not `program.source`) to capture every + // top-level statement as the user wrote it, including `src` + // directives. The loader rewrites `src` into the imported file's + // content before parsing `program.source`, so the loader-expanded + // document no longer contains a Stmt::Command for `src @foo`. + // We need that statement to drive the `--load` echo path. + let entry_top_level: Vec = { + let entry_parsed = vw_htcl::parse(input); + let entry_idx = LineIndex::new(input); + let mut out = Vec::new(); + for stmt in &entry_parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let (line, _) = entry_idx.range(cmd.span); + let snippet = input[cmd.span.start as usize..cmd.span.end as usize] + .trim_end() + .to_string(); + out.push(Origin { + file: None, + line: line.line + 1, + snippet, + via: Vec::new(), + }); + } + out + }; + + let mut commands = Vec::new(); + let mut extern_names: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + + // Auto-emit machinery for enums + overload dispatchers. Both + // ship as synthetic PreparedCommand entries up front so the + // user's statements (which may construct enum values or call + // overloaded procs) find the supporting Tcl already in scope. + // The classification + overload-table build also re-runs the + // multi-decl signature collection — diagnostics from THAT pass + // already fired through the validator above, so we discard + // them here. + let mut _ignored_diags = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored_diags); + // Merge prior-batch type declarations so wrap_with_repr can + // see newtypes declared in earlier `src @lib` batches (e.g. + // `type Properties = dict` from + // @vivado-cmd, when the user types + // `util::props -object $cips` at a later REPL prompt). + // Without this merge, the wrap can't recurse into + // Properties's underlying to ship + // `dict_string_Property::repr`, and the user's + // `Properties::repr` body fails with `invalid command + // name`. + let mut type_decl_table = session.type_decl_table(); + let batch_type_decls = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored_diags); + for (name, td) in batch_type_decls { + type_decl_table.insert(name, td); + } + let newtype_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sig_table, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &newtype_names, + &mut _ignored_diags, + ); + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if prelude.is_empty() { + continue; + } + commands.push(PreparedCommand { + tcl: prelude, + origin: Origin { + file: None, + line: 0, + snippet: format!( + "", + ed.name.as_deref().unwrap_or("?") + ), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + commands.push(PreparedCommand { + tcl: dispatcher, + origin: Origin { + file: None, + line: 0, + snippet: format!("", info.public_name), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + + // Eagerly emit the primitive repr prelude (string/int/bool/ + // unit) so user procs that call e.g. `extern::string::repr` + // from inside their bodies see those procs in scope. Without + // this, the primitives are only emitted by `wrap_with_repr` + // at top-level REPL eval sites, leaving inner uses dead. + for proc in vw_htcl::repr::emit_primitive_prelude() { + commands.push(PreparedCommand { + tcl: proc, + origin: Origin { + file: None, + line: 0, + snippet: "".into(), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + + // Eagerly emit monomorphized generic reprs for every declared + // type alias whose underlying is a generic + // (`dict<…>` / `list<…>`). Without this, a user-written + // `T::repr` body that delegates to the compiler-synthesized + // monomorphized name (e.g. `Properties::repr` calling + // `extern::dict_string_Property::repr`) errors at runtime + // when invoked from inside a proc body — `wrap_with_repr` + // only emits the monomorphization chain at top-level REPL + // eval sites, not for inner uses. By emitting here, the + // procs are in scope everywhere within the session. + // + // Dedup-by-text within the batch prevents shipping the same + // monomorphization more than once when two type aliases + // resolve to the same underlying generic. + let mut emitted_mono_reprs: std::collections::HashSet = + std::collections::HashSet::new(); + for td in type_decl_table.values() { + let Some(underlying) = td.underlying.as_ref() else { + continue; + }; + if !matches!(underlying, vw_htcl::TypeExpr::Generic { .. }) { + continue; + } + let emission = + vw_htcl::repr::emit_repr_with_types(underlying, &type_decl_table); + for proc in emission.procs { + if !emitted_mono_reprs.insert(proc.clone()) { + continue; + } + commands.push(PreparedCommand { + tcl: proc, + origin: Origin { + file: None, + line: 0, + snippet: format!( + "", + td.name.as_deref().unwrap_or("?") + ), + via: Vec::new(), + }, + expected_return_type: None, + is_set_binding: false, + }); + } + } + + for stmt in &parsed.document.stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + let (line_one_based, _) = line_index.range(cmd.span); + let origin = build_origin( + &program, + cmd.span, + line_one_based.line + 1, + &scratch.path, + ); + // If this command is a proc that's been classified as an + // overload specialization, lower it under its mangled name + // so the dispatcher (shipped above) can find it. Otherwise + // take the normal path. + let lowered_raw = + match overload_specialization_mangle(cmd, &overload_table) { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!() + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &program.source, + &table, + Some(&mangled), + &putr_map, + &line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &program.source, + &table, + &putr_map, + &line_index, + ), + }; + let rewritten = vw_htcl::rewrite_externs(&lowered_raw); + for name in rewritten.names { + extern_names.insert(name); + } + if rewritten.text.trim().is_empty() { + continue; + } + // Resolve the command's expected return type and, if any, + // wrap the lowered Tcl so it dispatches through the type's + // `repr` proc. The wrapped form runs the user's expression + // into a sentinel local then formats via the repr; the + // sentinel-binding step preserves `set var [...]`-style + // bindings (the user's `$var` still gets the raw value). + let expected_return_type = resolve_return_type(cmd, &table); + let final_tcl = match expected_return_type.as_ref() { + Some(ty) => wrap_with_repr(&rewritten.text, ty, &type_decl_table), + None => rewritten.text, + }; + let is_set_binding = matches!(cmd.kind, vw_htcl::CommandKind::Set); + commands.push(PreparedCommand { + tcl: final_tcl, + origin, + expected_return_type, + is_set_binding, + }); + } + + // No prelude needed in the current architecture: wrappers + // live in the `vivado::` namespace and `extern::name` rewrites + // to `::name`, which Tcl resolves at the global root regardless + // of the calling namespace. We still drain `extern_names` so + // the analyzer can grow future per-extern bookkeeping without + // rewiring this path. + let _ = extern_names; + + let procs = build_proc_locations(&parsed.document, &program, &scratch.path); + // The dedicated pre-flight `collect_warnings` is gone — the + // validator now treats "unknown call with `-flag` args" as a + // hard error and the REPL has already returned via `LowerError` + // above when one fires. + let warnings: Vec = Vec::new(); + + Ok(Prepared { + commands, + batch: SessionBatch { + program, + document: parsed.document, + procs, + }, + warnings, + entry_top_level, + }) +} + +/// Walk every proc declaration (top-level + nested inside +/// `namespace eval` blocks) and record its body's source location +/// keyed by the proc's qualified name. Same recursion shape as +/// `vw_htcl::validate::collect_signatures` — kept in sync by +/// convention rather than refactor so this crate stays a leaf +/// consumer of vw-htcl. +pub fn build_proc_locations( + doc: &vw_htcl::Document, + program: &vw_htcl::LoadedProgram, + scratch_path: &Path, +) -> std::collections::HashMap { + use std::collections::HashMap; + let mut out: HashMap = HashMap::new(); + collect_procs(&doc.stmts, "", program, scratch_path, &mut out); + out +} + +fn collect_procs( + stmts: &[vw_htcl::Stmt], + prefix: &str, + program: &vw_htcl::LoadedProgram, + scratch_path: &Path, + out: &mut std::collections::HashMap, +) { + use vw_htcl::CommandKind; + for stmt in stmts { + let vw_htcl::Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + if let Some(loc) = + proc_body_location(program, proc.body_span, scratch_path) + { + out.insert(qualified, loc); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = qualify(prefix, name); + collect_procs(&ns.body, &nested, program, scratch_path, out); + } + _ => {} + } + } +} + +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +fn proc_body_location( + program: &vw_htcl::LoadedProgram, + body_span: vw_htcl::Span, + scratch_path: &Path, +) -> Option { + let (file_index, file_span) = program.locate_span(body_span)?; + let file = &program.files[file_index]; + let file_path = if file.path == scratch_path { + None + } else { + Some(file.path.clone()) + }; + // Tcl's `(procedure "X" line N)` counts the line **containing + // the opening `{`** as line 1, the next line as line 2, etc. + // `body_span.start` is the byte right after the `{`, so the + // `{` itself sits at `file_span.start - 1`. The line at that + // byte is what Tcl calls "line 1." When the proc body is on a + // single line (`proc f {x} {puts $x}`) that line is also the + // content line. + let brace_pos = file_span.start.saturating_sub(1); + let body_start_line = file_line_at(&file.source, brace_pos); + // For the body_lines vector we want every file line from the + // one with the `{` up to (and including) the one with the + // matching `}` — so `resolve_body_line(N)` returns the + // corresponding source. Anything past the body is irrelevant. + let body_end_line = + file_line_at(&file.source, file_span.end.saturating_sub(1)); + let body_lines: Vec = file + .source + .lines() + .skip(body_start_line.saturating_sub(1) as usize) + .take((body_end_line - body_start_line + 1) as usize) + .map(str::to_string) + .collect(); + Some(ProcLocation { + file: file_path, + body_start_line, + body_lines, + }) +} + +/// Return the declared return type of `cmd`'s head call, when we +/// can resolve it from the signature table. Currently handles two +/// shapes: +/// +/// - Direct call: `proc-name arg arg …` → look up `proc-name`'s +/// return type in the table. +/// - Bracket-bound assignment: `set var [proc-name …]` → look up +/// the inner bracketed call's return type (since `set` returns +/// the value being set, which is the type of the inner call). +/// +/// Anything else (control flow, variable substitution, raw Tcl, +/// unknown commands) returns `None`. The App falls back to the +/// untyped-display path for those. +fn resolve_return_type( + cmd: &vw_htcl::ast::Command, + table: &std::collections::HashMap, +) -> Option { + let head = cmd.words.first()?.as_text()?; + if head == "set" { + // `set var [EXPR]` → recurse into the bracketed + // expression on the third word (words[2]). Other `set` + // shapes (set var literal, set var $other) leave the + // type unknown — we'd need real expression type-inference + // to do better, and that's out of scope for v1. + let val_word = cmd.words.get(2)?; + // Look for a CmdSubst part — `[…]` — at the top of the + // value word. If found, recurse into the bracketed + // command's first statement. + for part in &val_word.parts { + if let vw_htcl::WordPart::CmdSubst { body, .. } = part { + let vw_htcl::Stmt::Command(inner) = body.first()? else { + continue; + }; + return resolve_return_type(inner, table); + } + } + return None; + } + let sig = table.get(head)?; + sig.return_type.clone() +} + +/// Wrap the lowered Tcl `inner` so that, after evaluating it, the +/// result is fed through `::repr` (or the appropriate +/// monomorphized generic repr) to produce a display string. +/// +/// Prepends: +/// 1. The primitive prelude (`string` / `int` / `bool` / `unit` +/// triplets) — cheap to redefine per-eval; Tcl `proc` +/// redefinition is idempotent. +/// 2. Any per-instantiation generic reprs needed for `ty`, in +/// topological order so each proc is defined before its +/// dependents call it. +/// 3. `set __vw_result []` — captures the user expression's +/// raw value into a sentinel local. This preserves any +/// `set var [...]` bindings the user wrote, since `set`'s +/// side effect runs before our sentinel-capture wraps it. +/// 4. ` $__vw_result` — calls the type's repr proc on +/// the captured value. The eval returns this formatted string. +fn wrap_with_repr( + inner: &str, + ty: &vw_htcl::TypeExpr, + types: &std::collections::HashMap, +) -> String { + use std::fmt::Write; + let mut out = String::new(); + for p in vw_htcl::repr::emit_primitive_prelude() { + out.push_str(&p); + } + // Walks the dispatch type's underlying when `ty` is a newtype + // — necessary for `Properties` (newtype wrapping + // `dict`) so the body of `Properties::repr` + // can call the monomorphized `dict_string_Property::repr`. + let emission = vw_htcl::repr::emit_repr_with_types(ty, types); + for p in &emission.procs { + out.push_str(p); + } + writeln!(out, "set __vw_result [{}]", inner.trim_end()) + .expect("writeln to String never fails"); + // All reprs (compiler-emitted primitives + generics + user- + // written newtype reprs + auto-generated enum reprs) share a + // single `{args}` envelope that uses `::vw::kwargs` to bind + // `$v`. The dispatch site always calls them as + // ` -v ` so the kwargs envelope binds + // uniformly regardless of which class of repr is being + // invoked. + write!(out, "{} -v $__vw_result", emission.dispatch) + .expect("write to String never fails"); + out +} + +/// If `cmd` is a top-level `proc` whose name appears in the +/// overload table AND whose first arg is a qualified-variant +/// annotation, return the mangled internal name that this +/// specialization should lower under. Otherwise `None`. +/// +/// This is what reroutes user-written `proc handle_prop {v: +/// Property::Scalar} { … }` from emitting under the literal +/// `handle_prop` name (which would collide with the synthesized +/// dispatcher) to emitting under `__handle_prop__Scalar` (which +/// the dispatcher's switch arm calls). +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +fn build_origin( + program: &vw_htcl::LoadedProgram, + span: vw_htcl::Span, + flat_line: u32, + scratch_path: &Path, +) -> Origin { + // Full span — for a multi-line `set proj [ … ]` the snippet + // includes every line of the command so the trace shows what + // the user actually wrote, not just `set proj [`. The renderer + // is responsible for indenting continuation lines. + let snippet = program.source[span.start as usize..span.end as usize] + .trim_end() + .to_string(); + + if let Some((file_index, file_span)) = program.locate_span(span) { + let file = &program.files[file_index]; + let file_path = if file.path == scratch_path { + None + } else { + Some(file.path.clone()) + }; + let file_line = file_line_at(&file.source, file_span.start); + let via = build_via_chain(program, file_index, scratch_path); + return Origin { + file: file_path, + line: file_line, + snippet, + via, + }; + } + Origin { + file: None, + line: flat_line, + snippet, + via: Vec::new(), + } +} + +/// Walk the loader's import chain from the leaf file back toward +/// the entry, turning each [`vw_htcl::ImportEdge`] into a renderable +/// frame. Nearest first. +fn build_via_chain( + program: &vw_htcl::LoadedProgram, + leaf_file: usize, + scratch_path: &Path, +) -> Vec { + program + .ancestry(leaf_file) + .map(|edge| { + let importer = &program.files[edge.importer_file]; + let line = file_line_at(&importer.source, edge.src_span.start); + let snippet = first_line( + &importer.source, + edge.src_span.start as usize, + edge.src_span.end as usize, + ); + OriginFrame { + file: if importer.path == scratch_path { + None + } else { + Some(importer.path.clone()) + }, + line, + snippet, + } + }) + .collect() +} + +fn first_line(source: &str, start: usize, end: usize) -> String { + let line_end = source[start..].find('\n').map(|n| start + n).unwrap_or(end); + source[start..line_end].trim().to_string() +} + +fn file_line_at(source: &str, offset: u32) -> u32 { + let upto = offset.min(source.len() as u32) as usize; + 1 + source[..upto].bytes().filter(|b| *b == b'\n').count() as u32 +} + +fn render_location( + program: &vw_htcl::LoadedProgram, + span: vw_htcl::Span, + flat_line: u32, + scratch_path: &Path, +) -> String { + if let Some((file_index, file_span)) = program.locate_span(span) { + let file = &program.files[file_index]; + if file.path != scratch_path { + let line = file_line_at(&file.source, file_span.start); + return format!("{}:{line}", file.path.display()); + } + } + format!("(input):{flat_line}") +} + +fn build_resolver(workspace_dir: Option<&Utf8Path>) -> Resolver { + let mut resolver = Resolver::new(); + let Some(ws) = workspace_dir else { + return resolver; + }; + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(ws) { + for (name, path) in paths { + resolver = resolver.with_dep(name, path); + } + } + resolver +} + +struct ScratchFile { + path: PathBuf, +} + +impl ScratchFile { + fn new(dir: &Path, contents: &str) -> std::io::Result { + let name = format!(".vw-repl-input-{}.htcl", std::process::id()); + let path = dir.join(name); + let mut f = std::fs::File::create(&path)?; + f.write_all(contents.as_bytes())?; + Ok(Self { path }) + } +} + +impl Drop for ScratchFile { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_session() -> Session { + Session::new() + } + + /// User-statement commands only — strips the synthetic + /// prelude entries (enum reprs, overload dispatchers, + /// primitive reprs, monomorphized generic reprs) the + /// preparer ships before each batch. Tests that assert + /// command count / shape only care about what the user + /// wrote, not the prelude scaffolding. + fn user_commands(prep: &Prepared) -> Vec<&PreparedCommand> { + prep.commands + .iter() + .filter(|c| !c.origin.snippet.starts_with('<')) + .collect() + } + + #[test] + fn unknown_keyword_call_inside_bracket_errors() { + // Mirrors the metroid project.htcl shape: a call to an + // unknown proc with keyword args, nested inside a `[ … ]` + // substitution. The validator now treats this as a hard + // error so the lowering returns `Err` and nothing ships + // to Vivado — the user is forced to either `src` a + // wrapper module or write `extern::create_project`. + let dir = tempfile::tempdir().unwrap(); + let err = prepare( + "set proj [\n create_project\n -in_memory 1\n -name foo\n]\n", + dir.path(), + &empty_session(), + ) + .unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("create_project"), "{msg}"); + assert!(msg.contains("extern::"), "{msg}"); + } + + /// Regression: a stack frame's line number must resolve to the + /// ACTUAL source line of the failing call, even when the proc body + /// mixes comments, a `\`-continued command, a multi-line `[ … ]` + /// substitution (which the lowering collapses to one line), and a + /// call nested inside an `if` block. The padding in + /// `lower_proc_decl_with_name_and_index` keeps Tcl's body-relative + /// line == source body line, and `ProcLocation::resolve_body_line` + /// depends on that. When the two drift, a Vivado error points at an + /// unrelated line (a comment 39 lines off, in the bug that + /// motivated this) instead of the call. + #[test] + fn stack_frame_line_survives_comments_brackets_and_nesting() { + let src = "\ +namespace eval demo { + proc thing {} unit { + # a comment + # another comment + set x [ helper \\ + -a 1 \\ + -b 2 ] + if {$x} { + # nested comment + boom + } + } +} +"; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("demo.htcl"); + std::fs::write(&path, src).unwrap(); + let program = + vw_htcl::load_program(&path, &vw_htcl::Resolver::new()).unwrap(); + let parsed = vw_htcl::parse(&program.source); + let line_index = vw_htcl::LineIndex::new(&program.source); + let table = vw_htcl::signature_table(&parsed.document); + let putr = vw_htcl::putr::rewrite(&program.source, &parsed.document); + + let plocs = build_proc_locations(&parsed.document, &program, &path); + let loc = plocs.get("demo::thing").expect("proc location"); + + fn find<'a>( + stmts: &'a [vw_htcl::Stmt], + want: &str, + ) -> Option<&'a vw_htcl::Proc> { + for s in stmts { + let vw_htcl::Stmt::Command(c) = s else { + continue; + }; + match &c.kind { + vw_htcl::CommandKind::Proc(p) + if p.name.as_deref() == Some(want) => + { + return Some(p) + } + vw_htcl::CommandKind::NamespaceEval(ns) => { + if let Some(p) = find(&ns.body, want) { + return Some(p); + } + } + _ => {} + } + } + None + } + let proc = find(&parsed.document.stmts, "thing").unwrap(); + let lowered = vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &program.source, + &table, + None, + &putr, + &line_index, + ); + // Tcl reports body-relative lines; lowered line 0 is the + // `{ ::vw::kwargs …` line == Tcl body line 1, so the `boom` + // call at lowered index i is Tcl body line i + 1. + let idx = lowered + .lines() + .position(|l| l.trim() == "boom") + .expect("boom in shipped body"); + let (_line, content) = + loc.resolve_body_line(idx as u32 + 1).expect("resolves"); + assert_eq!( + content.trim(), + "boom", + "stack-frame line drifted: Tcl body line {} resolved to `{}`, \ + not the `boom` call", + idx + 1, + content.trim(), + ); + } + + #[test] + fn extern_prefixed_call_is_accepted() { + // The opt-out: `extern::create_project` is explicitly a + // raw Tcl call, no wrapper required. Lowering strips the + // prefix so the bare native resolves through Tcl's global + // namespace at runtime — no rename plumbing, no prelude. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "extern::create_project -name foo\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1, "{:?}", cmds); + assert!( + cmds[0].tcl.contains("create_project -name foo"), + "{}", + cmds[0].tcl + ); + assert!(!cmds[0].tcl.contains("extern::"), "{}", cmds[0].tcl); + } + + #[test] + fn prior_batch_procs_resolve_in_next_batch() { + // Reproduces the REPL "src @lib then call" pattern: a + // wrapper declared in a previous batch should be visible + // to the analyzer/lowering when we lower a bare call in + // the next batch — and the new batch should ship only + // its own statement (not re-emit the wrapper). + let dir = tempfile::tempdir().unwrap(); + let mut session = Session::new(); + // Batch 1: declare the wrapper. Commit so it joins the + // session — same flow the App follows on successful eval. + let first = prepare( + "namespace eval vivado {\n \ + proc current_project {\n \ + @enum(0, 1) @default(0) quiet\n \ + @enum(0, 1) @default(0) verbose\n \ + @default(\"\") project\n \ + } string {\n \ + set cmd [list ::current_project]\n \ + return [{*}$cmd]\n \ + }\n\ + }\n", + dir.path(), + &session, + ) + .unwrap(); + // The first batch ships its own declaration to the worker + // exactly once — that's what makes the wrapper exist in + // Tcl. Subsequent batches must NOT re-emit it. + assert!( + first + .commands + .iter() + .any(|c| c.tcl.contains("namespace eval")), + "first batch must ship the namespace decl: {:?}", + first.commands + ); + session.commit(first.batch); + + // Batch 2: bare call to the wrapper. Should ship as-is + // (htcl is keyword-only at the call site; the wrapper + // parses its own kwargs at runtime via the ::vw::kwargs + // prelude), with no rewriting and no re-emission of the + // prior batch's declaration. + let prep = + prepare("vivado::current_project\n", dir.path(), &session).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1, "{:?}", cmds); + assert!( + cmds[0].tcl.contains("vivado::current_project"), + "{}", + cmds[0].tcl + ); + // And nothing in the new batch's source mentions the + // wrapper body — we never re-parsed the prior batch. + assert!( + !prep.batch.program.source.contains("namespace eval vivado"), + "{}", + prep.batch.program.source + ); + } + + #[test] + fn known_keyword_call_is_not_errored() { + // When the called proc IS in scope, no error fires. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc create_project { @default(\"\") name } { }\n\ + set proj [ create_project -name foo ]\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + assert!(prep.warnings.is_empty(), "{:?}", prep.warnings); + } + + #[test] + fn lowers_plain_proc_call_to_tcl() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare("puts hello", dir.path(), &empty_session()).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1); + assert!(cmds[0].tcl.contains("puts hello")); + // Input is at line 1 of the buffer. + assert_eq!(cmds[0].origin.line, 1); + assert!(cmds[0].origin.file.is_none()); + assert_eq!(cmds[0].origin.snippet, "puts hello"); + } + + #[test] + fn each_statement_gets_its_own_origin() { + let dir = tempfile::tempdir().unwrap(); + let prep = + prepare("set x 1\nset y 2\nset z 3", dir.path(), &empty_session()) + .unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 3); + assert_eq!(cmds[0].origin.line, 1); + assert_eq!(cmds[1].origin.line, 2); + assert_eq!(cmds[2].origin.line, 3); + } + + #[test] + fn proc_body_line_resolution_matches_tcl_line_counting() { + // Tcl counts the proc-body line **containing the opening + // `{`** as line 1 — so a `(procedure "ip::check" line 2)` + // frame should point at the first content line of the body, + // not the line after it. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + // Lines 1-2: blank + the namespace header; line 3 has `{` + // (the proc body opener); content lives on lines 4+. + std::fs::write( + dep.join("module.htcl"), + "namespace eval foo {\n proc bar {} {\n puts hi\n error oh-no\n }\n}\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + let prep = prepare("src @dep", dir.path(), &empty_session()).unwrap(); + let loc = prep + .batch + .procs + .get("foo::bar") + .expect("expected foo::bar in proc map"); + // The proc body opens on file line 2 (the `} {`-style line + // here is just `proc bar {} {`), so Tcl line 1 → line 2 of + // the file. + assert_eq!(loc.body_start_line, 2); + // Tcl line 2 → file line 3 → `puts hi`. + let (line, content) = loc.resolve_body_line(2).unwrap(); + assert_eq!(line, 3); + assert_eq!(content.trim(), "puts hi"); + // Tcl line 3 → file line 4 → `error oh-no`. + let (line, content) = loc.resolve_body_line(3).unwrap(); + assert_eq!(line, 4); + assert_eq!(content.trim(), "error oh-no"); + } + + #[test] + fn origin_via_chain_walks_back_through_src_imports() { + // entry → mid → leaf, all via `src`. A command in `leaf` + // should carry a 2-frame via chain (mid → entry/input). + let dir = tempfile::tempdir().unwrap(); + let mid_dep = dir.path().join("mid_dep"); + let leaf_dep = dir.path().join("leaf_dep"); + std::fs::create_dir_all(&mid_dep).unwrap(); + std::fs::create_dir_all(&leaf_dep).unwrap(); + std::fs::write(leaf_dep.join("module.htcl"), "set leaf_var 1\n") + .unwrap(); + std::fs::write(mid_dep.join("module.htcl"), "src @leaf\n").unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.mid]\npath = \"{}\"\n\ + [dependencies.leaf]\npath = \"{}\"\n", + mid_dep.display(), + leaf_dep.display() + ), + ) + .unwrap(); + + let prep = prepare("src @mid", dir.path(), &empty_session()).unwrap(); + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 1); + let origin = &cmds[0].origin; + // Leaf-most command lives in leaf_dep/module.htcl. + assert!( + origin + .file + .as_ref() + .unwrap() + .ends_with("leaf_dep/module.htcl"), + "{:?}", + origin.file + ); + // The via chain: leaf was imported by mid (line 1), and mid + // was imported by the entry input (line 1). + assert_eq!(origin.via.len(), 2, "{:?}", origin.via); + assert!(origin.via[0] + .file + .as_ref() + .unwrap() + .ends_with("mid_dep/module.htcl")); + assert_eq!(origin.via[0].snippet, "src @leaf"); + // The outermost frame is the user's input (file = None). + assert!(origin.via[1].file.is_none(), "{:?}", origin.via[1].file); + assert_eq!(origin.via[1].snippet, "src @mid"); + } + + #[test] + fn src_imported_statements_resolve_to_imported_file() { + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "proc hello {} { puts world }\nhello\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + let prep = prepare("src @dep", dir.path(), &empty_session()).unwrap(); + // Two commands from the imported file: `proc hello` and the + // bare `hello` call. Both must carry the imported file's + // path as origin. + let cmds = user_commands(&prep); + assert_eq!(cmds.len(), 2); + for cmd in &cmds { + let file = cmd.origin.file.as_ref().expect("import has file"); + assert!(file.ends_with("dep/module.htcl"), "{:?}", file); + } + // Line numbers point into the imported file. + assert_eq!(cmds[0].origin.line, 1); + assert_eq!(cmds[1].origin.line, 2); + } + + #[test] + fn second_batch_parses_only_its_own_files() { + // Regression guard against the lag bug: after `src @dep` + // commits, a subsequent bare call must NOT cause the + // loader to re-parse the dep's files. We assert by hooking + // the loader's per-file observer and counting parses on + // each batch. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("dep"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "namespace eval lib {\n \ + proc f { @default(0) x } int { return $x }\n\ + }\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.dep]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + #[derive(Default)] + struct Counter { + parsed: Vec, + } + impl vw_htcl::LoadObserver for Counter { + fn on_parsed(&mut self, file: &Path, _raw: Option<&str>) { + self.parsed.push(file.to_path_buf()); + } + } + + let mut session = Session::new(); + + // First batch: imports the dep. Two files parse — the + // entry scratch and the dep's module.htcl. + let mut counter = Counter::default(); + let first = prepare_with_observer( + "src @dep\n", + dir.path(), + &session, + &mut counter, + ) + .unwrap(); + assert_eq!( + counter.parsed.len(), + 2, + "first batch should parse entry + dep, got {:?}", + counter.parsed + ); + session.commit(first.batch); + + // Second batch: bare call to the wrapper. The prior + // batch's signatures are merged in via `session`, so the + // loader must NOT re-read the dep's file — only the new + // scratch parses. + let mut counter = Counter::default(); + let _second = prepare_with_observer( + "lib::f -x 1\n", + dir.path(), + &session, + &mut counter, + ) + .unwrap(); + assert_eq!( + counter.parsed.len(), + 1, + "second batch should parse only the new input, got {:?}", + counter.parsed + ); + // And the one file parsed is the scratch, not the dep. + let only = &counter.parsed[0]; + assert!( + !only.starts_with(&dep), + "the dep's files must not be re-parsed on a fresh \ + batch: {:?}", + only + ); + } + + #[test] + fn prior_batch_proc_location_survives_for_drilldown() { + // The user-reported bug: `src @vivado-cmd` declares + // `vivado::create_bd_design` in batch A, then a later + // `vivado::create_bd_design -name metroid` fires in batch + // B and the Tcl error frame names that proc. The + // proc-location lookup must resolve to the REAL .htcl + // file the wrapper came from — not the disposable scratch + // path of either batch. + let dir = tempfile::tempdir().unwrap(); + let dep = dir.path().join("vivado_cmd"); + std::fs::create_dir_all(&dep).unwrap(); + std::fs::write( + dep.join("module.htcl"), + "namespace eval vivado {\n \ + proc create_bd_design {\n \ + @default(\"\") name\n \ + } string {\n \ + set cmd [list ::create_bd_design]\n \ + return [{*}$cmd]\n \ + }\n\ + }\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("vw.toml"), + format!( + "[workspace]\nname=\"t\"\nversion=\"0.1.0\"\n\n\ + [dependencies.vivado-cmd]\npath = \"{}\"\n", + dep.display() + ), + ) + .unwrap(); + + let mut session = Session::new(); + // Batch A: pull the wrapper in. + let first = prepare("src @vivado-cmd\n", dir.path(), &session).unwrap(); + session.commit(first.batch); + + // Batch B: call the wrapper. Look up its location through + // the session — which is exactly the path the App's error + // renderer takes when resolving a Tcl drill-down frame. + let _second = prepare( + "vivado::create_bd_design -name metroid\n", + dir.path(), + &session, + ) + .unwrap(); + let loc = session.lookup_proc("vivado::create_bd_design").expect( + "wrapper from a prior `src @vivado-cmd` batch must be \ + reachable through session.lookup_proc", + ); + // The crucial assertion: the file pointer is the REAL + // imported .htcl, not `None` (the scratch) and not some + // huge synthetic offset. + let file = loc.file.as_ref().expect( + "wrapper from imported module must carry its real \ + .htcl path, not the disposable scratch", + ); + assert!( + file.ends_with("vivado_cmd/module.htcl"), + "expected the imported module path, got {:?}", + file + ); + // And `body_start_line` is the file-local line of the + // proc body opener — small, not a combined-scratch offset. + assert!( + loc.body_start_line < 100, + "body_start_line should be a small file-local number, \ + got {}", + loc.body_start_line + ); + } + + // --- typed-expression wrap (step 3) ---------------------------- + + #[test] + fn typed_proc_call_wraps_with_repr_dispatch() { + let dir = tempfile::tempdir().unwrap(); + // A proc annotated dict, called bare. The + // wrap should: + // - capture the call's result into __vw_result + // - invoke the monomorphized dict repr proc on it + // PreparedCommand.expected_return_type carries the type. + let prep = prepare( + "proc props {} dict { return {} }\n\ + props\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // Two commands: proc decl (drops to empty Tcl) + call. + // proc decl ships as a regular command; the call should + // be wrapped. + let call = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected the `props` call to be repr-wrapped"); + assert!( + call.tcl.contains("set __vw_result [props]"), + "tcl: {}", + call.tcl + ); + assert!( + call.tcl + .contains("dict_string_string::repr -v $__vw_result"), + "tcl: {}", + call.tcl + ); + // Primitive prelude is included so the dict repr's + // element calls (string::repr) resolve. Both the + // primitive procs and the monomorphized generic procs + // are wrapped in explicit `namespace eval` blocks so + // Tcl's namespace-conflict heuristic doesn't reject the + // declaration (the bare `proc string::repr` form trips + // over Tcl's built-in `string` command). + assert!( + call.tcl.contains("namespace eval string"), + "expected primitive prelude in wrapped tcl: {}", + call.tcl + ); + // Plus the dict repr itself. + assert!( + call.tcl.contains("namespace eval dict_string_string"), + "expected monomorphized dict repr: {}", + call.tcl + ); + // The expected_return_type rides along for App-side use. + let ty = call + .expected_return_type + .as_ref() + .expect("expected_return_type set"); + match ty { + vw_htcl::TypeExpr::Generic { name, args, .. } => { + assert_eq!(name, "dict"); + assert_eq!(args.len(), 2); + } + _ => panic!("expected Generic, got {:?}", ty), + } + } + + #[test] + fn set_var_call_inherits_inner_return_type() { + // `set cips [props]` — `set` returns the value being set, + // so its type is whatever `props` returns. The wrap should + // bind `$cips` correctly AND dispatch on the inner call's + // declared type. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc props {} dict { return {} }\n\ + set x [props]\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let set_cmd = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected the `set x [...]` to be repr-wrapped"); + assert!( + set_cmd.tcl.contains("set __vw_result [set x [props]]"), + "expected the original set to be inner-wrapped: {}", + set_cmd.tcl + ); + assert!( + set_cmd + .tcl + .contains("dict_string_string::repr -v $__vw_result"), + "tcl: {}", + set_cmd.tcl + ); + } + + #[test] + fn unannotated_call_is_not_wrapped() { + // No return type → no wrap, no `__vw_result` capture, + // and `expected_return_type` is None. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc plain {} { puts whatever }\n\ + plain\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let plain_call = prep + .commands + .iter() + .find(|c| c.tcl.trim() == "plain") + .expect("expected raw `plain` call without wrap"); + assert!(plain_call.expected_return_type.is_none()); + assert!( + !plain_call.tcl.contains("__vw_result"), + "unannotated calls shouldn't get the repr wrap: {}", + plain_call.tcl + ); + } + + #[test] + fn unit_typed_call_is_wrapped_with_unit_dispatch() { + // `unit`-typed expressions still get wrapped — the wrap + // returns the empty string from `unit::repr`. The App's + // EvalDone handler is what suppresses the Result push; + // the lowerer is uniform. + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "proc do_thing {} unit { puts hi }\n\ + do_thing\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + let call = prep + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected repr-wrap on unit-typed call"); + assert!(call.tcl.contains("unit::repr -v $__vw_result")); + let ty = call.expected_return_type.as_ref().unwrap(); + match ty { + vw_htcl::TypeExpr::Named { name, .. } => { + assert_eq!(name, "unit"); + } + _ => panic!(), + } + } + + // --- enum / overload pipeline (step 5) ------------------------- + + #[test] + fn enum_decl_ships_namespace_eval_prelude() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "enum Direction = {\n North\n South\n}\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // The prelude is shipped as a synthetic PreparedCommand. + let prelude = prep + .commands + .iter() + .find(|c| c.tcl.contains("namespace eval Direction")) + .expect("expected enum prelude in prepared commands"); + assert!(prelude.tcl.contains("proc North {}")); + assert!(prelude.tcl.contains("proc South {}")); + assert!(prelude.tcl.contains("proc tag {v}")); + assert!(prelude.tcl.contains("proc payload {v}")); + assert!(prelude.tcl.contains("proc repr {args}")); + } + + #[test] + fn overload_set_ships_dispatcher_and_mangled_specializations() { + let dir = tempfile::tempdir().unwrap(); + let prep = prepare( + "enum E = {\n A: string\n B: int\n}\n\ + proc f {v: E::A} string { return $v }\n\ + proc f {v: E::B} string { return $v }\n", + dir.path(), + &empty_session(), + ) + .unwrap(); + // Dispatcher emitted with the switch body. The dispatcher + // takes the standard kwargs envelope (`{args}`), walks + // kwargs for `-v `, then switches on the + // tag. + let dispatcher = prep + .commands + .iter() + .find(|c| { + c.tcl.contains("proc f {args}") + && c.tcl.contains("switch") + && c.tcl.contains("__f__") + }) + .expect("expected dispatcher for `f`"); + assert!(dispatcher.tcl.contains("__f__A")); + assert!(dispatcher.tcl.contains("__f__B")); + // Specializations emitted under mangled names — look for + // the `proc __f__A` declaration (rather than `proc f`). + assert!( + prep.commands + .iter() + .any(|c| c.tcl.contains("proc __f__A {args}")), + "expected specialization under __f__A: tcls={:?}", + prep.commands.iter().map(|c| &c.tcl).collect::>() + ); + assert!( + prep.commands + .iter() + .any(|c| c.tcl.contains("proc __f__B {args}")), + "expected specialization under __f__B" + ); + // The user-visible name `f` should NOT appear as a + // user-procedure declaration — only as the dispatcher. + // (The dispatcher's body has `proc f {v args}` which we + // already accounted for above; what we're guarding + // against is a leaked `proc f {args} { ::vw::kwargs ... }` + // specialization.) + let leaked_f = prep + .commands + .iter() + .filter(|c| c.tcl.contains("proc f {args} { ::vw::kwargs")) + .count(); + assert_eq!( + leaked_f, 0, + "specialization should NOT have leaked under public name `f`" + ); + } + + #[test] + fn cross_batch_newtype_recursion_emits_generic_repr() { + // Reproduces the user-reported regression: batch 1 + // declares `type Properties = dict` and a + // proc returning Properties; batch 2 calls that proc. + // The wrap_with_repr in batch 2 should walk Properties's + // underlying (the dict generic) and emit + // `dict_string_string::repr` so the user's + // `Properties::repr` body can find it. + // + // Pre-fix: type_decl_table was per-batch, so batch 2 + // couldn't see Properties; the recursion didn't fire; + // dict_string_string::repr was never emitted; the + // user's body errored with `invalid command name`. + let dir = tempfile::tempdir().unwrap(); + let mut session = Session::new(); + let first = prepare( + "type Properties = {dict}\n\ + proc Properties::repr {v} { return $v }\n\ + proc Properties::from {v} { return $v }\n\ + proc Properties::to {v} { return $v }\n\ + proc get_props {} Properties { return {a 1 b 2} }\n", + dir.path(), + &session, + ) + .unwrap(); + session.commit(first.batch); + + // Batch 2: just the call. parsed.document doesn't have + // the type decl — it must come from `session`. + let second = prepare("get_props\n", dir.path(), &session).unwrap(); + let call = second + .commands + .iter() + .find(|c| c.tcl.contains("__vw_result")) + .expect("expected wrapped call to get_props"); + // The wrap must include the monomorphized dict repr + // (reached by recursing through Properties's underlying). + assert!( + call.tcl.contains("namespace eval dict_string_string"), + "expected dict_string_string::repr in wrap (newtype \ + recursion across batches): {}", + call.tcl + ); + // And the top-level dispatch goes through Properties::repr + // with the `-v` form, not positional. + assert!( + call.tcl.contains("Properties::repr -v $__vw_result"), + "expected Properties::repr dispatch via -v form: {}", + call.tcl + ); + } +} diff --git a/vw-repl/src/popup.rs b/vw-repl/src/popup.rs new file mode 100644 index 0000000..efa1679 --- /dev/null +++ b/vw-repl/src/popup.rs @@ -0,0 +1,758 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Overlay popups attached to the input editor — completion (slice 4), +//! signature help (slice 5), hover (slice 6). +//! +//! Each popup is a [`PopupState`] variant held on the App as +//! `Option`. The key handler in +//! [`crate::app::App::handle_terminal_event`] intercepts navigation +//! and dismissal keys when a popup is active, BEFORE the catch-all +//! editor handoff — so Up/Down/Enter/Esc go to the popup rather than +//! moving the text cursor. +//! +//! Popups render in [`crate::ui::draw`] as an additional pass on top +//! of the input editor; anchor coordinates are derived from the +//! cursor's screen position so they appear next to where the user is +//! typing. +//! +//! Coexistence: only one popup is active at a time. Completion +//! takes precedence over signature help; both dismiss when hover +//! opens; any popup dismisses on Esc. + +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, +}; +use ratatui::Frame; +use vw_htcl::complete::{Completion, CompletionKind}; + +/// One of the popup kinds the input editor can show. +#[derive(Debug)] +pub enum PopupState { + Completion(CompletionPopup), + SignatureHelp(SignatureHelpPopup), + Hover(HoverPopup), + Help(HelpPopup), + /// Fuzzy symbol picker (Ctrl-T). The picker owns its own + /// matcher state and rendering; this enum just multiplexes + /// into [`crate::symbol_search`]. + SymbolSearch(crate::symbol_search::SymbolPicker), + /// Fuzzy diagnostic finder (Ctrl-F). Snapshots scrollback + /// Error/Warning/Notice entries; Enter jumps the viewport to + /// the chosen entry and drops a persistent left-gutter + /// marker so the user can find it in a busy log. + DiagnosticSearch(crate::diag_search::DiagnosticPicker), +} + +/// Keybinding cheat-sheet shown by Ctrl-H. Lists every key chord the +/// REPL responds to, with a short description, so users don't have +/// to read the source to discover features. Any key dismisses. +#[derive(Debug)] +pub struct HelpPopup; + +/// Per-keystroke signature help — shows the proc's args + active +/// parameter while the user is typing arg values. Owns its strings +/// so we can store it on the App across frames without borrowing +/// from any short-lived `vw_htcl` document. Recomputed on every +/// buffer-mutating keystroke. +#[derive(Clone, Debug)] +pub struct SignatureHelpPopup { + pub proc_name: String, + pub args: Vec, + pub return_type: Option, + pub doc_brief: Option, + /// Index into `args` of the parameter under the cursor. + pub active: Option, + /// Cursor cell where the popup should anchor (above the cursor + /// — the renderer flips to below if there's no room above). + pub anchor: (u16, u16), + /// First arg index to show — wide-signature procs + /// (`create_cpm5_*` has 50+ args) overflow the popup's vertical + /// budget and get truncated. Ctrl-↑ / Ctrl-↓ adjusts this so + /// the user can scroll through the full list. Preserved across + /// `refresh_signature_help` rebuilds (App reads the old popup's + /// value before overwriting) so manual scrolling sticks while + /// the user is typing. + pub scroll_offset: usize, +} + +#[derive(Clone, Debug)] +pub struct SigHelpArg { + pub name: String, + pub type_str: Option, + /// Pre-formatted `@default(...)` value when the arg declares + /// one. Already truncated by the caller (see + /// `crate::app::format_default_value`) to a sensible width so + /// the popup doesn't blow wide on multi-KB paired-dict + /// defaults from generated IP wrappers. + pub default_str: Option, +} + +/// Hover popup — Ctrl-K opens it, any keystroke dismisses. Shows the +/// proc/var the cursor sits on, with full doc-comment body when +/// available. Distinct from signature help: hover is explicit + shows +/// the FULL docs; sig help is auto + shows a one-line brief. +#[derive(Clone, Debug)] +pub struct HoverPopup { + /// First line — usually the proc signature or `$variable: type`. + pub title: String, + /// Full doc body, reflowed. May be empty. + pub body: String, + pub anchor: (u16, u16), +} + +/// Render the hover popup at the cursor anchor. Sized to fit the +/// content, capped to ~70% of the frame so a long doc doesn't +/// swallow the whole screen. +pub fn draw_hover_popup(f: &mut Frame, popup: &HoverPopup, frame_area: Rect) { + let max_width = ((frame_area.width as usize) * 7 / 10).max(40); + let max_height = ((frame_area.height as usize) * 6 / 10).max(8); + + // Wrap body lines to fit max_width-2 (border padding). + let inner_w = max_width.saturating_sub(2); + let mut body_lines: Vec = Vec::new(); + for paragraph in popup.body.split('\n') { + if paragraph.is_empty() { + body_lines.push(String::new()); + continue; + } + let mut line = String::new(); + for word in paragraph.split_whitespace() { + if line.chars().count() + word.chars().count() + 1 > inner_w { + body_lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + body_lines.push(line); + } + } + let total_lines = 1 + body_lines.len(); // title + body + let height = (total_lines + 2).min(max_height) as u16; + let width = max_width as u16; + + let (anchor_x, anchor_y) = popup.anchor; + let y = if anchor_y + height < frame_area.y + frame_area.height { + (anchor_y + 1) + .min(frame_area.y + frame_area.height.saturating_sub(height)) + } else if anchor_y >= frame_area.y + height { + anchor_y.saturating_sub(height) + } else { + frame_area.y + }; + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let area = Rect { + x, + y, + width: width.min(frame_area.width), + height, + }; + f.render_widget(Clear, area); + + let mut lines: Vec> = Vec::new(); + lines.push(Line::from(Span::styled( + popup.title.clone(), + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD), + ))); + for body in body_lines { + lines.push(Line::from(Span::styled( + body, + Style::default().fg(Color::Gray), + ))); + } + + let para = Paragraph::new(lines).block( + Block::default() + .borders(Borders::ALL) + .title(" hover — any key to dismiss "), + ); + f.render_widget(para, area); +} + +/// Render the signature help popup as an overlay anchored just above +/// the cursor cell. If there's not enough vertical room above, the +/// renderer flips it below. +pub fn draw_signature_help_popup( + f: &mut Frame, + popup: &SignatureHelpPopup, + frame_area: Rect, +) { + // Multi-line layout: proc name on line 1, then one indented + // arg per line, then `→ return_type` (when annotated), then an + // optional doc-brief line. + // + // Single-line layouts blew past the popup width on procs with + // many args — `create_versal_cips` has ~20 — and ratatui + // truncates beyond the box. One-arg-per-line keeps each arg + // readable and scales naturally. The active-arg highlighting + // (bold + underline on the arg's name) still works per-line. + let name_style_keyword = Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD); + let arg_name_style_default = Style::default().fg(Color::Gray); + let arg_name_style_active = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD | Modifier::UNDERLINED); + let type_style = Style::default().fg(Color::Rgb(100, 200, 200)); + let dim = Style::default().add_modifier(Modifier::DIM); + let default_style = Style::default() + .fg(Color::Rgb(180, 200, 130)) + .add_modifier(Modifier::DIM); + + // Build the proc name line (always visible — never scrolled + // off) and the body lines (arg lines + return + doc) as + // separate vectors so we can apply scroll_offset to the body + // independently of the title. + let name_line = + Line::from(Span::styled(popup.proc_name.clone(), name_style_keyword)); + let mut body_lines: Vec> = Vec::new(); + for (i, arg) in popup.args.iter().enumerate() { + let active = popup.active == Some(i); + let mut spans: Vec> = vec![Span::raw(" ")]; + let name_style = if active { + arg_name_style_active + } else { + arg_name_style_default + }; + spans.push(Span::styled(format!("-{}", arg.name), name_style)); + if let Some(ty) = arg.type_str.as_deref() { + spans.push(Span::styled(": ".to_string(), dim)); + spans.push(Span::styled(ty.to_string(), type_style)); + } + if let Some(default) = arg.default_str.as_deref() { + spans.push(Span::styled(" = ".to_string(), dim)); + spans.push(Span::styled(default.to_string(), default_style)); + } + body_lines.push(Line::from(spans)); + } + if let Some(ret) = popup.return_type.as_deref() { + body_lines.push(Line::from(vec![ + Span::styled(" → ".to_string(), dim), + Span::styled(ret.to_string(), type_style), + ])); + } + if let Some(brief) = popup.doc_brief.as_deref() { + if !brief.is_empty() { + body_lines.push(Line::from("")); + body_lines.push(Line::from(Span::styled( + brief.to_string(), + Style::default().fg(Color::Gray), + ))); + } + } + + // Body budget: total height budget minus borders, the proc + // name row, and (when scrolled / overflowing) one row for the + // "(N more · ctrl-↑/↓ to scroll)" footer hint. + let max_height_rows = ((frame_area.height as usize) * 7 / 10).max(4); + let chrome_rows = 2 /* borders */ + 1 /* proc name */; + let max_body_rows = max_height_rows.saturating_sub(chrome_rows); + + // Apply scroll_offset, clamping so we never scroll past a point + // where the visible window would underfill: the user can scroll + // until the LAST body line is at the bottom of the window. + let total_body = body_lines.len(); + let needs_scroll_hint = total_body > max_body_rows; + let visible_body_rows = if needs_scroll_hint { + max_body_rows.saturating_sub(1) // reserve one row for the hint + } else { + max_body_rows + }; + let max_offset = total_body.saturating_sub(visible_body_rows); + let scroll_offset = popup.scroll_offset.min(max_offset); + let body_end = (scroll_offset + visible_body_rows).min(total_body); + let mut lines: Vec> = Vec::with_capacity( + 1 + (body_end - scroll_offset) + needs_scroll_hint as usize, + ); + lines.push(name_line); + lines.extend(body_lines[scroll_offset..body_end].iter().cloned()); + if needs_scroll_hint { + let visible_lo = scroll_offset + 1; + let visible_hi = body_end; + let hint = format!( + " … showing {visible_lo}-{visible_hi} of {total_body} \ + · shift-↑/shift-↓ to scroll", + ); + lines.push(Line::from(Span::styled( + hint, + Style::default().add_modifier(Modifier::DIM), + ))); + } + + // Width: longest line + borders, capped at 70% of frame so + // wide types don't push the popup off-screen. + let widest: usize = lines + .iter() + .map(|l| { + l.spans + .iter() + .map(|s| s.content.chars().count()) + .sum::() + }) + .max() + .unwrap_or(20); + let max_width = ((frame_area.width as usize) * 7 / 10).max(40); + let width = ((widest + 2).clamp(20, max_width)) as u16; + // Recompute height now that the line list is bounded. + let height = ((lines.len() + 2) as u16) + .min(frame_area.height.saturating_sub(1).max(3)); + + let (anchor_x, anchor_y) = popup.anchor; + // Anchor ABOVE cursor when room exists (don't fight the completion + // popup which anchors below). Falls back to below, then clamps to + // the frame bottom — never lets the rect extend past the buffer + // (would panic inside ratatui's `render_widget`). + let y_above = anchor_y.saturating_sub(height); + let y_below = anchor_y.saturating_add(1); + let y = if anchor_y >= frame_area.y + height { + y_above + } else { + y_below + }; + let max_y = frame_area + .y + .saturating_add(frame_area.height) + .saturating_sub(height); + let y = y.min(max_y).max(frame_area.y); + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let area = Rect { + x, + y, + width, + height, + }; + + f.render_widget(Clear, area); + let para = Paragraph::new(lines) + .block(Block::default().borders(Borders::ALL).title(" signature ")); + f.render_widget(para, area); +} + +/// One row in the cheat-sheet. `keys` is shown left-aligned in its +/// own column; `description` fills the rest of the row. +struct HelpRow { + keys: &'static str, + description: &'static str, +} + +/// The full keybinding + meta-command catalog. Keep in sync with +/// `crate::app::App::handle_terminal_event` (key chords) and +/// `crate::app::META_COMMANDS` (the `:foo` REPL commands) — every +/// recognized chord and meta-command should appear here so the +/// help is authoritative. +/// +/// Rows are grouped with blank-spacer rows between sections; the +/// renderer treats `("", "")` as a section separator. +const HELP_ROWS: &[HelpRow] = &[ + // --- popups + completion --- + HelpRow { + keys: "Ctrl-H", + description: "show this help", + }, + HelpRow { + keys: "Tab", + description: "open completion popup (procs, flags, :commands)", + }, + HelpRow { + keys: "↑ / ↓ / Enter", + description: "navigate / accept inside any popup", + }, + HelpRow { + keys: "Esc", + description: "dismiss popup or reverse search", + }, + HelpRow { + keys: "", + description: "", + }, + // --- input / submit --- + HelpRow { + keys: "Enter", + description: "submit input (newline if parse incomplete)", + }, + HelpRow { + keys: "Shift+Enter, Ctrl+Enter, Alt+Enter", + description: "insert literal newline (Shift+Enter reaches the app as Ctrl+J on legacy-encoding terminals; we bind both)", + }, + HelpRow { + keys: "Ctrl-P / Ctrl-N", + description: "prev / next in input history", + }, + HelpRow { + keys: "Ctrl-R", + description: "reverse-search history", + }, + HelpRow { + keys: "Ctrl-C", + description: "clear current input", + }, + HelpRow { + keys: "", + description: "", + }, + // --- scrollback --- + HelpRow { + keys: "Alt-K, PageUp", + description: "scroll scrollback up", + }, + HelpRow { + keys: "Alt-J, PageDown", + description: "scroll scrollback down", + }, + HelpRow { + keys: "End, Ctrl-G", + description: "jump to bottom (resume tail-follow)", + }, + HelpRow { + keys: "Mouse wheel", + description: "scroll scrollback (mouse capture must be on)", + }, + HelpRow { + keys: "Mouse drag", + description: "select for clipboard copy (auto-scrolls past edges)", + }, + HelpRow { + keys: "Shift + click", + description: "expand / collapse a ▶ / ▼ non-diagnostic block", + }, + HelpRow { + keys: "Ctrl-C (during eval)", + description: "cancel the running Vivado eval (session survives)", + }, + HelpRow { + keys: "F2", + description: "toggle mouse capture (mouse-app vs terminal-native)", + }, + HelpRow { + keys: "", + description: "", + }, + // --- session / exit --- + HelpRow { + keys: "Ctrl-D", + description: "exit REPL", + }, + HelpRow { + keys: ":quit / :q / :exit", + description: "exit REPL via meta-command", + }, + HelpRow { + keys: ":load ", + description: "evaluate the contents of a file in this session", + }, + HelpRow { + keys: ":libs", + description: "list loaded libraries and their symbol counts", + }, + HelpRow { + keys: ":restart", + description: + "restart the Vivado worker (stubbed — not yet implemented)", + }, + HelpRow { + keys: "", + description: "", + }, + HelpRow { + keys: "(auto)", + description: "signature help shows while you type a call's args", + }, + HelpRow { + keys: "Shift-↑ / Shift-↓", + description: "scroll the signature-help popup (long arg lists)", + }, + HelpRow { + keys: "Ctrl-Y", + description: "hover docs under cursor", + }, + HelpRow { + keys: "Ctrl-S", + description: "fuzzy symbol search (Tab toggles libraries view)", + }, + HelpRow { + keys: "Ctrl-F", + description: "find diagnostic in scrollback (Ctrl-E/K/W/N toggle Error/Critical/Warning/Info filters; Enter jumps + marks)", + }, + HelpRow { + keys: "Alt-C", + description: "clear the diagnostic-finder jump marker", + }, +]; + +/// Render the help modal centered in the frame. Width fits the +/// longest row + small padding; height fits all rows + borders. +pub fn draw_help_popup(f: &mut Frame, frame_area: Rect) { + let key_col_w: usize = HELP_ROWS + .iter() + .map(|r| r.keys.chars().count()) + .max() + .unwrap_or(8); + let desc_col_w: usize = HELP_ROWS + .iter() + .map(|r| r.description.chars().count()) + .max() + .unwrap_or(20); + // +4 = " │ " separator + outer 1-cell padding on each side. + let width = ((key_col_w + desc_col_w + 6).clamp(40, 80)) as u16; + let height = (HELP_ROWS.len() + 2) as u16; // +2 borders + let x = frame_area.x + frame_area.width.saturating_sub(width) / 2; + let y = frame_area.y + frame_area.height.saturating_sub(height) / 2; + let area = Rect { + x, + y, + width: width.min(frame_area.width), + height: height.min(frame_area.height), + }; + f.render_widget(Clear, area); + + let key_style = Style::default() + .fg(Color::Rgb(180, 130, 220)) + .add_modifier(Modifier::BOLD); + let desc_style = Style::default().fg(Color::Gray); + let sep_style = Style::default().add_modifier(Modifier::DIM); + + let lines: Vec> = HELP_ROWS + .iter() + .map(|row| { + if row.keys.is_empty() && row.description.is_empty() { + return Line::from(""); + } + let pad = key_col_w.saturating_sub(row.keys.chars().count()); + Line::from(vec![ + Span::styled(format!(" {}", row.keys), key_style), + Span::raw(" ".repeat(pad)), + Span::styled(" │ ", sep_style), + Span::styled(row.description.to_string(), desc_style), + ]) + }) + .collect(); + + let para = Paragraph::new(lines).block( + Block::default() + .borders(Borders::ALL) + .title(" help — press any key to dismiss "), + ); + f.render_widget(para, area); +} + +/// Completion popup state. Owns the list of items the call to +/// `vw_htcl::complete_at` produced, plus the currently-selected +/// index and the cursor offset at which the popup was anchored +/// (used by Enter to know what range to replace). +#[derive(Debug)] +pub struct CompletionPopup { + pub items: Vec, + pub selected: usize, + /// Screen-anchor cell — where the popup's top-left should appear + /// relative to the frame. Set by the caller (which knows the + /// terminal cursor position) and used by the renderer. + pub anchor: (u16, u16), +} + +impl CompletionPopup { + pub fn new(items: Vec, anchor: (u16, u16)) -> Option { + if items.is_empty() { + None + } else { + Some(Self { + items, + selected: 0, + anchor, + }) + } + } + + pub fn move_up(&mut self) { + if self.selected > 0 { + self.selected -= 1; + } + } + + pub fn move_down(&mut self) { + if self.selected + 1 < self.items.len() { + self.selected += 1; + } + } + + /// The item the user would accept by pressing Enter. + pub fn current(&self) -> Option<&Completion> { + self.items.get(self.selected) + } +} + +/// Render a completion popup as an overlay. Sized to fit a sensible +/// row count from the available frame height (10 rows default, less +/// when near the bottom edge). Anchored just below-and-right of the +/// cursor position the popup was created at; clamped to stay inside +/// the frame. +pub fn draw_completion_popup( + f: &mut Frame, + popup: &CompletionPopup, + frame_area: Rect, +) { + if popup.items.is_empty() { + return; + } + let max_rows = popup.items.len().min(10) as u16; + // Width: longest label + space + longest detail + padding, capped + // at 60 cells. + let max_label = popup + .items + .iter() + .map(|c| c.label.chars().count()) + .max() + .unwrap_or(8); + let max_detail = popup + .items + .iter() + .map(|c| c.detail.as_deref().unwrap_or("").chars().count()) + .max() + .unwrap_or(0); + let width = (max_label + max_detail + 4).clamp(20, 60) as u16; + let height = max_rows + 2; // +2 for borders + + let (anchor_x, anchor_y) = popup.anchor; + let x = anchor_x.min(frame_area.x + frame_area.width.saturating_sub(width)); + let y = (anchor_y + 1) + .min(frame_area.y + frame_area.height.saturating_sub(height)); + let area = Rect { + x, + y, + width, + height, + }; + + f.render_widget(Clear, area); + + let items: Vec = popup + .items + .iter() + .enumerate() + .map(|(i, item)| { + let kind_glyph = match item.kind { + CompletionKind::Proc => "·", + CompletionKind::Flag => "-", + CompletionKind::EnumValue => "=", + CompletionKind::Constructor => "+", + }; + let kind_style = match item.kind { + CompletionKind::Proc => { + Style::default().fg(Color::Rgb(230, 200, 120)) + } + CompletionKind::Flag => { + Style::default().fg(Color::Rgb(180, 130, 220)) + } + CompletionKind::EnumValue => { + Style::default().fg(Color::Rgb(100, 200, 200)) + } + CompletionKind::Constructor => { + Style::default().fg(Color::Rgb(120, 200, 140)) + } + }; + let detail = item + .detail + .as_deref() + .map(|d| format!(" {d}")) + .unwrap_or_default(); + let mut spans = vec![ + Span::styled(format!("{kind_glyph} "), kind_style), + Span::styled( + item.label.clone(), + if i == popup.selected { + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::White) + } else { + Style::default().fg(Color::Gray) + }, + ), + ]; + if !detail.is_empty() { + spans.push(Span::styled( + detail, + Style::default().add_modifier(Modifier::DIM), + )); + } + let style = if i == popup.selected { + Style::default().bg(Color::Rgb(40, 40, 60)) + } else { + Style::default() + }; + ListItem::new(Line::from(spans)).style(style) + }) + .collect(); + + let mut list_state = ListState::default(); + list_state.select(Some(popup.selected)); + + let list = List::new(items) + .block( + Block::default() + .borders(Borders::ALL) + .title(" completions "), + ) + .highlight_style(Style::default().bg(Color::Rgb(40, 40, 60))); + f.render_stateful_widget(list, area, &mut list_state); +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::span::Span as HtclSpan; + + fn item(label: &str, kind: CompletionKind) -> Completion { + Completion { + label: label.to_string(), + kind, + detail: None, + documentation: None, + replace: HtclSpan { start: 0, end: 0 }, + insert_text: None, + snippet: false, + } + } + + #[test] + fn new_returns_none_on_empty() { + assert!(CompletionPopup::new(Vec::new(), (0, 0)).is_none()); + } + + #[test] + fn move_up_down_clamps() { + let items = vec![ + item("a", CompletionKind::Proc), + item("b", CompletionKind::Proc), + ]; + let mut p = CompletionPopup::new(items, (0, 0)).unwrap(); + assert_eq!(p.selected, 0); + p.move_up(); + assert_eq!(p.selected, 0); // can't go below 0 + p.move_down(); + assert_eq!(p.selected, 1); + p.move_down(); + assert_eq!(p.selected, 1); // can't go past last + p.move_up(); + assert_eq!(p.selected, 0); + } + + #[test] + fn current_returns_selected_item() { + let items = vec![ + item("a", CompletionKind::Proc), + item("b", CompletionKind::Flag), + ]; + let mut p = CompletionPopup::new(items, (0, 0)).unwrap(); + assert_eq!(p.current().unwrap().label, "a"); + p.move_down(); + assert_eq!(p.current().unwrap().label, "b"); + } +} diff --git a/vw-repl/src/render.rs b/vw-repl/src/render.rs new file mode 100644 index 0000000..aba65bf --- /dev/null +++ b/vw-repl/src/render.rs @@ -0,0 +1,757 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Scrollback rendering helpers shared between `ui::draw_scrollback` +//! and `App` mouse-selection. Both need the same view of "how does the +//! scrollback look on screen, row by row" — the UI to render it, the +//! App to map mouse clicks to text positions and extract the selected +//! substring on copy. +//! +//! The flow is: [`entry_lines`] turns each `ScrollbackEntry` into one +//! styled [`Line`] per source line; [`wrap_lines`] then breaks each of +//! those at the rendered column width into screen-row–sized chunks. +//! After wrapping, screen-row N is `wrapped[scroll + N]` — that 1:1 +//! mapping is what makes mouse-cell → text-cell trivial. With +//! ratatui's built-in `Wrap { trim: false }` we'd have to replay +//! ratatui's word-wrap to find the same mapping, which we don't want +//! to maintain in lockstep. + +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; + +use crate::app::{ScrollbackEntry, ScrollbackKind}; + +/// One styled [`Line`] per source line in `entry`. The leading +/// 2-cell column is the kind-prefix (`› `, `· `, `⚠ `, etc.) on the +/// first source line and two spaces on continuation lines, so a +/// multi-line entry visually hangs together. +/// +/// `area_width` is the terminal column count — used to +/// right-justify the per-input timer marker on the first line of +/// an `Input` entry. Pass the same width the renderer will wrap +/// to so the timer ends up flush at the right margin. +/// Backwards-compatible full-entry render. Same output as +/// [`entry_lines_windowed`] called with `0..u32::MAX`. Used by tests +/// and by the clipboard-copy path (which needs every source line +/// regardless of what's on screen). +pub fn entry_lines( + entry: &ScrollbackEntry, + area_width: u16, +) -> Vec> { + entry_lines_windowed(entry, area_width, 0..u32::MAX).0 +} + +/// Produce styled [`Line`] objects for only the source lines whose +/// **wrapped** rows overlap `window` (0-indexed within this entry's +/// wrapped row range). +/// +/// Returns `(lines, offset_into_first)` where `offset_into_first` is +/// how many wrapped rows of the first emitted line's own wrapped +/// output precede `window.start`. Callers pass this back through the +/// ratatui `Paragraph::scroll` residual so the on-screen scroll +/// alignment stays byte-for-byte identical to what the full-entry +/// render would have produced. +/// +/// This is the primary path — a huge scrollback entry (200K lines of +/// a `list` repr, say) that intersects the viewport was +/// previously O(entry.text.lines().count()) work per redraw because +/// `entry_lines` walked every source line, called `highlight_line` on +/// each, allocated a `Line` per line, and then handed the whole Vec +/// to `wrap_lines` which processed it end-to-end. Only the visible +/// window (typically ≤ area.height rows) ever mattered. This function +/// still walks the source-line iterator to *count* rows for skipped +/// lines (that's cheap — just `chars().count()` + arithmetic per +/// line), but does no allocation or highlighting until we hit the +/// window. Same output visually; O(entry) → O(visible + prefix_scan). +pub fn entry_lines_windowed( + entry: &ScrollbackEntry, + area_width: u16, + window: std::ops::Range, +) -> (Vec>, u32) { + // Collapsed entries: exactly one placeholder row regardless of + // window — the placeholder itself is the whole render. + if entry.collapse_state == Some(true) { + return (collapsible_lines(entry, true), 0); + } + // Expanded (Some(false)) and non-collapsible (None) both walk + // the windowed source-line path with the entry-kind's normal + // prefix/style. Expanded collapsible entries additionally get + // a `▼` marker in the leftmost 2-cell column so the toggleable + // affordance is visible — users can see this cell is collapsible + // without having to try Shift+click on every entry. None entries + // (single-line, no meaningful collapse) skip the marker column. + let orange = Color::Rgb(255, 140, 0); + let (kind_prefix_str, prefix_style) = kind_prefix(entry.kind, orange); + let marker_style = Style::default().fg(Color::Gray); + // Marker column shows the group / block collapse affordance: + // + // * Input entries always get a marker (▶ collapsed / ▼ + // expanded) — every command groups its output, so the + // affordance is always available. + // * Non-Input entries only get the marker when they're + // themselves an expanded collapsible block + // (collapse_state == Some(false)) — the intra-entry + // "this multi-line body can be collapsed" case. + let has_marker = matches!(entry.kind, ScrollbackKind::Input) + || entry.collapse_state == Some(false); + // Which glyph goes in the marker column. For Input rows we + // key off `group_collapsed`; for expanded non-Input blocks + // it's always `▼` (`▶` collapsed non-Input blocks are + // handled entirely by `collapsible_lines` above and never + // reach this branch). + let marker_glyph = if matches!(entry.kind, ScrollbackKind::Input) { + if entry.group_collapsed { + "▶ " + } else { + "▼ " + } + } else { + "▼ " + }; + // Continuation rows get an indent matching row 0's total prefix + // width so text stays visually aligned across a wrapped entry. + let cont_indent = if has_marker { " " } else { " " }; + let body_style = match entry.kind { + ScrollbackKind::Input => Style::default().fg(Color::White), + ScrollbackKind::Result => Style::default().fg(Color::Gray), + ScrollbackKind::Stdout => Style::default().fg(Color::White), + ScrollbackKind::Error => Style::default().fg(Color::Red), + ScrollbackKind::Warning => Style::default().fg(orange), + ScrollbackKind::Notice => Style::default().fg(Color::Gray), + ScrollbackKind::Chatter => { + Style::default().fg(Color::Gray).add_modifier(Modifier::DIM) + } + }; + // For Input entries with a timer, render `` flush + // right on the first line. Color follows whether it's still + // running (dim while live) vs. completed (subtle gray). + let timer = timer_for(entry); + // Highlighting strategy per kind: + // - Result / Stdout: repr highlighter (per-line shape recognition + // for compiler-emitted enum reprs). + // - Input: htcl-source highlighter (whole-entry parse, per-line + // span slicing) — colors keywords, calls, $vars, types, + // comments etc. the same as the input editor. + // - Error / Warning / Notice: single body color (not repr-formatted). + let repr_highlight = + matches!(entry.kind, ScrollbackKind::Result | ScrollbackKind::Stdout); + let input_highlight = matches!(entry.kind, ScrollbackKind::Input); + // Input entries: parse the whole entry text once and slice per-line. + // The body_style on Input is the bright cyan we'd otherwise apply + // flatly; the htcl highlighter overrides it for recognized tokens + // and leaves it for the gaps. + let input_per_line: Option>>> = if input_highlight { + Some(crate::highlight_htcl::highlight_per_line( + &entry.text, + body_style, + )) + } else { + None + }; + let mut out = Vec::new(); + // Per-source-line wrapped-row accounting — matches the formula + // in `count_wrapped_rows` exactly. Any drift here would misalign + // the caller's global row math with the actual rendered rows. + let w = area_width.max(1) as usize; + let prefix_width = 2; + let mut cumulative: u32 = 0; + let mut first_emitted_row_start: Option = None; + let mut had_lines = false; + for (i, line) in entry.text.lines().enumerate() { + had_lines = true; + let body_chars = line.chars().count(); + let total = body_chars.saturating_add(prefix_width).max(1); + let line_rows = (total.div_ceil(w).max(1)) as u32; + let line_end = cumulative.saturating_add(line_rows); + // Skip source lines whose wrapped-row range ends before the + // window even starts. Cheap — just chars().count() + arith, + // no allocations, no highlight calls. + if line_end <= window.start { + cumulative = line_end; + continue; + } + // Stop once we've moved past the window's end. `cumulative` + // here is the FIRST row this line contributes; if that's + // already past the window, everything remaining is offscreen. + if cumulative >= window.end { + break; + } + if first_emitted_row_start.is_none() { + first_emitted_row_start = Some(cumulative); + } + // Row-0 gutter for a collapsible expanded entry: `▼ ` marker + // in dim gray, then the entry-kind's normal prefix. Row 0+ of + // the same entry uses `cont_indent` (4 spaces to reserve room + // for both the marker column and the kind prefix) so wrapped + // text hangs cleanly under row 0's body. + let mut spans: Vec> = Vec::new(); + if i == 0 { + if has_marker { + spans + .push(Span::styled(marker_glyph.to_string(), marker_style)); + } + spans.push(Span::styled(kind_prefix_str.to_string(), prefix_style)); + } else { + spans.push(Span::styled(cont_indent.to_string(), prefix_style)); + } + if let Some(per_line) = input_per_line.as_ref() { + if let Some(line_spans) = per_line.get(i) { + spans.extend(line_spans.iter().cloned()); + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + } else if repr_highlight { + if let Some(highlighted) = crate::highlight::highlight_line(line) { + spans.extend(highlighted); + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + } else { + spans.push(Span::styled(line.to_string(), body_style)); + } + if i == 0 { + // Severity badges on Input rows: `✗` (red bold) when + // the group's children include any Error / CW, and + // `⚠` (orange bold, same glyph the Warning gutter + // uses) when they include any plain Warning. Both + // can render together; each shows a count when >1. + // Placement is between the input text and the timer + // so the far-right timer position stays stable. + let mut badges: Vec<(String, Style)> = Vec::new(); + if matches!(entry.kind, ScrollbackKind::Input) { + if entry.error_child_count > 0 { + let text = if entry.error_child_count > 1 { + format!("✗ {}", entry.error_child_count) + } else { + "✗".to_string() + }; + badges.push(( + text, + Style::default() + .fg(Color::Red) + .add_modifier(Modifier::BOLD), + )); + } + if entry.warning_child_count > 0 { + let text = if entry.warning_child_count > 1 { + format!("⚠ {}", entry.warning_child_count) + } else { + "⚠".to_string() + }; + badges.push(( + text, + Style::default() + .fg(orange) + .add_modifier(Modifier::BOLD), + )); + } + } + let timer_ref = timer.as_ref(); + if timer_ref.is_some() || !badges.is_empty() { + let used: usize = + spans.iter().map(|s| display_cells(&s.content)).sum(); + // Each badge contributes its glyph width + a + // trailing space to separate it from the next + // element. + let badges_w: usize = + badges.iter().map(|(t, _)| display_cells(t) + 1).sum(); + let timer_w = + timer_ref.map(|(l, _)| display_cells(l)).unwrap_or(0); + let total_right = badges_w + timer_w; + if (used + total_right + 1) as u16 <= area_width { + let pad = area_width as usize - used - total_right; + spans.push(Span::raw(" ".repeat(pad))); + for (btext, bstyle) in badges { + spans.push(Span::styled(btext, bstyle)); + spans.push(Span::raw(" ")); + } + if let Some((label, label_style)) = timer_ref { + spans.push(Span::styled(label.clone(), *label_style)); + } + } + } + } + out.push(Line::from(spans)); + cumulative = line_end; + } + if !had_lines && out.is_empty() { + // Empty text: match the pre-windowed behavior (one blank + // prefix-only line). Only render it if window includes row 0. + if window.start == 0 && window.end > 0 { + let mut blank_spans: Vec> = Vec::new(); + if has_marker { + blank_spans + .push(Span::styled(marker_glyph.to_string(), marker_style)); + } + blank_spans + .push(Span::styled(kind_prefix_str.to_string(), prefix_style)); + out.push(Line::from(blank_spans)); + first_emitted_row_start = Some(0); + } + } + let offset = first_emitted_row_start + .map(|s| window.start.saturating_sub(s)) + .unwrap_or(0); + (out, offset) +} + +/// Prefix + style for an entry-kind's row-0 gutter. Extracted so +/// the collapsed-placeholder renderer can use the same colors as +/// the expanded path — a collapsed `Result` reads with the same +/// gray body as an expanded `Result`, a collapsed `Warning` reads +/// with the same orange bold, etc. Only the entry's Chatter kind +/// specifically stays dim (that's the "background noise" bucket). +fn kind_prefix(kind: ScrollbackKind, orange: Color) -> (&'static str, Style) { + match kind { + ScrollbackKind::Input => ( + "› ", + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Result => (" ", Style::default().fg(Color::Gray)), + ScrollbackKind::Stdout => (" ", Style::default().fg(Color::White)), + ScrollbackKind::Error => ( + "✗ ", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Warning => ( + "⚠ ", + Style::default().fg(orange).add_modifier(Modifier::BOLD), + ), + ScrollbackKind::Notice => ("· ", Style::default().fg(Color::Gray)), + // Chatter: 2-space prefix (no diagnostic glyph) + DIM + // Gray — background-noise bucket for classifier-produced + // NONE blocks. Multi-line Chatter still auto-collapses per + // COLLAPSE_AUTO_THRESHOLD; the dim style just signals "this + // is elidable" even before the user thinks about collapsing. + ScrollbackKind::Chatter => ( + " ", + Style::default().fg(Color::Gray).add_modifier(Modifier::DIM), + ), + } +} + +/// Render the single-row placeholder for a **collapsed** entry: +/// dimmed preview of the first non-empty content line plus a +/// `(N lines hidden)` tail. Uses a uniform dim dark-gray body +/// style regardless of entry kind — the whole point of collapse +/// is "this is elided content; expand to read." Rendering it in +/// the kind's normal color (bright red for Error, orange for +/// Warning, etc.) makes the placeholder compete visually with +/// entries that aren't collapsed, defeating the "elided noise" +/// signal. When the user Shift-clicks to expand, the entry +/// reverts to its normal kind coloring — that's when you actually +/// want the Warning to look like a Warning. +fn collapsible_lines( + entry: &ScrollbackEntry, + collapsed: bool, +) -> Vec> { + debug_assert!( + collapsed, + "expanded entries go through entry_lines_windowed's \ + regular source-line path — collapsible_lines is the \ + collapsed-only placeholder helper" + ); + let marker_style = Style::default().fg(Color::Gray); + let dim = Style::default().fg(Color::Gray).add_modifier(Modifier::DIM); + let source_lines: Vec<&str> = entry.text.lines().collect(); + // Preview: first non-empty line (else first line, else ""). + let preview = source_lines + .iter() + .find(|l| !l.trim().is_empty()) + .copied() + .unwrap_or_default(); + let count = source_lines.len(); + let suffix = if count > 1 { + format!(" ({count} lines hidden)") + } else { + String::new() + }; + let mut spans = vec![Span::styled("▶ ".to_string(), marker_style)]; + spans.push(Span::styled(preview.to_string(), dim)); + if !suffix.is_empty() { + spans.push(Span::styled(suffix, dim)); + } + vec![Line::from(spans)] +} + +/// `(label, style)` for an entry's elapsed-time marker, or `None` +/// when the entry isn't timed. Color hints whether the timer is +/// still live (running) or frozen (completed). +fn timer_for(entry: &ScrollbackEntry) -> Option<(String, Style)> { + let start = entry.started_at?; + let end = entry.completed_at.unwrap_or_else(std::time::Instant::now); + let elapsed = end.saturating_duration_since(start); + let label = format_duration(elapsed); + let style = if entry.completed_at.is_some() { + // Frozen at final value — quiet, post-fact. + Style::default().fg(Color::Gray) + } else { + // Live — slightly more present so the user sees it's + // still moving. + Style::default().fg(Color::Yellow) + }; + Some((label, style)) +} + +/// Format a duration as `Ns`, `M:SS`, or `H:MM:SS` depending on +/// magnitude. Always second-granularity; never fractional. Matches +/// what users expect for "how long did this take" markers. +pub fn format_duration(d: std::time::Duration) -> String { + let total = d.as_secs(); + if total < 60 { + format!("{total}s") + } else if total < 3600 { + format!("{}:{:02}", total / 60, total % 60) + } else { + let h = total / 3600; + let m = (total % 3600) / 60; + let s = total % 60; + format!("{h}:{m:02}:{s:02}") + } +} + +/// Crude width estimator — counts chars, treating each as one +/// terminal cell. Good enough for our prefix glyphs (`› ` etc., +/// each rendered as one cell in monospace terminals) and ASCII +/// timer labels. A full unicode-width crate would be more +/// correct but isn't worth the dep for the small set of +/// characters this code emits. +fn display_cells(s: &str) -> usize { + s.chars().count() +} + +/// Split each input line into screen-row-sized chunks of `width` +/// columns, preserving span styles across the split. The output +/// renders 1:1 against screen rows when fed to a `Paragraph` with no +/// further wrapping, so screen-row N is `out[scroll + N]`. +/// +/// Splitting is character-based (no word-boundary respect) — this is +/// REPL output, not prose; long Vivado property dicts and Tcl errors +/// don't have natural break points. +/// Cheap pre-computation of how many wrapped terminal rows an +/// entry will occupy at the given width — WITHOUT actually +/// allocating wrapped lines. O(text length) per entry, no heap +/// allocations beyond the iterator. +/// +/// Used by the viewport-slicing render path to find which +/// entries intersect the visible window in linear time, so the +/// expensive [`entry_lines`] + [`wrap_lines`] only runs on the +/// handful of entries actually in view. Without this, a huge +/// entry (e.g. the formatted `util::props` output) gets +/// fully re-wrapped on every draw — turning every wheel event +/// into multi-MB of per-char allocation. +/// +/// The count must match what [`entry_lines`] + [`wrap_lines`] +/// actually produce: each natural text line contributes +/// `ceil((prefix + body_chars) / width)` wrapped rows (min 1). +/// The Input-entry timer suffix is ignored — when it fits it +/// pads the first line to exactly `width` (still 1 row); when +/// it doesn't fit it isn't added (so the body wraps normally +/// without it). Either way the row count matches. +pub fn count_wrapped_rows(entry: &ScrollbackEntry, width: u16) -> u32 { + if width == 0 { + return 1; + } + let w = width as usize; + // Row-0 gutter width matches what `entry_lines_windowed` emits: + // - collapsible + expanded (Some(false)): 4 cells ("▼ " + + // kind prefix) on row 0, 4-space `cont_indent` on + // continuation rows. + // - collapsed (Some(true)): handled by the `▶` placeholder + // branch below. + // - non-collapsible (None): 2 cells ("kind_prefix"). + // Mirror the has-marker rule in `entry_lines_windowed`: Input + // entries always carry the group-collapse marker; other entries + // carry a marker only when they're expanded collapsibles. + let prefix_width: usize = if matches!(entry.kind, ScrollbackKind::Input) + || entry.collapse_state == Some(false) + { + 4 + } else { + 2 + }; + // Collapsed entry: exactly one placeholder row of content, + // wrapped like any other line if the preview + suffix exceed + // the terminal width. The formula has to MATCH what + // `collapsible_lines` actually emits, character for character + // — any drift shifts every downstream entry's row index by the + // rounding error, and a click on the visible content maps to + // an adjacent buffer row (visible: "A total of 4711…", copies: + // "· INFO: [Common 17-83] Releasing license…"). + if let Some(true) = entry.collapse_state { + let preview = entry + .text + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(""); + let count = entry.text.lines().count(); + // Suffix exactly matches the `format!(" ({count} lines + // hidden)")` in `collapsible_lines`: 3 (`" ("`) + digits + + // 14 (`" lines hidden)"`). + let suffix_width = if count > 1 { + 3 + count.to_string().chars().count() + 14 + } else { + 0 + }; + let body_chars = preview.chars().count(); + let total = body_chars + .saturating_add(prefix_width) + .saturating_add(suffix_width) + .max(1); + return total.div_ceil(w).max(1) as u32; + } + let mut rows: u32 = 0; + let mut had_lines = false; + for line in entry.text.lines() { + had_lines = true; + let body_chars = line.chars().count(); + let total = body_chars.saturating_add(prefix_width).max(1); + let line_rows = total.div_ceil(w).max(1); + rows = rows.saturating_add(line_rows as u32); + } + if !had_lines { + // Empty text → entry_lines emits one blank line. + rows = 1; + } + rows +} + +pub fn wrap_lines(input: Vec>, width: u16) -> Vec> { + if width == 0 { + return input; + } + let w = width as usize; + let mut out = Vec::with_capacity(input.len()); + for line in input { + // Flatten spans → (char, style) so chunking can ignore the + // span boundaries and only care about per-cell style. + let mut chars: Vec<(char, Style)> = Vec::new(); + for span in &line.spans { + for c in span.content.chars() { + chars.push((c, span.style)); + } + } + if chars.is_empty() { + out.push(Line::from("")); + continue; + } + for chunk in chars.chunks(w) { + out.push(merge_to_line(chunk)); + } + } + out +} + +fn merge_to_line(chunk: &[(char, Style)]) -> Line<'static> { + let mut spans = Vec::new(); + let mut buf = String::new(); + let mut cur_style = chunk[0].1; + for (c, st) in chunk { + if *st != cur_style { + spans.push(Span::styled(std::mem::take(&mut buf), cur_style)); + cur_style = *st; + } + buf.push(*c); + } + if !buf.is_empty() { + spans.push(Span::styled(buf, cur_style)); + } + Line::from(spans) +} + +/// Plain-text content of a [`Line`] — span styles dropped, content +/// concatenated. Used to extract the selected substring for clipboard +/// copy. +pub fn line_plain_text(line: &Line<'_>) -> String { + let mut out = String::new(); + for span in &line.spans { + out.push_str(span.content.as_ref()); + } + out +} + +/// Re-style cells in `lines` that fall inside the selection range, +/// `[start, end)`. Both endpoints are `(row, col)` indices into +/// `lines` (the post-wrap, post-scroll Vec). The range may be +/// inverted (cursor before anchor); callers should normalize first. +pub fn apply_selection_highlight( + lines: &mut [Line<'static>], + start: (usize, usize), + end: (usize, usize), +) { + let (sr, sc) = start; + let (er, ec) = end; + for (row_idx, line) in lines.iter_mut().enumerate() { + if row_idx < sr || row_idx > er { + continue; + } + let row_start = if row_idx == sr { sc } else { 0 }; + let row_end = if row_idx == er { ec } else { usize::MAX }; + highlight_cols(line, row_start, row_end); + } +} + +fn highlight_cols(line: &mut Line<'static>, start: usize, end: usize) { + // Rebuild spans, splitting any that straddle the selection + // boundary so the REVERSED modifier applies to exactly the cells + // in [start, end). + let mut new_spans: Vec> = Vec::new(); + let mut col = 0usize; + for span in line.spans.drain(..) { + let span_chars: Vec = span.content.chars().collect(); + let len = span_chars.len(); + let span_start = col; + let span_end = col + len; + col = span_end; + + if span_end <= start || span_start >= end { + // Wholly outside selection — push unchanged. + new_spans.push(span); + continue; + } + + // Compute the three potential sub-pieces [..lo, lo..hi, hi..] + // where lo, hi are local offsets within span_chars. + let lo = start.saturating_sub(span_start).min(len); + let hi = end.saturating_sub(span_start).min(len); + + if lo > 0 { + let s: String = span_chars[..lo].iter().collect(); + new_spans.push(Span::styled(s, span.style)); + } + if hi > lo { + let s: String = span_chars[lo..hi].iter().collect(); + new_spans.push(Span::styled( + s, + span.style.add_modifier(Modifier::REVERSED), + )); + } + if hi < len { + let s: String = span_chars[hi..].iter().collect(); + new_spans.push(Span::styled(s, span.style)); + } + } + line.spans = new_spans; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn duration_seconds_under_minute() { + assert_eq!(format_duration(Duration::from_secs(0)), "0s"); + assert_eq!(format_duration(Duration::from_secs(1)), "1s"); + assert_eq!(format_duration(Duration::from_secs(59)), "59s"); + } + + #[test] + fn duration_mss_minute_to_hour() { + assert_eq!(format_duration(Duration::from_secs(60)), "1:00"); + assert_eq!(format_duration(Duration::from_secs(75)), "1:15"); + assert_eq!(format_duration(Duration::from_secs(3599)), "59:59"); + } + + #[test] + fn duration_hmmss_hour_plus() { + assert_eq!(format_duration(Duration::from_secs(3600)), "1:00:00"); + assert_eq!(format_duration(Duration::from_secs(3661)), "1:01:01"); + assert_eq!(format_duration(Duration::from_secs(36_000)), "10:00:00"); + } + + #[test] + fn duration_truncates_subsecond() { + // 5.9s should render as "5s" — second granularity only, + // never fractional. + assert_eq!(format_duration(Duration::from_millis(5_900)), "5s"); + } + + // ------------------------------------------------------------------ + // Windowed slicing regression + correctness + // ------------------------------------------------------------------ + + fn result_entry(text: &str) -> crate::app::ScrollbackEntry { + crate::app::ScrollbackEntry { + kind: crate::app::ScrollbackKind::Result, + text: text.to_string(), + started_at: None, + completed_at: None, + collapse_state: None, + is_critical_warning: false, + parent_input_idx: None, + group_collapsed: false, + error_child_count: 0, + warning_child_count: 0, + } + } + + /// A huge Result entry is what triggered the REPL lock-up: 200K + /// source lines, each cheap on its own but O(entry) work per + /// redraw when we allocated `Line`s for every one. `entry_lines` + /// (which delegates to the windowed impl with a full range) + /// still produces every row on demand — for the clipboard-copy + /// path — but `entry_lines_windowed` with a tight range should + /// produce only what fits. + #[test] + fn windowed_slicer_emits_only_visible_source_lines() { + // Build an entry where each source line fits on ONE wrapped + // row (short body + 2-cell prefix < area_width). Then the + // number of wrapped rows the entry contributes is exactly + // its source-line count, making the assertions easy to + // reason about. + let source: String = + (0..200_000).map(|i| format!("pin_{i}\n")).collect(); + let entry = result_entry(&source); + let (lines, offset) = + entry_lines_windowed(&entry, 80, 100_000..100_030); + // 30 lines requested, 30 lines emitted. + assert_eq!(lines.len(), 30); + // Window starts exactly on a source-line boundary — no + // sub-line offset. + assert_eq!(offset, 0); + } + + /// The FIRST emitted source line may land mid-window when the + /// window's start lands inside a line's wrapped output. The + /// slicer emits the whole line (row 0 of that line) and + /// reports the offset so ratatui's Paragraph::scroll can trim + /// the top. + #[test] + fn windowed_slicer_returns_sub_line_offset_when_window_starts_mid_line() { + // Build lines that wrap TO exactly 2 wrapped rows each on + // a width-10 area (2 prefix + 12 body chars → 14 total → + // ceil(14/10) = 2 rows). + let source: String = + (0..10).map(|i| format!("abcdefghjkl_{i:02}\n")).collect(); + let entry = result_entry(&source); + // Window that starts on the SECOND row of source line 3. + // Line 3's wrapped rows are [6, 8) globally; window [7, 12) + // should emit lines 3..6 with an offset of 1 wrapped row. + let (lines, offset) = entry_lines_windowed(&entry, 10, 7..12); + assert!(!lines.is_empty(), "expected some lines emitted"); + assert_eq!( + offset, 1, + "first line's row 0 is one row before window.start" + ); + } + + /// Full-range windowing must be identical to `entry_lines` — + /// otherwise the `entry_lines` shim would silently render + /// something different from what tests / clipboard-copy expect. + #[test] + fn windowed_slicer_full_range_matches_entry_lines() { + let source = "one\ntwo\nthree\nfour\nfive\nsix\nseven\neight\nnine\n"; + let entry = result_entry(source); + let unwindowed = entry_lines(&entry, 80); + let (windowed, offset) = entry_lines_windowed(&entry, 80, 0..u32::MAX); + assert_eq!(unwindowed.len(), windowed.len()); + for (a, b) in unwindowed.iter().zip(windowed.iter()) { + assert_eq!(a.spans.len(), b.spans.len()); + } + assert_eq!(offset, 0); + } +} diff --git a/vw-repl/src/session.rs b/vw-repl/src/session.rs new file mode 100644 index 0000000..468cb3a --- /dev/null +++ b/vw-repl/src/session.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! In-memory REPL session, held as parsed batches rather than a +//! re-concatenated text blob. +//! +//! Every successful input contributes one [`SessionBatch`] — the +//! loaded program (own source + import edges), its parsed +//! [`vw_htcl::Document`], and the map from proc-name to +//! [`ProcLocation`] for every proc the batch declared. Prior +//! batches are read by [`crate::lower::prepare`] when lowering the +//! next input: their signatures resolve unknown calls; their proc +//! locations let the error renderer translate Tcl's `(procedure +//! "X" line N)` frames back to the real `.htcl` file the wrapper +//! body was declared in. +//! +//! Why this shape (vs. the original text-blob prelude): +//! +//! 1. **Performance.** After a few `src @lib` imports the prelude +//! is hundreds of thousands of lines. Re-parsing and re-walking +//! it on every input is what made the REPL feel laggy. Storing +//! parsed state means each new input parses only its own +//! content + transitive imports — O(new), not O(total). +//! 2. **Error rendering.** A drill-down frame for a wrapper proc +//! declared in an earlier batch knows the real `.htcl` path it +//! came from, so `(procedure "vivado::create_bd_design" line +//! 2)` resolves to `vivado-cmd/bd.htcl:42` instead of +//! `(input):199185` of the giant combined scratch. + +use std::collections::HashMap; + +use vw_htcl::{Document, LoadedProgram, ProcSignature, TypeDecl}; + +use crate::lower::ProcLocation; + +/// One committed input: the parsed program it produced, plus the +/// proc-location map the lowerer derived from it. Stored on a +/// per-batch basis so signatures and proc lookups can fold across +/// the whole session without ever re-parsing a prior batch. +#[derive(Debug)] +pub struct SessionBatch { + /// Loader output for this batch — file paths, import edges, + /// and the flattened source. Held alongside the parsed + /// document so future analyzer features (completion, goto- + /// def, hover) can walk back to per-file context without + /// re-running the loader. Spans inside [`document`] are + /// offsets into [`program.source`](LoadedProgram::source); + /// keeping the program alive keeps those spans meaningful. + #[allow(dead_code)] + pub program: LoadedProgram, + pub document: Document, + pub procs: HashMap, +} + +/// A live REPL session: every committed batch in order. +#[derive(Debug, Default)] +pub struct Session { + batches: Vec, +} + +impl Session { + pub fn new() -> Self { + Self::default() + } + + /// Append a batch — called from the App after every successful + /// eval (including the pure-`src` no-Tcl-to-eval case, which + /// commits immediately because no eval can fail). + pub fn commit(&mut self, batch: SessionBatch) { + self.batches.push(batch); + } + + /// Build a merged signature table covering every proc declared + /// in the session so far. Later batches shadow earlier ones, + /// matching Tcl's "second `proc` redefines" semantics. The + /// returned map borrows from `self`; held only for the duration + /// of the next batch's prepare() call. + pub fn signature_table(&self) -> HashMap { + let mut table: HashMap = HashMap::new(); + for batch in &self.batches { + // Per-batch table merges into the running table; later + // batches' entries overwrite earlier ones via `insert`. + let batch_table = vw_htcl::signature_table(&batch.document); + for (name, sig) in batch_table { + table.insert(name, sig); + } + } + table + } + + /// Same as [`signature_table`] but for `type NAME = …` + /// declarations. Needed when wrapping a typed expression's + /// result through its `repr` proc — the dispatch type may be + /// a newtype declared in a prior batch (e.g. `Properties` + /// from a sourced `@vivado-cmd` library), and the repr + /// codegen walks the underlying to emit the dependent generic + /// repr (`dict_string_Property::repr` in that case). + pub fn type_decl_table(&self) -> HashMap { + let mut table: HashMap = HashMap::new(); + for batch in &self.batches { + let mut diags = Vec::new(); + let batch_table = + vw_htcl::build_type_decl_table(&batch.document, &mut diags); + for (name, td) in batch_table { + table.insert(name, td); + } + } + table + } + + /// Union of every top-level variable name defined across every + /// batch. Passed to the validator so the undef-var pass doesn't + /// false-positive `set p …` in an earlier REPL input followed + /// by `$p` in a later one. Only top-level names — proc-body + /// locals don't leak across evals (Tcl semantics). + pub fn top_level_var_names(&self) -> std::collections::HashSet { + let mut names = std::collections::HashSet::new(); + for batch in &self.batches { + names.extend(vw_htcl::top_level_var_names( + &batch.document, + &batch.program.source, + )); + } + names + } + + /// Companion to [`top_level_var_names`] returning inferred + /// types for the top-level `set` bindings across every + /// committed batch. Later batches shadow earlier ones so + /// re-binding `set foo […]` overrides the previous entry. + /// + /// Signature lookup is CUMULATIVE: batch N's `set foo [bar]` + /// resolves against every proc defined in batches ≤ N. This + /// matters at the REPL when a user runs `src @vw` in one + /// batch (populating `vw::vhdl_dependency_sources`) and then + /// `set deps [vw::vhdl_dependency_sources]` in the next — + /// without the cumulative table, `deps` would type-infer as + /// `None` and the next batch's `putr $deps` would fall + /// through to plain `puts` and dump the flat Tcl list. + pub fn top_level_var_types( + &self, + ) -> std::collections::HashMap { + let mut types = std::collections::HashMap::new(); + // Accumulated signatures across every committed batch — + // owned entries because each batch's sig_table borrows + // from the batch's own document; keeping references + // across batches would require self-referential lifetimes. + let mut cumulative_sigs: std::collections::HashMap< + String, + vw_htcl::ProcSignature, + > = std::collections::HashMap::new(); + for batch in &self.batches { + let mut sig_diags = Vec::new(); + let batch_sigs = vw_htcl::validate::build_signature_table( + &batch.document, + &mut sig_diags, + ); + // Merge into the cumulative store — later batches win + // on name collisions (Tcl re-definition semantics). + for (name, sig) in &batch_sigs { + cumulative_sigs.insert(name.clone(), (*sig).clone()); + } + // Re-project as `&ProcSignature` for + // `top_level_var_types`, which takes borrows. + let sig_view: std::collections::HashMap< + String, + &vw_htcl::ProcSignature, + > = cumulative_sigs + .iter() + .map(|(n, s)| (n.clone(), s)) + .collect(); + let batch_types = + vw_htcl::top_level_var_types(&batch.document, &sig_view); + for (name, ty) in batch_types { + types.insert(name, ty); + } + } + types + } + + /// Per-file `(path, mtime-at-load-time)` map covering every + /// committed batch's loaded files. Passed to the next + /// batch's loader as the `preloaded` set: the loader + /// short-circuits `src ` only when the current on-disk + /// mtime matches the stored one, so a user editing a + /// `.htcl` file and re-running `src` at the REPL actually + /// picks up the change. Later batches shadow earlier ones + /// on overlapping paths (matches "most-recent read wins"). + /// + /// Cross-batch this is what avoids re-parsing every + /// transitive dep when the target hasn't changed — a + /// `src ip/gtm` after `--load prime.htcl` used to re-parse + /// hundreds of vivado-cmd files unconditionally; now it + /// stats each preloaded file once, matches mtimes, and + /// skips. + pub fn loaded_paths( + &self, + ) -> std::collections::HashMap + { + let mut out = std::collections::HashMap::new(); + for batch in &self.batches { + for f in &batch.program.files { + // Files whose mtime we couldn't capture at + // load time stay OUT of the map — the loader + // treats "not preloaded" as "always reload", + // which is the safe default. + if let Some(t) = f.mtime { + out.insert(f.path.clone(), t); + } + } + } + out + } + + /// Look up the most-recent proc location across every batch. + /// Returns `None` when no batch has declared that proc — the + /// error renderer's drill-down path silently skips such frames + /// (Tcl proc frames for builtins, dynamically-defined procs, + /// etc.). + /// Iterate committed batches newest-first. Used by the + /// signature-help / hover paths to walk back through documents + /// looking for a proc's doc comments — Tcl's "later proc + /// shadows earlier" semantics mean the most-recent definition + /// is the one the user expects to see described. + pub fn batches_for_doc_search( + &self, + ) -> impl Iterator { + self.batches.iter().rev() + } + + pub fn lookup_proc(&self, name: &str) -> Option<&ProcLocation> { + for batch in self.batches.iter().rev() { + if let Some(loc) = batch.procs.get(name) { + return Some(loc); + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::parse; + + fn batch_from(source: &str) -> SessionBatch { + // Build a minimal in-memory LoadedProgram from a string for + // tests that don't care about the loader pipeline. + let parsed = parse(source); + assert!(parsed.errors.is_empty(), "{:?}", parsed.errors); + SessionBatch { + program: LoadedProgram { + source: source.to_string(), + files: Vec::new(), + regions: Vec::new(), + }, + document: parsed.document, + procs: HashMap::new(), + } + } + + #[test] + fn signature_table_folds_across_batches() { + let mut s = Session::new(); + s.commit(batch_from("proc foo { x } { }\n")); + s.commit(batch_from("proc bar { y } { }\n")); + let table = s.signature_table(); + assert!(table.contains_key("foo")); + assert!(table.contains_key("bar")); + } + + #[test] + fn later_batch_shadows_earlier_signature() { + // Second `proc foo` redefines the first — the merged table + // returns the newer signature. + let mut s = Session::new(); + s.commit(batch_from("proc foo { x } { }\n")); + s.commit(batch_from("proc foo { y z } { }\n")); + let table = s.signature_table(); + let sig = table.get("foo").unwrap(); + let arg_names: Vec<&str> = + sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(arg_names, vec!["y", "z"]); + } + + #[test] + fn top_level_var_types_resolves_across_batches() { + // Regression: batch 1 defines `proc lookup {} dict { ... }`; + // batch 2 does `set d [lookup]`. `top_level_var_types` must + // report `d: dict` — before the fix, each batch's inference + // saw only its own procs, so `d` came back untyped and the + // downstream `putr $d` fell through to plain `puts`. + let mut s = Session::new(); + s.commit(batch_from("proc lookup {} dict { return {a 1} }\n")); + s.commit(batch_from("set d [lookup]\n")); + let types = s.top_level_var_types(); + let ty = types + .get("d") + .expect("`d` should have an inferred type across batches"); + match ty { + vw_htcl::TypeExpr::Named { name, .. } => { + assert_eq!(name, "dict") + } + other => panic!("expected `dict`, got {other:?}"), + } + } + + #[test] + fn lookup_proc_returns_latest_batch() { + // Two batches both register `foo` in their `procs` map (the + // lowerer normally does this, but here we set it manually). + // `lookup_proc` returns the entry from the most recent + // batch. + let mut a = batch_from("proc foo { x } { }\n"); + let mut b = batch_from("proc foo { y } { }\n"); + a.procs.insert( + "foo".into(), + ProcLocation { + file: None, + body_start_line: 10, + body_lines: vec!["from-a".into()], + }, + ); + b.procs.insert( + "foo".into(), + ProcLocation { + file: None, + body_start_line: 20, + body_lines: vec!["from-b".into()], + }, + ); + let mut s = Session::new(); + s.commit(a); + s.commit(b); + let got = s.lookup_proc("foo").unwrap(); + assert_eq!(got.body_start_line, 20); + assert_eq!(got.body_lines[0], "from-b"); + assert!(s.lookup_proc("missing").is_none()); + } +} diff --git a/vw-repl/src/symbol_index.rs b/vw-repl/src/symbol_index.rs new file mode 100644 index 0000000..414fd5b --- /dev/null +++ b/vw-repl/src/symbol_index.rs @@ -0,0 +1,497 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Catalog of every named thing the session knows about — procs +//! (including overload dispatchers), type aliases, enum +//! declarations and their variants. Each entry carries the +//! **library** it came from (the `src @` import that brought +//! it into scope, or `` for the user's directly-typed +//! batch) plus its doc comments and a one-line signature brief. +//! +//! Consumed by the fuzzy symbol-search popup (slice 8) and the +//! libraries view (slice 9). The picker doesn't need to know +//! about parse state; it just needs a flat list of `Symbol`s with +//! enough metadata to display, filter, and rank. +//! +//! The index is purely structural — no fuzzy-matching here. That's +//! the picker's job; the index just hands it the candidate list. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use vw_htcl::ast::{CommandKind, Document, EnumVariant, ProcSignature, Stmt}; +use vw_htcl::loader::LoadedProgram; +use vw_htcl::span::Span; + +use crate::session::{Session, SessionBatch}; + +#[derive(Clone, Debug)] +pub struct Symbol { + /// Fully-qualified name (`util::props`, `Property::Scalar`, etc.). + pub name: String, + pub kind: SymbolKind, + pub library: LibraryRef, + /// First paragraph of the doc comments, reflowed for a compact + /// summary in the picker's result row. + pub doc_summary: String, + /// Full reflowed doc body, shown in the picker's preview pane + /// (when added) and in the hover popup (already wired). + pub doc_full: String, + /// One-line "signature brief" — `name -arg: type -arg: type → ret` + /// for procs, `enum Name = { V1; V2 }` for enums, etc. + pub signature_brief: String, + /// Origin span in the source the symbol was declared in. `None` + /// for variables (whose `set` site isn't a "declaration" in the + /// AST sense we want to jump to). Used by a future goto-def + /// keybinding; the picker doesn't need it. + pub def_span: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum SymbolKind { + Proc, + Type, + EnumDecl, + EnumVariant, + Variable, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LibraryRef { + /// Symbol was declared in the user's directly-typed batch (no + /// `src @` import). Shown as `` in the picker. + Entry, + /// Symbol was declared in a file pulled in via a `src @` + /// import (transitively — nested imports still attribute to the + /// top-level `@`). + Import { name: String, path: PathBuf }, +} + +impl LibraryRef { + /// Short display name — `` or the library's import name. + /// Used as the prefix in picker rows. + pub fn display(&self) -> String { + match self { + LibraryRef::Entry => "".to_string(), + LibraryRef::Import { name, .. } => name.clone(), + } + } +} + +#[derive(Clone, Debug)] +pub struct LibraryInfo { + pub library: LibraryRef, + /// Number of symbols this library contributes to the index. + pub symbol_count: usize, +} + +/// Flat catalog of every symbol the session + in-flight input +/// knows about. Build via [`SymbolIndex::build`]; iterate via +/// [`SymbolIndex::all`] or [`SymbolIndex::libraries`]. +#[derive(Clone, Debug, Default)] +pub struct SymbolIndex { + all: Vec, +} + +impl SymbolIndex { + pub fn build( + session: &Session, + pending: Option<&SessionBatch>, + in_flight: Option<&Document>, + ) -> Self { + let mut symbols = Vec::new(); + for batch in session.batches_for_doc_search() { + collect_from_batch(batch, &mut symbols); + } + // Pending batch — the in-flight eval. During a long + // `src @vivado-cmd` load, every proc the user just sourced + // lives here, NOT yet in session.signature_table. Including + // it makes Ctrl-S / :libs work mid-eval, same way the + // Tab-completion fix did. + if let Some(batch) = pending { + collect_from_batch(batch, &mut symbols); + } + if let Some(doc) = in_flight { + collect_from_doc(doc, None, &mut symbols); + } + // Dedupe by (name, kind) — later-batch decls shadow earlier + // (Tcl's "later proc redefines" semantics). Since we walked + // session newest-first via batches_for_doc_search, the FIRST + // occurrence in our list is the one to keep. + let mut seen: BTreeMap<(String, SymbolKind), ()> = BTreeMap::new(); + symbols.retain(|s| seen.insert((s.name.clone(), s.kind), ()).is_none()); + Self { all: symbols } + } + + pub fn all(&self) -> &[Symbol] { + &self.all + } + + /// Distinct libraries with their symbol counts, sorted by + /// descending count (heavyweights like `vivado-cmd` first). + pub fn libraries(&self) -> Vec { + let mut counts: BTreeMap = BTreeMap::new(); + for sym in &self.all { + let entry = counts + .entry(sym.library.display()) + .or_insert_with(|| (sym.library.clone(), 0)); + entry.1 += 1; + } + let mut out: Vec = counts + .into_values() + .map(|(library, symbol_count)| LibraryInfo { + library, + symbol_count, + }) + .collect(); + out.sort_by_key(|b| std::cmp::Reverse(b.symbol_count)); + out + } +} + +fn collect_from_batch(batch: &SessionBatch, out: &mut Vec) { + collect_from_doc(&batch.document, Some(&batch.program), out); +} + +fn collect_from_doc( + doc: &Document, + program: Option<&LoadedProgram>, + out: &mut Vec, +) { + walk_stmts(&doc.stmts, "", program, out); +} + +fn walk_stmts( + stmts: &[Stmt], + prefix: &str, + program: Option<&LoadedProgram>, + out: &mut Vec, +) { + for stmt in stmts { + let Stmt::Command(cmd) = stmt else { + continue; + }; + match &cmd.kind { + CommandKind::Proc(proc) => { + let Some(name) = proc.name.as_deref() else { + continue; + }; + let Some(sig) = proc.signature.as_ref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let signature_brief = render_signature(&qualified, sig); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified, + kind: SymbolKind::Proc, + library, + doc_summary, + doc_full, + signature_brief, + def_span: Some(proc.name_span), + }); + // Recurse into proc body so nested procs / type decls + // also enter the index. Rare but supported. + walk_stmts(&proc.body, prefix, program, out); + } + CommandKind::TypeDecl(td) => { + let Some(name) = td.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let underlying = td + .underlying + .as_ref() + .map(render_type) + .unwrap_or_else(|| "?".into()); + let signature_brief = + format!("type {qualified} = {underlying}"); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified, + kind: SymbolKind::Type, + library, + doc_summary, + doc_full, + signature_brief, + def_span: Some(td.name_span), + }); + } + CommandKind::EnumDecl(ed) => { + let Some(name) = ed.name.as_deref() else { + continue; + }; + let qualified = qualify(prefix, name); + let library = library_for_span(program, cmd.span); + let variants_brief: Vec = ed + .variants + .iter() + .map(|v| { + if let Some(ty) = v.payload.as_ref() { + format!("{}: {}", v.name, render_type(ty)) + } else { + v.name.clone() + } + }) + .collect(); + let signature_brief = format!( + "enum {qualified} = {{ {} }}", + variants_brief.join("; ") + ); + let doc_summary = + vw_htcl::doc::brief(&cmd.doc_comments).unwrap_or_default(); + let doc_full = + vw_htcl::doc::reflow_doc_comments(&cmd.doc_comments); + out.push(Symbol { + name: qualified.clone(), + kind: SymbolKind::EnumDecl, + library: library.clone(), + doc_summary: doc_summary.clone(), + doc_full: doc_full.clone(), + signature_brief, + def_span: Some(ed.name_span), + }); + // One Symbol per variant so users can search for + // variant names directly (`Scalar`, `Nested`, …). + for v in &ed.variants { + push_variant(&qualified, v, library.clone(), out); + } + } + CommandKind::NamespaceEval(ns) => { + let Some(name) = ns.name.as_deref() else { + continue; + }; + let nested = qualify(prefix, name); + walk_stmts(&ns.body, &nested, program, out); + } + CommandKind::Set => { + // Top-level / proc-body variable. Take the first + // word as the variable name. Variables are tagged + // with the library of the enclosing batch but get + // no def_span (set isn't a declaration in the + // jump-to-def sense we want). + if let Some(name_word) = cmd.words.get(1) { + if let Some(name) = name_word.as_text() { + let library = library_for_span(program, cmd.span); + out.push(Symbol { + name: qualify(prefix, name), + kind: SymbolKind::Variable, + library, + doc_summary: String::new(), + doc_full: String::new(), + signature_brief: format!("set {name}"), + def_span: None, + }); + } + } + } + _ => {} + } + } +} + +fn push_variant( + enum_qualified: &str, + v: &EnumVariant, + library: LibraryRef, + out: &mut Vec, +) { + let qualified = format!("{enum_qualified}::{}", v.name); + let signature_brief = if let Some(ty) = v.payload.as_ref() { + format!("{qualified}({})", render_type(ty)) + } else { + qualified.clone() + }; + out.push(Symbol { + name: qualified, + kind: SymbolKind::EnumVariant, + library, + doc_summary: String::new(), + doc_full: String::new(), + signature_brief, + def_span: Some(v.name_span), + }); +} + +fn qualify(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}::{name}") + } +} + +/// Attribute a source span to its originating library. Walks the +/// loader's `regions` + `files` tables to find which file the span +/// came from, then climbs the `imported_via` chain to the +/// top-level import (the `src @` directly under the entry +/// file). Returns `LibraryRef::Entry` when the span lies in the +/// entry file itself or when no `LoadedProgram` is provided +/// (in-flight input that hasn't been loaded through the import +/// machinery). +fn library_for_span(program: Option<&LoadedProgram>, span: Span) -> LibraryRef { + let Some(program) = program else { + return LibraryRef::Entry; + }; + let Some((file_idx, _)) = program.locate(span.start) else { + return LibraryRef::Entry; + }; + // Build the chain: [origin_file, ..., entry_file]. + let mut chain = vec![file_idx]; + let mut cur = file_idx; + while let Some(edge) = program.files[cur].imported_via { + chain.push(edge.importer_file); + cur = edge.importer_file; + } + if chain.len() <= 1 { + // The origin is the entry file. + return LibraryRef::Entry; + } + // The element just before the entry is the top-level imported + // file — that's our library. + let top_imported = chain[chain.len() - 2]; + let file = &program.files[top_imported]; + let name = library_name_for_path(&file.path); + LibraryRef::Import { + name, + path: file.path.clone(), + } +} + +/// Short, user-facing library name. We use the parent directory's +/// name when it's available (so +/// `/home/ry/src/htcl/amd/vivado-cmd/module.htcl` becomes +/// `vivado-cmd`); otherwise fall back to the file's stem. +fn library_name_for_path(path: &std::path::Path) -> String { + if let Some(parent) = path.parent() { + if let Some(name) = parent.file_name() { + return name.to_string_lossy().into_owned(); + } + } + path.file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.display().to_string()) +} + +fn render_type(ty: &vw_htcl::TypeExpr) -> String { + use vw_htcl::TypeExpr; + match ty { + TypeExpr::Named { name, .. } => name.clone(), + TypeExpr::Generic { name, args, .. } => { + let inner: Vec = args.iter().map(render_type).collect(); + format!("{name}<{}>", inner.join(", ")) + } + TypeExpr::Qualified { + namespace, variant, .. + } => format!("{namespace}::{variant}"), + } +} + +fn render_signature(name: &str, sig: &ProcSignature) -> String { + let mut out = name.to_string(); + for arg in &sig.args { + out.push_str(" -"); + out.push_str(&arg.name); + if let Some(ty) = arg.type_annotation.as_ref() { + out.push_str(": "); + out.push_str(&render_type(ty)); + } + } + if let Some(ret) = sig.return_type.as_ref() { + out.push_str(" → "); + out.push_str(&render_type(ret)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use vw_htcl::parser::parse; + + #[test] + fn empty_session_yields_empty_index() { + let session = Session::default(); + let idx = SymbolIndex::build(&session, None, None); + assert!(idx.all().is_empty()); + assert!(idx.libraries().is_empty()); + } + + #[test] + fn in_flight_proc_decl_indexed() { + let session = Session::default(); + let parsed = parse("proc foo {x: int} bool { return $x }"); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + assert_eq!(idx.all().len(), 1); + let sym = &idx.all()[0]; + assert_eq!(sym.name, "foo"); + assert_eq!(sym.kind, SymbolKind::Proc); + assert_eq!(sym.library, LibraryRef::Entry); + assert!(sym.signature_brief.contains("foo")); + assert!(sym.signature_brief.contains("int")); + assert!(sym.signature_brief.contains("bool")); + } + + #[test] + fn namespaced_procs_qualified() { + let session = Session::default(); + let src = "namespace eval util {\n proc props {x: int} string { return $x }\n}"; + let parsed = parse(src); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + assert!( + idx.all().iter().any(|s| s.name == "util::props"), + "expected util::props in index: {:?}", + idx.all().iter().map(|s| &s.name).collect::>() + ); + } + + #[test] + fn enum_emits_decl_and_variants() { + let session = Session::default(); + let src = + "enum Property = {\n Scalar: string\n Nested: Properties\n}"; + let parsed = parse(src); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let names: Vec<&str> = + idx.all().iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"Property"), "{names:?}"); + assert!(names.contains(&"Property::Scalar"), "{names:?}"); + assert!(names.contains(&"Property::Nested"), "{names:?}"); + } + + #[test] + fn type_decl_indexed() { + let session = Session::default(); + let parsed = parse("type Properties = {dict}"); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let ty = idx + .all() + .iter() + .find(|s| s.name == "Properties" && s.kind == SymbolKind::Type) + .expect("Properties not found"); + assert!(ty.signature_brief.contains("type")); + assert!(ty.signature_brief.contains("dict")); + } + + #[test] + fn libraries_count_symbols() { + let session = Session::default(); + let parsed = parse( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let idx = SymbolIndex::build(&session, None, Some(&parsed.document)); + let libs = idx.libraries(); + assert_eq!(libs.len(), 1); + assert_eq!(libs[0].symbol_count, 3); + } +} diff --git a/vw-repl/src/symbol_search.rs b/vw-repl/src/symbol_search.rs new file mode 100644 index 0000000..2e532e4 --- /dev/null +++ b/vw-repl/src/symbol_search.rs @@ -0,0 +1,488 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Fuzzy symbol picker — Ctrl-T opens a centered modal listing every +//! known procedure / type / enum / variant across the session and +//! the in-flight input. Typing filters; ↑/↓ navigates; Enter inserts +//! the symbol's qualified name at the cursor in the input editor. +//! +//! The matcher is [`nucleo_matcher`] — the same engine Helix and +//! several other Rust TUIs use. Each candidate is scored twice: +//! once against its `name` (high weight) and once against its +//! `doc_summary` (low weight). The final score takes the max of +//! the two, so a query that hits ONLY a doc-comment phrase still +//! surfaces the symbol but ranks below any name match. +//! +//! Tab inside the picker switches to the **library view** — +//! `(library, symbol-count)` rows. Enter on a library row filters +//! the symbol list to that library. + +use std::sync::Arc; + +use nucleo_matcher::pattern::{CaseMatching, Normalization, Pattern}; +use nucleo_matcher::{Config, Matcher, Utf32String}; +use ratatui::layout::Rect; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, List, ListItem, ListState, Paragraph, +}; +use ratatui::Frame; + +use crate::symbol_index::{LibraryInfo, LibraryRef, SymbolIndex, SymbolKind}; + +/// Boost factor applied to name-match scores so name hits always +/// outrank doc-only hits. Picked by trial — a value of 3 keeps a +/// short doc match (e.g. one word) below any name fuzzy hit. +const NAME_WEIGHT: u32 = 3; + +/// Symbol-picker overlay state. +#[derive(Debug)] +pub struct SymbolPicker { + /// Snapshot of the symbol index when the picker was opened. + /// We don't update mid-search — the index may not change in + /// practice (no commits land while a popup is open), and a + /// stable index avoids selection-index churn while typing. + pub index: Arc, + /// Live query string the user is typing. + pub query: String, + /// Scored result indices into `index.all()` in display order. + pub results: Vec, + pub selected: usize, + /// When `Some(name)`, the result list is filtered to that + /// library. Set by accepting a row in the libraries sub-view. + pub library_filter: Option, + /// Toggle between symbol list and library list. + pub view: PickerView, + /// Cached library list (computed once on open). + pub libraries: Vec, + /// Selected library row (only used in `Libraries` view). + pub selected_library: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PickerView { + Symbols, + Libraries, +} + +#[derive(Clone, Copy, Debug)] +pub struct Scored { + pub sym_idx: usize, + pub score: u32, +} + +impl SymbolPicker { + pub fn new(index: Arc) -> Self { + let libraries = index.libraries(); + let mut p = Self { + index, + query: String::new(), + results: Vec::new(), + selected: 0, + library_filter: None, + view: PickerView::Symbols, + libraries, + selected_library: 0, + }; + p.recompute(); + p + } + + pub fn move_up(&mut self) { + match self.view { + PickerView::Symbols => { + if self.selected > 0 { + self.selected -= 1; + } + } + PickerView::Libraries => { + if self.selected_library > 0 { + self.selected_library -= 1; + } + } + } + } + + pub fn move_down(&mut self) { + match self.view { + PickerView::Symbols => { + if self.selected + 1 < self.results.len() { + self.selected += 1; + } + } + PickerView::Libraries => { + if self.selected_library + 1 < self.libraries.len() { + self.selected_library += 1; + } + } + } + } + + pub fn push_char(&mut self, c: char) { + self.query.push(c); + self.recompute(); + } + + pub fn pop_char(&mut self) { + self.query.pop(); + self.recompute(); + } + + pub fn toggle_view(&mut self) { + self.view = match self.view { + PickerView::Symbols => PickerView::Libraries, + PickerView::Libraries => PickerView::Symbols, + }; + } + + /// Apply a library filter — sets `library_filter` and switches + /// back to the symbols view. Used when Enter is pressed on a + /// library row. + pub fn apply_library_filter(&mut self) { + if let Some(lib) = self.libraries.get(self.selected_library) { + self.library_filter = Some(lib.library.display()); + self.view = PickerView::Symbols; + self.selected = 0; + self.recompute(); + } + } + + /// Currently-selected symbol, if any (Symbols view only). + pub fn current_symbol(&self) -> Option<&crate::symbol_index::Symbol> { + let idx = self.results.get(self.selected)?.sym_idx; + self.index.all().get(idx) + } + + /// Re-run the matcher with the current query and library + /// filter. Called from `new` / `push_char` / `pop_char` / + /// `apply_library_filter`. O(N) over the index. + fn recompute(&mut self) { + let mut matcher = Matcher::new(Config::DEFAULT); + let all = self.index.all(); + // Indices that survive the library filter (if any). + let candidate_indices: Vec = all + .iter() + .enumerate() + .filter(|(_, s)| match &self.library_filter { + Some(lib) => s.library.display() == *lib, + None => true, + }) + .map(|(i, _)| i) + .collect(); + + if self.query.is_empty() { + // No query: show all candidates sorted alphabetically, + // capped at 200 so the popup is bounded. + let mut sorted = candidate_indices.clone(); + sorted.sort_by(|a, b| all[*a].name.cmp(&all[*b].name)); + sorted.truncate(200); + self.results = sorted + .into_iter() + .map(|sym_idx| Scored { sym_idx, score: 0 }) + .collect(); + self.selected = + self.selected.min(self.results.len().saturating_sub(1)); + return; + } + + let pattern = Pattern::parse( + &self.query, + CaseMatching::Smart, + Normalization::Smart, + ); + let mut scored: Vec = candidate_indices + .iter() + .filter_map(|&sym_idx| { + let sym = &all[sym_idx]; + let name_haystack = Utf32String::from(sym.name.as_str()); + let doc_haystack = Utf32String::from(sym.doc_summary.as_str()); + let name_score = + pattern.score(name_haystack.slice(..), &mut matcher); + let doc_score = + pattern.score(doc_haystack.slice(..), &mut matcher); + let combined = match (name_score, doc_score) { + (None, None) => 0, + (Some(n), None) => n * NAME_WEIGHT, + (None, Some(d)) => d, + (Some(n), Some(d)) => (n * NAME_WEIGHT).max(d), + }; + if combined == 0 { + None + } else { + Some(Scored { + sym_idx, + score: combined, + }) + } + }) + .collect(); + scored.sort_by_key(|b| std::cmp::Reverse(b.score)); + scored.truncate(200); + self.results = scored; + if self.selected >= self.results.len() { + self.selected = self.results.len().saturating_sub(1); + } + } +} + +/// Render the picker as a centered modal. Sized at 70% width × 75% +/// height of the frame. +pub fn draw_symbol_picker(f: &mut Frame, picker: &SymbolPicker) { + let frame = f.area(); + let width = (frame.width as f32 * 0.7) as u16; + let height = (frame.height as f32 * 0.75) as u16; + let x = frame.x + (frame.width.saturating_sub(width)) / 2; + let y = frame.y + (frame.height.saturating_sub(height)) / 2; + let area = Rect { + x, + y, + width, + height, + }; + f.render_widget(Clear, area); + + // Top: query line (1 row). Middle: list (fills). Bottom: hint + // (1 row). Use a manual vertical split since this is a small + // fixed layout. + let inner_top = Rect { + x: area.x + 1, + y: area.y + 1, + width: area.width.saturating_sub(2), + height: 1, + }; + let inner_hint = Rect { + x: area.x + 1, + y: area.y + area.height.saturating_sub(2), + width: area.width.saturating_sub(2), + height: 1, + }; + let inner_list = Rect { + x: area.x + 1, + y: area.y + 2, + width: area.width.saturating_sub(2), + height: area.height.saturating_sub(4), + }; + + let title = match picker.view { + PickerView::Symbols => " symbol search ", + PickerView::Libraries => " libraries — Enter to filter ", + }; + f.render_widget(Block::default().borders(Borders::ALL).title(title), area); + + let prompt_style = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD); + let dim = Style::default().add_modifier(Modifier::DIM); + let prompt_prefix = match picker.view { + PickerView::Symbols => "› ", + PickerView::Libraries => "» ", + }; + let prompt_text = match (picker.view, picker.library_filter.as_deref()) { + (PickerView::Symbols, Some(lib)) => { + format!("{prompt_prefix}[{lib}] {}", picker.query) + } + (PickerView::Symbols, None) => { + format!("{prompt_prefix}{}", picker.query) + } + (PickerView::Libraries, _) => "(use ↑/↓; Enter to filter)".to_string(), + }; + f.render_widget( + Paragraph::new(Span::styled(prompt_text, prompt_style)), + inner_top, + ); + + match picker.view { + PickerView::Symbols => render_symbols(f, picker, inner_list), + PickerView::Libraries => render_libraries(f, picker, inner_list), + } + + let hint = match picker.view { + PickerView::Symbols => "Tab: libraries · Enter: insert · Esc: dismiss", + PickerView::Libraries => "Tab: symbols · Enter: filter · Esc: dismiss", + }; + f.render_widget(Paragraph::new(Span::styled(hint, dim)), inner_hint); +} + +fn render_symbols(f: &mut Frame, picker: &SymbolPicker, area: Rect) { + let all = picker.index.all(); + let items: Vec = picker + .results + .iter() + .filter_map(|scored| all.get(scored.sym_idx)) + .map(|sym| { + let icon = match sym.kind { + SymbolKind::Proc => "·", + SymbolKind::Type => "≡", + SymbolKind::EnumDecl => "◆", + SymbolKind::EnumVariant => "◇", + SymbolKind::Variable => "$", + }; + let icon_style = match sym.kind { + SymbolKind::Proc => { + Style::default().fg(Color::Rgb(230, 200, 120)) + } + SymbolKind::Type + | SymbolKind::EnumDecl + | SymbolKind::EnumVariant => { + Style::default().fg(Color::Rgb(100, 200, 200)) + } + SymbolKind::Variable => { + Style::default().fg(Color::Rgb(130, 200, 230)) + } + }; + let lib_style = Style::default().add_modifier(Modifier::DIM); + let name_style = Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD); + let doc_style = Style::default().fg(Color::Gray); + + let mut spans = vec![ + Span::styled(format!("{icon} "), icon_style), + Span::styled(format!("{} ", sym.library.display()), lib_style), + Span::styled(":: ", lib_style), + Span::styled(sym.name.clone(), name_style), + ]; + if !sym.doc_summary.is_empty() { + spans.push(Span::raw(" ")); + spans.push(Span::styled(sym.doc_summary.clone(), doc_style)); + } + ListItem::new(Line::from(spans)) + }) + .collect(); + + let mut state = ListState::default(); + if !picker.results.is_empty() { + state.select(Some(picker.selected)); + } + let list = List::new(items).highlight_style( + Style::default() + .bg(Color::Rgb(40, 40, 60)) + .add_modifier(Modifier::BOLD), + ); + f.render_stateful_widget(list, area, &mut state); +} + +fn render_libraries(f: &mut Frame, picker: &SymbolPicker, area: Rect) { + let items: Vec = picker + .libraries + .iter() + .map(|info| { + let name = info.library.display(); + let path_str = match &info.library { + LibraryRef::Entry => "".to_string(), + LibraryRef::Import { path, .. } => path.display().to_string(), + }; + ListItem::new(Line::from(vec![ + Span::styled( + format!(" {:5} ", info.symbol_count), + Style::default() + .fg(Color::Rgb(230, 200, 120)) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + name, + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled( + path_str, + Style::default().add_modifier(Modifier::DIM), + ), + ])) + }) + .collect(); + let mut state = ListState::default(); + if !picker.libraries.is_empty() { + state.select(Some(picker.selected_library)); + } + let list = List::new(items).highlight_style( + Style::default() + .bg(Color::Rgb(40, 40, 60)) + .add_modifier(Modifier::BOLD), + ); + f.render_stateful_widget(list, area, &mut state); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use crate::session::Session; + use crate::symbol_index::SymbolIndex; + + fn build_index_from_src(src: &str) -> Arc { + let session = Session::default(); + let parsed = vw_htcl::parse(src); + Arc::new(SymbolIndex::build(&session, None, Some(&parsed.document))) + } + + #[test] + fn empty_query_shows_all_alphabetic() { + let idx = build_index_from_src( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let picker = SymbolPicker::new(idx); + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + assert_eq!(names, vec!["bar", "baz", "foo"]); + } + + #[test] + fn query_filters_results() { + let idx = build_index_from_src( + "proc foo {} unit {}\nproc bar {} unit {}\nproc baz {} unit {}", + ); + let mut picker = SymbolPicker::new(idx); + picker.push_char('b'); + picker.push_char('a'); + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + assert!(names.contains(&"bar")); + assert!(names.contains(&"baz")); + assert!(!names.contains(&"foo")); + } + + #[test] + fn name_match_outranks_doc_only() { + // A proc whose name doesn't match but whose docs mention the + // query word should rank BELOW a proc whose name matches. + let src = "\ +## a wonderful procedure that does foobar things +proc unrelated {} unit {} +proc foobar_proc {} unit {}"; + let idx = build_index_from_src(src); + let mut picker = SymbolPicker::new(idx); + for c in "foobar".chars() { + picker.push_char(c); + } + let names: Vec<&str> = picker + .results + .iter() + .map(|s| picker.index.all()[s.sym_idx].name.as_str()) + .collect(); + // foobar_proc must come first + assert_eq!(names.first().copied(), Some("foobar_proc"), "{names:?}"); + } + + #[test] + fn library_view_toggles() { + let idx = build_index_from_src("proc foo {} unit {}"); + let mut picker = SymbolPicker::new(idx); + assert_eq!(picker.view, PickerView::Symbols); + picker.toggle_view(); + assert_eq!(picker.view, PickerView::Libraries); + picker.toggle_view(); + assert_eq!(picker.view, PickerView::Symbols); + } +} diff --git a/vw-repl/src/trace.rs b/vw-repl/src/trace.rs new file mode 100644 index 0000000..0340c42 --- /dev/null +++ b/vw-repl/src/trace.rs @@ -0,0 +1,163 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Stack-frame rewriting for Vivado error / warning messages. +//! +//! Vivado reports errors with frames like +//! ` at :14 in ::configure_cips` +//! +//! where `` is the scratch path the lowerer ships and the +//! line number is body-relative inside the proc. This module maps +//! those back to the original htcl source: +//! ` at ip/cips.htcl:69 in ::configure_cips` +//! +//! Both the REPL (driven by a multi-batch [`crate::session::Session`]) +//! and the `vw run` CLI driver (single batch) feed messages through +//! the same `resolve_stack_frames_with` machinery — they differ only +//! in how they answer the "where does proc P live?" question, supplied +//! as a closure. + +use std::path::Path; + +use crate::lower::ProcLocation; + +/// One stack-frame line after rewriting. Callers dedupe adjacent +/// frames that resolve to the same `(proc, line)` because Vivado +/// often emits two frames per logical site (one for the `proc`'s +/// `kwargs` wrapper, one for the real body). +pub struct RewrittenFrame { + pub proc: String, + pub line: u32, + pub formatted: String, +} + +/// Walk a message line-by-line, rewriting any `at :N in +/// ::proc` frames using `lookup`. Lines that don't match the +/// stack-frame grammar (regular message prose) pass through +/// unchanged. Adjacent frames that resolve to the same +/// `(proc, line)` are collapsed — the Vivado kwargs-wrapper + +/// body-call doubling becomes a single rendered frame. +pub fn resolve_stack_frames_with( + msg: &str, + lookup: F, + input_file: Option<&Path>, +) -> String +where + F: Fn(&str) -> Option, +{ + let mut out = String::with_capacity(msg.len()); + let mut last_resolved_key: Option<(String, u32)> = None; + for (i, line) in msg.lines().enumerate() { + if i > 0 { + out.push('\n'); + } + let Some(rewritten) = rewrite_stack_line(line, &lookup, input_file) + else { + out.push_str(line); + last_resolved_key = None; + continue; + }; + let key = (rewritten.proc.clone(), rewritten.line); + if last_resolved_key.as_ref() == Some(&key) { + if out.ends_with('\n') { + out.pop(); + } + continue; + } + last_resolved_key = Some(key); + out.push_str(&rewritten.formatted); + } + out +} + +/// Parse a single line like ` at :14 in ::configure_cips` +/// and rewrite it to point at the user's actual htcl source. +/// Returns `None` when the line isn't a stack frame (regular +/// message text) or when the proc isn't one we know about (Vivado +/// builtins, dynamic procs, etc.) — caller passes such lines +/// through unchanged. +pub fn rewrite_stack_line( + line: &str, + lookup: F, + input_file: Option<&Path>, +) -> Option +where + F: Fn(&str) -> Option, +{ + // Grammar emitted by `vw::format_frame`: + // " at :N in ::procname" ← lookup ProcLocation by name + // " at :N in ::procname" ← already absolute + // " at :N" ← anonymous eval / top-level + // " at " ← location-less + let rest = line.strip_prefix(" at ")?; + let (loc_str, proc_part) = match rest.split_once(" in ") { + Some((l, p)) => (l, Some(p.trim().to_string())), + None => (rest, None), + }; + let (file_part, line_part) = loc_str.rsplit_once(':')?; + let body_line: u32 = line_part.parse().ok()?; + + // Top-level `:N` frame (no proc). + let Some(proc) = proc_part else { + if file_part != "" { + return None; + } + let path = input_file?; + return Some(RewrittenFrame { + proc: String::new(), + line: body_line, + formatted: format!(" at {}:{body_line}", display_path(path)), + }); + }; + + // Already-absolute frames don't need rewriting; pass through + // (dedup downstream still benefits from parsed proc+line). + if file_part != "" { + return Some(RewrittenFrame { + proc, + line: body_line, + formatted: line.to_string(), + }); + } + // `:N in ::proc` — Tcl reports "line N of the proc + // body." Resolve through the lookup. Tcl always reports + // fully-qualified names (leading `::`); the proc table + // indexes them without (see `lower::qualify`), so strip + // before lookup. + let lookup_name = proc.strip_prefix("::").unwrap_or(&proc); + let loc = lookup(lookup_name)?; + let (abs_line, _content) = loc.resolve_body_line(body_line)?; + let path_str = match loc.file.as_deref() { + Some(p) => display_path(p), + None => match input_file { + Some(p) => display_path(p), + None => "".to_string(), + }, + }; + Some(RewrittenFrame { + proc: proc.clone(), + line: abs_line, + formatted: format!(" at {path_str}:{abs_line} in {proc}"), + }) +} + +/// Pretty-print a file path for diagnostics: prefer the cwd- +/// relative form (`ip/cips.htcl`) when the path is under the +/// current working directory, then home-relative (`~/src/…`), +/// then the absolute form. Matches the REPL's scrollback so +/// vw run + vw repl render the same way. +pub fn display_path(path: &Path) -> String { + if let Ok(cwd) = std::env::current_dir() { + if let Ok(rel) = path.strip_prefix(&cwd) { + return rel.display().to_string(); + } + } + if let Ok(home) = std::env::var("HOME") { + let home_path = Path::new(&home); + if let Ok(rel) = path.strip_prefix(home_path) { + return format!("~/{}", rel.display()); + } + } + path.display().to_string() +} diff --git a/vw-repl/src/ui.rs b/vw-repl/src/ui.rs new file mode 100644 index 0000000..b1eca80 --- /dev/null +++ b/vw-repl/src/ui.rs @@ -0,0 +1,600 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! ratatui rendering for the REPL. +//! +//! Layout (top-to-bottom): +//! +//! ```text +//! ┌──────────────────────────────────────────┐ +//! │ scrollback ▲│ +//! │ (eval log, oldest top, newest bottom) ║│ +//! │ ▼│ +//! ├──────────────────────────────────────────┤ +//! │ input (multi-line, tui-textarea owned) │ +//! ├──────────────────────────────────────────┤ +//! │ status: vivado state | hints │ +//! └──────────────────────────────────────────┘ +//! ``` +//! +//! When Ctrl-R is active, a centered overlay replaces the input area +//! with the search query and the matching history entry. + +use crate::app::{App, ReverseSearch}; +use ratatui::layout::{Constraint, Direction, Layout, Rect}; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{ + Block, Borders, Clear, Paragraph, Scrollbar, ScrollbarOrientation, + ScrollbarState, Wrap, +}; +use ratatui::Frame; + +pub fn draw(f: &mut Frame, app: &mut App) { + let layout = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Min(1), // scrollback (fills) + Constraint::Length(input_height(app)), // input + Constraint::Length(1), // status bar + ]) + .split(f.area()); + + draw_scrollback(f, layout[0], app); + draw_input(f, layout[1], app); + draw_status(f, layout[2], app); + + if let Some(rs) = app.reverse_search() { + draw_reverse_search(f, layout[1], rs); + } + if let Some(popup) = app.popup_state() { + match popup { + crate::popup::PopupState::Completion(c) => { + crate::popup::draw_completion_popup(f, c, f.area()); + } + crate::popup::PopupState::SignatureHelp(s) => { + crate::popup::draw_signature_help_popup(f, s, f.area()); + } + crate::popup::PopupState::Hover(h) => { + crate::popup::draw_hover_popup(f, h, f.area()); + } + crate::popup::PopupState::Help(_) => { + crate::popup::draw_help_popup(f, f.area()); + } + crate::popup::PopupState::SymbolSearch(p) => { + crate::symbol_search::draw_symbol_picker(f, p); + } + crate::popup::PopupState::DiagnosticSearch(p) => { + crate::diag_search::draw_diagnostic_picker(f, p); + } + } + } +} + +/// Per-entry wrapped-row counts, with hidden children of a +/// collapsed input group forced to 0. The renderer + viewport +/// math treat count=0 entries as if they weren't in the buffer +/// (the "skip above viewport" branch triggers immediately; the +/// windowed-slicer sees an empty range and returns nothing), so +/// this cleanly hides them without special-casing every consumer. +/// +/// Kept as a small `ui`-local helper rather than pushed down into +/// `render::count_wrapped_rows` because visibility is a +/// cross-entry property (each row needs its PARENT'S state) and +/// pushing it down would force the signature to accept the +/// whole scrollback slice. +fn compute_visible_counts(app: &App, width: u16) -> Vec { + let sb = app.scrollback(); + sb.iter() + .map(|entry| { + if let Some(parent_idx) = entry.parent_input_idx { + if let Some(parent) = sb.get(parent_idx) { + if parent.group_collapsed { + return 0; + } + } + } + crate::render::count_wrapped_rows(entry, width) + }) + .collect() +} + +fn input_height(app: &App) -> u16 { + // Start at a 5-line minimum so the user has room to draft a + // multi-statement entry without the input box flickering taller + // mid-typing. Grows past 5 with the buffer, capped at 12 so + // very long entries don't squeeze the scrollback out. + let lines = app.input_line_count().clamp(5, 12) as u16; + lines + 2 // +2 for the top/bottom block border +} + +fn draw_scrollback(f: &mut Frame, area: Rect, app: &mut App) { + if area.width == 0 || area.height == 0 { + app.set_scrollback_area(area); + return; + } + + // Pass 1: cheap per-entry wrapped-row count, no allocations. + // O(text length) for each, vs. the old approach which built + // and wrapped every entry per draw — turning a single huge + // entry into multi-MB of per-char allocation on every wheel + // tick. With this pass, total work per draw is O(scrollback) + // for counting + O(viewport) for actually wrapping the + // visible window. + // + // Auto-hide scrollbar policy: count once at full width to + // decide "does content fit?"; if it doesn't, narrow the + // paragraph by one column to make room for the scrollbar and + // recount. This costs a second pass ONLY on frames where the + // scrollbar is visible — the common overflow-free case pays + // just one pass. + // Row counts, zeroed for entries whose parent input's group is + // collapsed. `count_wrapped_rows` itself doesn't know about + // grouping (it only sees one entry at a time), so the visibility + // filter lives here. Result: hidden children contribute 0 rows + // to the scrollback total AND consume 0 rows in the viewport + // math downstream, exactly as if they weren't in the buffer. + let counts_full: Vec = compute_visible_counts(app, area.width); + let total_full: u32 = + counts_full.iter().fold(0u32, |a, b| a.saturating_add(*b)); + let needs_scrollbar = total_full > area.height as u32; + + // `paragraph_area` is where wrapped text actually renders. + // When the scrollbar is on, it takes the rightmost column of + // `area`, leaving `area.width - 1` for text. Also the + // scrollback_area we hand back to App (for mouse coord + // mapping) so a click on the scrollbar column doesn't get + // interpreted as a text-selection click. + let (paragraph_area, counts, total) = if needs_scrollbar && area.width >= 2 + { + let narrower = Rect { + width: area.width - 1, + ..area + }; + let cs: Vec = compute_visible_counts(app, narrower.width); + let t: u32 = cs.iter().fold(0u32, |a, b| a.saturating_add(*b)); + (narrower, cs, t) + } else { + (area, counts_full, total_full) + }; + // Hand paragraph_area (not `area`) back to App: mouse-coord + // translation now excludes the scrollbar column. + app.set_scrollback_area(paragraph_area); + + let max_scroll = total.saturating_sub(paragraph_area.height as u32); + // Consume any pending "jump this entry into view" request from + // the diagnostics-finder popup. Translation happens here (not + // in App) because it needs the per-entry wrapped-row counts we + // just computed — the popup key handler doesn't know + // area.width. Center-aligned when possible so the marked + // entry sits in the middle of the viewport; otherwise + // clamped by max_scroll. + if let Some(jump_idx) = app.take_pending_jump() { + let mut acc: u32 = 0; + for (i, c) in counts.iter().enumerate() { + if i == jump_idx { + break; + } + acc = acc.saturating_add(*c); + } + let center_offset = acc + .saturating_sub(paragraph_area.height as u32 / 3) + .min(max_scroll) + .min(u32::from(u16::MAX)) as u16; + // Writing scrollback_scroll (via App's setter) requires a + // path through the crate; simpler is to write the offset + // through set_last_rendered_scroll AND scrollback_scroll — + // we do both here so tail-follow disengage (done by the + // popup handler) sticks and the effective offset the + // renderer uses below is the jumped one. + app.set_scrollback_scroll(center_offset); + } + let scroll_offset = if app.scrollback_follow() { + max_scroll + } else { + u32::from(app.scrollback_scroll()).min(max_scroll) + }; + app.set_last_rendered_scroll(scroll_offset.min(u32::from(u16::MAX)) as u16); + app.set_last_max_scroll(max_scroll.min(u32::from(u16::MAX)) as u16); + + // Pass 2: walk entries; build wrapped lines only for those + // intersecting the viewport. Entries entirely above viewport + // are skipped (their row count contributes to the offset we + // pass to ratatui's `Paragraph::scroll`). Entries entirely + // below viewport stop the walk. + let viewport_start = scroll_offset; + let viewport_end = + viewport_start.saturating_add(paragraph_area.height as u32); + + let mut visible: Vec> = + Vec::with_capacity(paragraph_area.height as usize + 16); + let mut accumulated: u32 = 0; + // `skipped_rows` counts wrapped rows preceding the first row we + // actually emit into `visible`. For entries fully above the + // viewport we add their whole `count`; for the FIRST partially- + // visible entry we add whatever wrapped rows come between the + // entry's start and the first source line the windowed slicer + // emits. Combined, this keeps `local_scroll = viewport_start - + // skipped_rows` correct even when we slice inside an entry. + let mut skipped_rows: u32 = 0; + { + let scrollback = app.scrollback(); + for (entry, &count) in scrollback.iter().zip(counts.iter()) { + // Hidden entries — a child of a collapsed input group, + // per `compute_visible_counts` — get count=0. Skip + // them explicitly: the `entry_end <= viewport_start` + // branch below only triggers when we've already + // walked past `viewport_start`, so a count=0 entry + // sitting INSIDE the viewport (accumulated < + // viewport_start hasn't happened yet) would otherwise + // slip past both guards and call + // `entry_lines_windowed`, which doesn't know about + // visibility and would happily render the entry's + // real content. That would defeat the whole + // group-collapse and put the ▶ marker on an Input + // that visually still expands. + if count == 0 { + continue; + } + let entry_end = accumulated.saturating_add(count); + if entry_end <= viewport_start { + // Entirely above viewport — count its rows toward + // the local scroll offset and move on without + // wrapping. + skipped_rows = entry_end; + accumulated = entry_end; + continue; + } + if accumulated >= viewport_end { + break; + } + // Windowed slice: emit only source lines whose wrapped- + // row range overlaps the viewport. Local window is in + // this entry's own row space (0-indexed), so subtract + // `accumulated` first. + let local_start = viewport_start.saturating_sub(accumulated); + let local_end = viewport_end.saturating_sub(accumulated); + let (lines, offset) = crate::render::entry_lines_windowed( + entry, + paragraph_area.width, + local_start..local_end, + ); + // Only the FIRST windowed entry contributes an intra- + // entry offset; subsequent entries fall wholly within + // the viewport and start rendering at row 0 locally. + // For the first sliced entry, add the wrapped rows the + // slicer skipped (source lines above the visible + // window) to `skipped_rows` so downstream math stays + // symmetric with the whole-entry-skipped case. + if visible.is_empty() { + skipped_rows = + skipped_rows.saturating_add(local_start - offset); + } + let wrapped = + crate::render::wrap_lines(lines, paragraph_area.width); + visible.extend(wrapped); + accumulated = entry_end; + } + } + + // Selection highlight: coords are global wrapped-row indices. + // Subtract `skipped_rows` so they index into the local + // `visible` Vec instead. + if let Some(sel) = app.selection() { + let (start, end) = sel.ordered(); + let skipped = skipped_rows as usize; + let local_start = (start.0.saturating_sub(skipped), start.1); + let local_end = (end.0.saturating_sub(skipped), end.1); + crate::render::apply_selection_highlight( + &mut visible, + local_start, + local_end, + ); + } + + // We've already skipped entries above viewport; ratatui only + // needs to skip the remaining rows within the first visible + // entry (i.e. the offset from where that entry started to + // where the viewport actually begins). + let local_scroll = viewport_start + .saturating_sub(skipped_rows) + .min(u32::from(u16::MAX)) as u16; + + // Blank the scrollback area first. ratatui's Paragraph doesn't + // guarantee overwriting cells past its own content, so a frame + // that emits shorter/fewer wrapped lines than the previous one + // (very common under live streaming — a new INFO chunk shifts + // the viewport and the tail cells of the prior frame stay + // dirty) shows up as leftover fragments in the wrong color, + // usually looking like `INFO:` / `.v:` / hex-digit tails + // grafted onto the front of the current line. `Clear` writes + // spaces with the default style over `area`, so subsequent + // paragraph render lands on a clean slate. Clear covers the + // full `area` (including any scrollbar column) — Scrollbar + // will paint over that column below. + f.render_widget(Clear, area); + // No surrounding block: the scrollback's main job is to be + // copy-pastable. A box-drawing border around each visible row + // means any selection that spans full lines pulls in `│` chars + // at the start and end of every line. The input box below the + // scrollback still has its own border, which provides enough + // visual separation between the two regions. + let paragraph = Paragraph::new(visible).scroll((local_scroll, 0)); + f.render_widget(paragraph, paragraph_area); + + // Marker overlay for the entry the user jumped to via + // the diagnostics finder (Ctrl-F → Enter). Persistent until + // Alt-C clears it. Paints a bright colored bar in the + // leftmost column of every visible wrapped row of the marked + // entry. Color follows the entry's kind (red for Error, + // orange for Warning, gray for Notice) so the marker also + // reinforces the severity at a glance. + if let Some(marker_idx) = app.marker_entry() { + // Compute absolute row range of the marked entry. + let mut acc: u32 = 0; + let mut marker_range: Option<(u32, u32)> = None; + for (i, c) in counts.iter().enumerate() { + if i == marker_idx { + marker_range = Some((acc, acc.saturating_add(*c))); + break; + } + acc = acc.saturating_add(*c); + } + if let Some((start, end)) = marker_range { + // Intersect with viewport row range. + let iso_start = start.max(scroll_offset); + let iso_end = end.min( + scroll_offset.saturating_add(paragraph_area.height as u32), + ); + if iso_start < iso_end { + let kind = app.scrollback().get(marker_idx).map(|e| e.kind); + let marker_color = match kind { + Some(crate::app::ScrollbackKind::Error) => Color::Red, + Some(crate::app::ScrollbackKind::Warning) => { + Color::Rgb(255, 140, 0) + } + Some(crate::app::ScrollbackKind::Notice) => { + Color::Rgb(180, 130, 220) + } + _ => Color::Rgb(180, 130, 220), + }; + let marker_style = Style::default() + .fg(marker_color) + .add_modifier(Modifier::BOLD); + let buf = f.buffer_mut(); + for row in iso_start..iso_end { + let y = paragraph_area.y + (row - scroll_offset) as u16; + // `▎` = left one-eighth block — reads as a + // continuous vertical bar down the left edge + // when stacked across rows. + buf.set_string(paragraph_area.x, y, "▎", marker_style); + } + } + } + } + + // Scrollbar overlay — only when content overflows the viewport. + // ScrollbarState's `content_length` is the total scrollable + // range (max_scroll + 1 so the thumb can reach the very + // bottom); `position` is the current scroll offset. Rendered + // on the rightmost column of `area`; the paragraph was drawn + // in `paragraph_area` which excludes that column so there's + // no overpaint on wrapped text. + if needs_scrollbar && area.width >= 2 { + let content_len = max_scroll.saturating_add(1) as usize; + let mut sb_state = ScrollbarState::new(content_len) + .position(scroll_offset as usize) + .viewport_content_length(paragraph_area.height as usize); + let scrollbar = Scrollbar::default() + .orientation(ScrollbarOrientation::VerticalRight); + f.render_stateful_widget(scrollbar, area, &mut sb_state); + } +} + +fn draw_input(f: &mut Frame, area: Rect, app: &mut App) { + // TextArea remains the editing model (every keystroke still flows + // through `ta.input(key)` in `handle_terminal_event`'s catch-all + // arm). We replace ONLY the visual rendering layer here so we can + // paint per-token highlighter spans — tui-textarea's built-in + // renderer is monochrome and its `line_spans` is `pub(crate)`, so + // there's no hook for syntax styling without taking over the + // viewport ourselves. + let block = Block::default() + .borders(Borders::ALL) + .title(input_title(app)); + let inner = block.inner(area); + f.render_widget(block, area); + if inner.width == 0 || inner.height == 0 { + return; + } + + let ta = app.input_mut(); + let lines: Vec = ta.lines().to_vec(); + let (cursor_row, cursor_col) = ta.cursor(); + + // Run the htcl highlighter on the full buffer in one parse, then + // slice per-line for rendering. Per-frame cost is one + // `vw_htcl::parse` over the input text — fine for typical REPL + // inputs (tens of lines); revisit if it shows up in a profile. + let body_style = + ratatui::style::Style::default().fg(ratatui::style::Color::White); + let buffer = lines.join("\n"); + let highlighted = + crate::highlight_htcl::highlight_per_line(&buffer, body_style); + + // Vertical viewport: anchor so the cursor stays visible. When the + // buffer fits, anchor at top; when it overflows, scroll so cursor + // is on the last visible row. + let view_h = inner.height as usize; + let scroll_top = if lines.len() <= view_h { + 0 + } else { + cursor_row.saturating_sub(view_h.saturating_sub(1)) + }; + + // Render each visible line as a single-row Paragraph. + for (visible_idx, line_idx) in (scroll_top..lines.len()).enumerate() { + if visible_idx >= view_h { + break; + } + let row = inner.y + visible_idx as u16; + let line_spans = highlighted.get(line_idx).cloned().unwrap_or_default(); + let line_widget = Paragraph::new(ratatui::text::Line::from(line_spans)); + let line_area = Rect { + x: inner.x, + y: row, + width: inner.width, + height: 1, + }; + f.render_widget(line_widget, line_area); + } + + // Position the terminal-native text cursor so the user sees their + // edit point. Only set when the cursor row is visible — when + // scrolled away (shouldn't happen given the viewport math above, + // but defensive), we just leave the cursor hidden. + if cursor_row >= scroll_top && cursor_row < scroll_top + view_h { + let cursor_screen_row = inner.y + (cursor_row - scroll_top) as u16; + // Horizontal: clamp to area width. Cursor positions past the + // visible width get pinned to the last column rather than + // drifting off the block border. + let cursor_screen_col = + inner.x + (cursor_col as u16).min(inner.width.saturating_sub(1)); + f.set_cursor_position((cursor_screen_col, cursor_screen_row)); + } +} + +fn input_title(app: &App) -> String { + if app.eval_in_flight() { + " input — vivado: running ".to_string() + } else if app.input_is_complete() { + " input — Enter to run ".to_string() + } else { + " input — Enter for newline (parse incomplete) ".to_string() + } +} + +fn draw_status(f: &mut Frame, area: Rect, app: &App) { + let (label, bg) = match app.worker_state() { + // Indigo when Vivado is sitting idle, ready for input — + // the "you can interact" steady state. + WorkerStatusView::Ready => (" vivado: ready ", Color::Rgb(75, 0, 130)), + // Orange for transient / busy states — starting up or + // mid-eval. Catches the eye so the user notices they + // can't (yet, or currently) drive the session, but + // signals "wait" rather than "broken". + WorkerStatusView::Starting => { + (" vivado: starting ", Color::Rgb(255, 140, 0)) + } + WorkerStatusView::Running => { + (" vivado: running ", Color::Rgb(255, 140, 0)) + } + // Red for `down` — distinct from the orange busy states + // so the user can immediately tell the difference between + // "wait a moment" and "the worker died, `:restart` to + // recover". Sharing orange with `running` hid this. + WorkerStatusView::Down => (" vivado: down ", Color::Rgb(200, 30, 30)), + }; + let hint = if app.reverse_search().is_some() { + "Esc cancel · Enter accept · Ctrl-R older".to_string() + } else { + // Single key-chord hint — the full cheat-sheet lives in + // the Ctrl-H modal so we don't have to keep this row in + // sync with every binding we add or change. + "Ctrl-H for help".to_string() + }; + // Split the status bar into [hint (left, fills) | status + // indicator (right, fixed width)] so the status badge always + // anchors to the bottom-right corner. + let badge_width = label.chars().count() as u16; + let layout = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Min(1), Constraint::Length(badge_width)]) + .split(area); + f.render_widget( + Paragraph::new(Span::styled(hint, Style::default().fg(Color::Gray))), + layout[0], + ); + f.render_widget( + Paragraph::new(Span::styled( + label, + Style::default() + .bg(bg) + .fg(Color::White) + .add_modifier(Modifier::BOLD), + )), + layout[1], + ); +} + +fn draw_reverse_search(f: &mut Frame, anchor: Rect, rs: &ReverseSearch) { + let area = centered_rect(80, 5, f.area(), anchor); + f.render_widget(Clear, area); + let title = format!( + " reverse-i-search ({}) ", + if rs.match_index.is_some() { + "match" + } else if rs.query.is_empty() { + "type to search" + } else { + "no match" + } + ); + let body = vec![ + Line::from(vec![ + Span::styled("query: ", Style::default().fg(Color::Gray)), + Span::styled( + rs.query.clone(), + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ), + ]), + Line::from(Span::raw("")), + Line::from(vec![ + Span::styled("match: ", Style::default().fg(Color::Gray)), + Span::raw(rs.match_text.clone()), + ]), + ]; + let para = Paragraph::new(body) + .block( + Block::default() + .borders(Borders::ALL) + .title(title) + .style(Style::default().bg(Color::Black)), + ) + .wrap(Wrap { trim: false }); + f.render_widget(para, area); +} + +/// Compute a centered rectangle for a popup. `anchor` is the area the +/// popup is logically attached to (the input area); we expand around +/// the screen center but never overflow the parent. +fn centered_rect( + percent_x: u16, + height_lines: u16, + full: Rect, + _anchor: Rect, +) -> Rect { + let popup_w = full.width.saturating_mul(percent_x) / 100; + let popup_h = height_lines.min(full.height); + let x = (full.width.saturating_sub(popup_w)) / 2; + let y = (full.height.saturating_sub(popup_h)) / 2; + Rect { + x, + y, + width: popup_w, + height: popup_h, + } +} + +/// Worker status as the UI sees it. Lives here (not in `app`) so the +/// renderer doesn't have to know about the worker's internal state +/// machine. +pub enum WorkerStatusView { + Starting, + Ready, + Running, + Down, +} diff --git a/vw-svc/Cargo.toml b/vw-svc/Cargo.toml new file mode 100644 index 0000000..0ec1972 --- /dev/null +++ b/vw-svc/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "vw-svc" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "VW as a service endpoint" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools", "command-line-utilities"] + +[[bin]] +name = "vw-svc" +path = "src/main.rs" + +[dependencies] +vw-lib = { path = "../vw-lib" } +vw-api = { path = "../vw-api" } +vw-api-client = { path = "../vw-api-client" } +vw-api-types-versions = { path = "../vw-api-types/versions" } +camino.workspace = true +dropshot.workspace = true +rustls.workspace = true +rustls-pemfile.workspace = true +thiserror.workspace = true +clap.workspace = true +tokio.workspace = true +oxide.workspace = true +serde.workspace = true +serde_json.workspace = true +iddqd.workspace = true +bytes = "1" +futures.workspace = true +http-body = "1" +http-body-util = "0.1" +tokio-stream = "0.1" +rust-s3 = { version = "0.37", default-features = false, features = ["with-tokio"] } +tokio-tungstenite.workspace = true +slog = "2.8.2" +slog-error-chain = { git = "https://github.com/oxidecomputer/slog-error-chain", branch = "main" } +slog-bunyan = "2.5.0" +slog-async = "2.8.0" +blake3 = "1" +redb = "4.1.0" +reqwest = { version = "0.13", features = ["json"] } +uuid.workspace = true +daft = { version = "0.1.8", features = ["derive"] } +ssh-key = { version = "0.6", features = ["ed25519", "rand_core", "getrandom"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-svc/dist/README.md b/vw-svc/dist/README.md new file mode 100644 index 0000000..7f6b4d6 --- /dev/null +++ b/vw-svc/dist/README.md @@ -0,0 +1,80 @@ +# Running vw-svc + +Deployment files for vw-svc on a systemd host. + +| File | Installed to | +| --- | --- | +| [`vw-svc.service`](vw-svc.service) | `/etc/systemd/system/vw-svc.service` | +| [`vw-svc.env.example`](vw-svc.env.example) | `/etc/vw-svc/vw-svc.env`, mode 0600, once | +| [`install.sh`](install.sh) | — | + +## Install + +```sh +cargo build --release -p vw-svc +sudo ./vw-svc/dist/install.sh +``` + +Or from a build CI already did, which is the same binary the images get their +agent from: + +```sh +sudo ./vw-svc/dist/install.sh --commit +``` + +The first install leaves the service enabled and stopped, because the +configuration it just wrote is the example. Edit `/etc/vw-svc/vw-svc.env`, then +`systemctl start vw-svc`. + +Re-running is safe. The binary and the unit are replaced; the configuration +file never is. A running service is **not** restarted unless you pass +`--restart` — vw-svc relays the connections builds run over, so a restart ends +whatever synthesis, REPL session or artifact download is in flight, and when to +do that is your call. + +## Configuration + +Everything site-specific is in `/etc/vw-svc/vw-svc.env`: which certificate to +serve, which rack to provision on, who administers the service. The unit reads +it and nothing else, so reinstalling never disturbs how a machine is set up. + +Each `VW_SVC_*` variable is split on whitespace into arguments, so values may +not contain spaces or quoting. `OXIDE_TOKEN` is the exception — vw-svc reads +that one from the environment by name. + +## Certificates + +vw-svc serves TLS from a certificate on disk and watches it. Get one from +Let's Encrypt: + +```sh +certbot certonly --standalone -d vw.example.com +``` + +Then point `VW_SVC_TLS` at the `live/` symlinks, not at the files under +`archive/`. Renewals are certbot's own systemd timer (`certbot.timer`, twice +daily, a no-op until a certificate is within 30 days of expiry) — there is no +deploy hook to configure and nothing to restart. vw-svc notices the replaced +certificate within a minute and serves it from the next handshake on; +connections already established are untouched. + +The service runs as root for this reason: certbot keeps `/etc/letsencrypt/live` +and `archive` at `0700 root` and re-creates them on each renewal, so any +group-readable arrangement made once does not survive. The unit is sandboxed +accordingly — read-only filesystem, no home, restricted syscalls. + +### Testing renewal before it happens for real + +Renewal will not fire for about two months, and `certbot renew --dry-run` +writes to a temporary directory, so it never touches the files vw-svc watches. +To exercise the whole path now: + +```sh +certbot renew --force-renewal +journalctl -u vw-svc | grep -i certificate +``` + +Expect `certificate replaced` followed by `now serving the replaced +certificate` from both `user_api` and `admin_api`, within a minute, with the +PID unchanged. Once is enough: `--force-renewal` counts against Let's Encrypt's +limit of five duplicate certificates per week. diff --git a/vw-svc/dist/install.sh b/vw-svc/dist/install.sh new file mode 100755 index 0000000..6b2b159 --- /dev/null +++ b/vw-svc/dist/install.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# +# Install vw-svc as a systemd service. +# +# Safe to re-run: the binary and the unit are replaced, the configuration file +# is not. A first install leaves the service enabled but stopped, because the +# configuration it was just given is an example and starting on it would only +# produce a confusing failure. +# +# ./install.sh # from ../../target/release/vw-svc +# ./install.sh --binary /path/to/vw-svc +# ./install.sh --commit # from that commit's buildomat build +# ./install.sh --restart # and restart a running service + +set -euo pipefail + +BUILDOMAT=https://buildomat.eng.oxide.computer/public/file/oxidecomputer/vw +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +BINARY="" +COMMIT="" +RESTART=false + +while [ $# -gt 0 ]; do + case "$1" in + --binary) BINARY="$2"; shift 2 ;; + --commit) COMMIT="$2"; shift 2 ;; + --restart) RESTART=true; shift ;; + -h | --help) sed -n '2,13p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +[ "$(id -u)" -eq 0 ] || { echo "install.sh must run as root" >&2; exit 1; } + +# Resolve what is being installed before touching anything, so a bad path or +# an unreachable buildomat fails before the unit has been replaced. +scratch="" +if [ -n "$COMMIT" ]; then + [ -z "$BINARY" ] || { echo "--binary and --commit are exclusive" >&2; exit 2; } + scratch="$(mktemp -d)" + trap 'rm -rf "$scratch"' EXIT + BINARY="$scratch/vw-svc" + echo "Fetching vw-svc from vw $COMMIT" + curl --proto '=https' --tlsv1.2 -fL -o "$BINARY" "$BUILDOMAT/linux/$COMMIT/vw-svc" + chmod +x "$BINARY" +fi +: "${BINARY:=$HERE/../../target/release/vw-svc}" + +[ -x "$BINARY" ] || { + echo "no vw-svc binary at $BINARY" >&2 + echo "build one with 'cargo build --release -p vw-svc', or pass --binary/--commit" >&2 + exit 1 +} + +# Run it once here. A binary that cannot start on this machine should say so +# now rather than as a restart loop after the unit is in place. +"$BINARY" serve --help >/dev/null + +echo "Installing /usr/local/bin/vw-svc" +install -o root -g root -m 0755 "$BINARY" /usr/local/bin/vw-svc + +install -d -o root -g root -m 0755 /etc/vw-svc + +# Never overwritten. It holds the rack token and everything about how this +# machine is configured, and an upgrade has no business resetting either. +fresh=false +if [ -e /etc/vw-svc/vw-svc.env ]; then + echo "Keeping /etc/vw-svc/vw-svc.env" +else + echo "Installing /etc/vw-svc/vw-svc.env" + install -o root -g root -m 0600 \ + "$HERE/vw-svc.env.example" /etc/vw-svc/vw-svc.env + fresh=true +fi + +echo "Installing /etc/systemd/system/vw-svc.service" +install -o root -g root -m 0644 \ + "$HERE/vw-svc.service" /etc/systemd/system/vw-svc.service + +systemctl daemon-reload +systemctl enable vw-svc.service >/dev/null + +if $fresh; then + cat <<-EOF + + vw-svc is installed and enabled, and has not been started. + + Edit /etc/vw-svc/vw-svc.env first. As shipped it names a + certificate that does not exist, and configures no rack -- so + starting on it would stop at the missing certificate, and + fixing only that would give you a service that records + environments and provisions nothing. Then: + + systemctl start vw-svc + journalctl -fu vw-svc + EOF + exit 0 +fi + +if systemctl is-active --quiet vw-svc.service; then + if $RESTART; then + echo "Restarting vw-svc" + systemctl restart vw-svc.service + else + # Deliberately not automatic. This service relays the connections + # builds run over, so a restart ends whatever synthesis runs, REPL + # sessions and downloads are in flight. Picking the moment for that + # is the operator's call. + cat <<-EOF + + The new binary is installed; the running service is still the + old one. Restarting ends any build, REPL session or download + currently being relayed, so it is left to you: + + systemctl restart vw-svc + + Or re-run this with --restart. + EOF + fi +else + echo + echo "vw-svc is installed and enabled. Start it with: systemctl start vw-svc" +fi diff --git a/vw-svc/dist/vw-svc.env.example b/vw-svc/dist/vw-svc.env.example new file mode 100644 index 0000000..fa8bcd5 --- /dev/null +++ b/vw-svc/dist/vw-svc.env.example @@ -0,0 +1,57 @@ +# Configuration for vw-svc, installed at /etc/vw-svc/vw-svc.env. +# +# Mode 0600: OXIDE_TOKEN below is a rack credential. install.sh will never +# overwrite this file once it exists, so editing it in place is safe across +# reinstalls and upgrades. +# +# Each VW_SVC_* variable is split on whitespace into arguments to +# `vw-svc serve`. That means no value here may contain spaces or quoting -- +# they are argument lists, not shell. Leave one empty to pass nothing. +# Everything is optional: with no rack configured the service still keeps +# environment records, it just never provisions anything. + +# --- TLS --------------------------------------------------------------- +# Point these at certbot's live symlinks rather than at the files in +# archive/. The service follows the symlinks and notices within a minute +# when a renewal repoints them, so a renewed certificate is served without a +# restart and without dropping any connection. +# +# Get the certificate once with: +# certbot certonly --standalone -d vw.example.com +# +# Left uncommented, unlike the rack settings below, because getting this +# wrong fails cleanly: a certificate that is not there stops the service at +# startup with a message naming the path. Deleting the line instead serves +# plain HTTP, which is a worse thing to do by accident. +VW_SVC_TLS=--tls --cert-file /etc/letsencrypt/live/vw.example.com/fullchain.pem --key-file /etc/letsencrypt/live/vw.example.com/privkey.pem + +# --- Rack -------------------------------------------------------------- +# The Oxide silo instances are created in. The token is read from the +# environment by name, so it is the one setting that is not an argument. +# +# Commented out on purpose, and the two must be uncommented together: +# +# both set the service provisions instances +# neither set the service records environments and provisions +# nothing, which is what an unconfigured install should +# do and what it says in the log +# endpoint only refuses to start, naming --oxide-token +# endpoint set and OXIDE_TOKEN= left empty +# starts, believes it has a rack, and fails every call +# against it -- the one state worth avoiding, which is +# why there is no empty assignment here to inherit +# +#VW_SVC_RACK=--oxide-api-endpoint https://oxide.sys.example.com --oxide-project redhawk +#OXIDE_TOKEN= + +# --- Who administers this service -------------------------------------- +# Github usernames. An administrator can see and delete every environment on +# the rack, including other people's; everybody else sees only their own. +VW_SVC_ADMINS=--admin-users you + +# --- Anything else ----------------------------------------------------- +# Ports, reconciler interval, and so on. `vw-svc serve --help` lists them. +# --db-path is set by the unit and must not be repeated here. +# +# VW_SVC_EXTRA=--user-api-port 2727 --admin-api-port 2728 +VW_SVC_EXTRA= diff --git a/vw-svc/dist/vw-svc.service b/vw-svc/dist/vw-svc.service new file mode 100644 index 0000000..8053c1b --- /dev/null +++ b/vw-svc/dist/vw-svc.service @@ -0,0 +1,65 @@ +# The vw service. +# +# Nothing site-specific lives here. Which certificate to serve, which rack to +# provision on and who administers it all come from /etc/vw-svc/vw-svc.env, so +# that reinstalling the unit never overwrites how a machine is configured. + +[Unit] +Description=vw service +Documentation=https://github.com/oxidecomputer/vw +After=network-online.target +Wants=network-online.target + +[Service] +Type=exec + +# Root because of the certificate. Certbot keeps /etc/letsencrypt/live and +# archive at 0700 root and re-creates them on every renewal, so a +# group-readable arrangement made once does not survive the first renewal -- +# and the alternative, handing the service a copy, is a copy that cannot be +# hot-reloaded. The sandboxing below gives back most of what this costs. +User=root + +# 0600, because it holds the rack token. +EnvironmentFile=/etc/vw-svc/vw-svc.env + +# The variables are deliberately unquoted: systemd splits them into separate +# arguments, which is the whole reason the configuration can live in a file +# rather than in this unit. An unset one expands to nothing. +# +# --db-path is fixed here rather than left to the environment file because it +# follows from StateDirectory below, and is not something a site chooses. +ExecStart=/usr/local/bin/vw-svc serve \ + --db-path /var/lib/vw-svc/vw-svc.redb \ + $VW_SVC_TLS $VW_SVC_RACK $VW_SVC_ADMINS $VW_SVC_EXTRA + +# Creates /var/lib/vw-svc on every start. The environment database lives here +# and outlives the service: it is the record of which environments exist. +StateDirectory=vw-svc + +# Environments outlive any one run of this service, and the reconciler puts +# them right when it comes back. +Restart=always +RestartSec=5 + +# Hardening. This reads a certificate, writes one database, and talks to a +# rack and to agents; it has no business doing anything else. ProtectSystem +# leaves /etc/letsencrypt readable, which is all the certificate watch needs. +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +SystemCallFilter=@system-service + +[Install] +WantedBy=multi-user.target diff --git a/vw-svc/src/admin_api.rs b/vw-svc/src/admin_api.rs new file mode 100644 index 0000000..6d60395 --- /dev/null +++ b/vw-svc/src/admin_api.rs @@ -0,0 +1,140 @@ +//! This module implements the admin api trait `[vw_api::VwAdminApi]` +//! +//! Everything here reaches across users, which is the whole reason it is a +//! separate API on a separate port: the user API can only ever see the caller's +//! own environments, and that property is easier to keep when the endpoints +//! that break it do not sit beside it. + +use crate::{auth, db, Context, ServerArgs}; +use dropshot::{ApiDescription, BuildError, ConfigDropshot}; +use slog::{info, o}; +use std::{net::SocketAddr, sync::Arc}; +use tokio::sync::Notify; +use vw_api::VwAdminApi; +use vw_api_types_versions::latest::UserEnvironmentPathParam; + +pub struct AdminApi {} +impl VwAdminApi for AdminApi { + type Context = Arc; + + async fn get_environments( + rqctx: dropshot::RequestContext, + ) -> Result< + dropshot::HttpResponseOk< + dropshot::ResultsPage< + vw_api_types_versions::latest::UserEnvironment, + >, + >, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let caller = auth::authorize_administrator(rqctx).await?; + + let environments = db::list_all_environments().inspect_err(|e| { + slog::error!(log, "cannot list environments"; + slog_error_chain::InlineErrorChain::new(e), + ); + })?; + + info!(log, "listed every environment"; + "administrator" => &caller.name, + "environments" => environments.len(), + ); + + // One complete page: this endpoint takes no pagination parameters, and + // the number of environments on a rack is bounded by the number of + // developers using it. If that ever stops being true this becomes a + // `ResultsPage::new` with a selector keyed on user and name. + Ok(dropshot::HttpResponseOk(dropshot::ResultsPage { + next_page: None, + items: environments, + })) + } + + async fn delete_environment( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::UserEnvironmentPathParam, + >, + ) -> Result { + let log = rqctx.log.clone(); + let reconcile = rqctx.context().reconcile.clone(); + let caller = auth::authorize_administrator(rqctx).await?; + let key: UserEnvironmentPathParam = path_params.into_inner(); + + // Named in the log before it happens as well as after: this is one + // person removing another person's work, and the record of who did it + // should survive whatever the deletion does next. + info!(log, "deleting an environment on behalf of the service"; + "administrator" => &caller.name, + "user" => &key.user, + "environment" => &key.name, + ); + + db::delete_environment(key.clone()).inspect_err(|e| { + slog::error!(log, "cannot delete an environment"; + "user" => &key.user, + "environment" => &key.name, + slog_error_chain::InlineErrorChain::new(e), + ); + })?; + + // Tear the instances down now rather than on the next tick, so a rack + // an administrator is reclaiming starts emptying immediately. + reconcile.notify_one(); + + Ok(dropshot::HttpResponseDeleted()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum StartServerError { + #[error("Server build error {0}")] + ServerBuildError(#[from] BuildError), + #[error("Unexpected server exit {0}")] + ServerExit(String), +} + +pub async fn start_server( + server_args: ServerArgs, + log: slog::Logger, + bind_address: SocketAddr, + tls: Option, + reconcile: Arc, +) -> Result<(), StartServerError> { + let scheme = crate::tls::scheme(&server_args); + let context = Arc::new(Context { + server_args, + reconcile, + }); + let cfg = ConfigDropshot { + bind_address, + default_request_body_max_bytes: usize::MAX, + ..Default::default() + }; + let lg = log.new(o!("component" => "admin_api")); + let api = api_description(); + + // Shared rather than owned so that the certificate can be replaced under a + // server that is already running. Dropping the last handle shuts the + // server down, so this one outlives the follower task below. + let server = Arc::new( + dropshot::ServerBuilder::new(api, context, lg.clone()) + .config(cfg) + .tls(crate::tls::initial(tls.as_ref())) + .start()?, + ); + + info!(lg, "listening on {scheme}://{}", server.local_addr()); + + crate::tls::follow_renewals(server.clone(), tls, lg.clone()); + + server + .wait_for_shutdown() + .await + .map_err(StartServerError::ServerExit) +} + +pub fn api_description() -> ApiDescription> { + vw_api::vw_admin_api_mod::api_description::().unwrap() +} diff --git a/vw-svc/src/artifacts.rs b/vw-svc/src/artifacts.rs new file mode 100644 index 0000000..97c2b37 --- /dev/null +++ b/vw-svc/src/artifacts.rs @@ -0,0 +1,180 @@ +//! Reading an environment's artifacts back out of its object store. +//! +//! The store lives on the rack's internal network. A developer's machine has +//! no route to it — the artifact instance's external address is often only +//! reachable over a VPN, and needing one to collect the output of a build +//! would defeat the point of building remotely. So this service, which is on +//! both networks, does the reading and passes the bytes through. + +use vw_api_types_versions::latest::{Artifact, S3Credentials, TargetKind}; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ArtifactError { + #[error("the environment has no object store yet")] + NoStore, + #[error("talking to the object store")] + Store(#[source] s3::error::S3Error), + #[error("the object store answered {0}")] + Refused(u16), + #[error("no artifact called '{0}'")] + NoSuchArtifact(String), +} + +/// A handle on one of an environment's buckets. +pub(crate) fn bucket( + credentials: &S3Credentials, +) -> Result, ArtifactError> { + let region = s3::Region::Custom { + region: credentials.region.clone(), + endpoint: credentials.endpoint.clone(), + }; + let creds = s3::creds::Credentials::new( + Some(&credentials.access_key_id), + Some(&credentials.secret_access_key), + None, + None, + None, + ) + .map_err(|_| ArtifactError::NoStore)?; + + // Path style: the bucket is reached by address, and there is no DNS inside + // the VPC that would resolve a bucket-as-subdomain name. + Ok(s3::Bucket::new(&credentials.bucket, region, creds) + .map_err(ArtifactError::Store)? + .with_path_style()) +} + +/// Everything in one bucket. +pub(crate) async fn list( + credentials: &S3Credentials, + kind: TargetKind, +) -> Result, ArtifactError> { + let bucket = bucket(credentials)?; + let pages = bucket + .list(String::new(), None) + .await + .map_err(ArtifactError::Store)?; + + Ok(pages + .into_iter() + .flat_map(|page| page.contents) + .map(|object| Artifact { + kind, + name: object.key, + size: object.size, + modified: Some(object.last_modified), + }) + .collect()) +} + +/// Remove everything in one bucket. +/// +/// Reported rather than silent about failures: an object that would not delete +/// is one the developer thinks is gone and is not, which is worse than an +/// error. +pub(crate) async fn clear( + credentials: &S3Credentials, +) -> Result<(usize, u64), ArtifactError> { + let bucket = bucket(credentials)?; + let pages = bucket + .list(String::new(), None) + .await + .map_err(ArtifactError::Store)?; + + let mut removed = 0usize; + let mut bytes = 0u64; + for object in pages.into_iter().flat_map(|page| page.contents) { + let response = bucket + .delete_object(format!("/{}", object.key)) + .await + .map_err(ArtifactError::Store)?; + if response.status_code() >= 300 { + return Err(ArtifactError::Refused(response.status_code())); + } + removed += 1; + bytes += object.size; + } + + Ok((removed, bytes)) +} + +/// One artifact's bytes, as a stream. +/// +/// Not read into memory first: an image runs to hundreds of megabytes, and +/// this service should not have to hold one to hand it on. +/// +/// The store's stream is pumped into a small bounded channel rather than +/// handed out directly. That is partly necessity — the response body has to be +/// shareable across threads and the store's stream is not — and partly the +/// better behaviour: a bounded channel means a developer on a slow connection +/// slows the read from the store rather than making this service buffer an +/// entire image on their behalf. +pub(crate) async fn fetch( + credentials: &S3Credentials, + artifact: &str, +) -> Result< + impl futures::Stream>, + ArtifactError, +> { + use futures::StreamExt; + + let bucket = bucket(credentials)?; + let response = bucket + .get_object_stream(format!("/{artifact}")) + .await + .map_err(|e| match e { + s3::error::S3Error::HttpFailWithBody(404, _) => { + ArtifactError::NoSuchArtifact(artifact.to_owned()) + } + other => ArtifactError::Store(other), + })?; + + if response.status_code >= 300 { + return Err(if response.status_code == 404 { + ArtifactError::NoSuchArtifact(artifact.to_owned()) + } else { + ArtifactError::Refused(response.status_code) + }); + } + + let (chunks, receive) = tokio::sync::mpsc::channel(4); + tokio::spawn(async move { + let mut bytes = response.bytes; + while let Some(chunk) = bytes.next().await { + // The store's own error type does not leave this module; what the + // body needs is an io error, and what a developer needs is the + // text. + let chunk = chunk.map_err(|e| std::io::Error::other(e.to_string())); + let failed = chunk.is_err(); + if chunks.send(chunk).await.is_err() { + // The developer stopped reading. So do we — there is no point + // pulling the rest of an image nobody is collecting. + break; + } + if failed { + break; + } + } + }); + + Ok(tokio_stream::wrappers::ReceiverStream::new(receive)) +} + +impl From for dropshot::HttpError { + fn from(value: ArtifactError) -> Self { + let message = value.to_string(); + match value { + ArtifactError::NoSuchArtifact(_) => { + dropshot::HttpError::for_not_found(None, message) + } + ArtifactError::NoStore => dropshot::HttpError::for_unavail( + None, + String::from( + "this environment has no object store yet; its artifact \ + instance may still be coming up", + ), + ), + _ => dropshot::HttpError::for_internal_error(message), + } + } +} diff --git a/vw-svc/src/auth.rs b/vw-svc/src/auth.rs new file mode 100644 index 0000000..82c3c9d --- /dev/null +++ b/vw-svc/src/auth.rs @@ -0,0 +1,398 @@ +//! This module holds common authorization functions. Authorization for vw-svc +//! is centered around Github access tokens. +use dropshot::RequestContext; +use reqwest::{ + header::{ACCEPT, AUTHORIZATION, USER_AGENT}, + StatusCode, +}; +use serde::Deserialize; +use slog::{error, info, warn, Logger}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +use crate::Context; + +/// Access to this repository is what grants access to the service. +const REDHAWK_REPO: &str = "oxidecomputer/redhawk"; + +/// Github requires a user agent on every API request. +const VW_USER_AGENT: &str = "vw-svc"; + +/// Header a caller can name themselves with when the service is running with +/// `--no-auth`. +/// +/// Deliberately not the authorization header: a real client sends a real +/// Github token there, and under `--no-auth` the caller's name is written to +/// the database as an environment key. Reading the name from its own header +/// keeps tokens out of persistent state on development servers. +const NO_AUTH_USER_HEADER: &str = "x-vw-user"; + +/// The caller identity used when the service is running with `--no-auth` and +/// the caller did not name themselves. +const ANONYMOUS_USER: &str = "anonymous"; + +/// Shared client so token checks reuse connections to the Github API. +static CLIENT: OnceLock = OnceLock::new(); + +/// How long a Github answer about a token is trusted for. +/// +/// Deciding whether a token is good costs two round trips to Github, and +/// without this every request pays them. That is barely noticeable for a +/// person clicking around and ruinous for a source sync, which sends one +/// request per file: a first sync of a few hundred files spent two minutes +/// waiting on Github and burned five hundred API calls doing it. +/// +/// A minute is short enough that revoking someone's access takes effect while +/// they are still reading the email about it, and long enough that a whole +/// sync costs one check. +const AUTH_CACHE_TTL: Duration = Duration::from_secs(60); + +/// What Github said about a token, and when it stops being worth believing. +struct CachedAuth { + name: String, + expires_at: Instant, +} + +/// Answers about tokens, keyed by a digest of the token rather than the token. +/// +/// Hashed because the raw value is a live credential and a map key is a poor +/// place to leave one lying: this way a dump of the process, or a stray +/// `Debug`, does not hand one over. +static AUTH_CACHE: OnceLock>> = + OnceLock::new(); + +fn auth_cache() -> &'static Mutex> { + AUTH_CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn cache_key(token: &str) -> [u8; 32] { + *blake3::hash(token.as_bytes()).as_bytes() +} + +/// The name Github last gave for this token, if that was recently enough. +fn cached_name(token: &str) -> Option { + let mut cache = auth_cache().lock().expect("the auth cache lock"); + let key = cache_key(token); + + match cache.get(&key) { + Some(entry) if entry.expires_at > Instant::now() => { + Some(entry.name.clone()) + } + Some(_) => { + cache.remove(&key); + None + } + None => None, + } +} + +fn remember(token: &str, name: &str) { + let mut cache = auth_cache().lock().expect("the auth cache lock"); + + // Expired entries are only noticed when their own token comes back, so a + // token used once and never again would otherwise sit here forever. This + // is the only place the map grows, so it is the right place to sweep. + let now = Instant::now(); + cache.retain(|_, entry| entry.expires_at > now); + + cache.insert( + cache_key(token), + CachedAuth { + name: name.to_owned(), + expires_at: now + AUTH_CACHE_TTL, + }, + ); +} + +pub(crate) struct AuthorizedCaller { + pub(crate) name: String, + pub(crate) is_admin: bool, + /// The token the caller authorized themselves with. + /// + /// Kept because an instance needs credentials of its own to fetch a + /// build's dependencies, and this is already the caller's answer to + /// "prove you may reach these repositories". Nothing stores it — it is + /// relayed to the instance and dropped when the request ends. + /// + /// Absent when the service runs with `--no-auth`, where there was no + /// token to begin with. + pub(crate) token: Option, +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum AuthError { + #[error("no token is present")] + NoAuthToken, + #[error("github rejected the supplied token")] + TokenRejected, + #[error("the supplied token does not have access to redhawk")] + NoRedhawkProjectAccess, + #[error( + "'{0}' is not an administrator of this service; ask whoever runs it to add you to --admin-users" + )] + NotAnAdministrator(String), + #[error("an error occured talking to github: {0}")] + GithubError(String), +} + +/// The subset of the Github `/user` response we care about. +#[derive(Deserialize)] +struct GithubUser { + login: String, +} + +pub(crate) async fn authorize_caller( + rqctx: RequestContext>, +) -> Result { + let args = &rqctx.context().server_args; + + let supplied = bearer_token(&rqctx); + + let name = if args.no_auth { + // Authorization is off, so there is nothing to ask Github about. Take + // the caller at their word about who they are so per-user endpoints + // are still exercisable, falling back to a fixed identity when the + // caller says nothing at all. + header(&rqctx, NO_AUTH_USER_HEADER) + .unwrap_or_else(|| ANONYMOUS_USER.to_owned()) + } else { + let token = supplied.clone().ok_or(AuthError::NoAuthToken)?; + + match cached_name(&token) { + Some(name) => name, + None => { + let client = client(); + let name = github_username(client, &token, &rqctx.log).await?; + check_redhawk_access(&name, client, &token, &rqctx.log).await?; + remember(&token, &name); + name + } + } + }; + + // Github usernames are case insensitive, so the admin list is too. + let is_admin = args + .admin_users + .iter() + .any(|admin| admin.eq_ignore_ascii_case(&name)); + + info!(rqctx.log, "authorized caller"; + "username" => &name, + "is_admin" => is_admin, + "req_id" => rqctx.request_id, + ); + + Ok(AuthorizedCaller { + name, + is_admin, + token: supplied, + }) +} + +/// Authorize a caller and require that they administer this service. +/// +/// Everything the admin API exposes reaches across users — listing every +/// environment on the rack, deleting somebody else's — so being a legitimate +/// user is not enough. Administrators are named in `--admin-users` when the +/// service starts; there is no way to grant it at runtime, deliberately, since +/// the alternative is an endpoint that can promote its own caller. +pub(crate) async fn authorize_administrator( + rqctx: RequestContext>, +) -> Result { + let log = rqctx.log.clone(); + let caller = authorize_caller(rqctx).await?; + + if !caller.is_admin { + warn!(log, "refusing an administrative request"; + "username" => &caller.name, + ); + return Err(AuthError::NotAnAdministrator(caller.name)); + } + + Ok(caller) +} + +fn client() -> &'static reqwest::Client { + CLIENT.get_or_init(reqwest::Client::new) +} + +/// The value of `name`, if the request carries it as a non-empty header. +fn header(rqctx: &RequestContext>, name: &str) -> Option { + let value = rqctx.request.headers().get(name)?.to_str().ok()?.trim(); + (!value.is_empty()).then(|| value.to_owned()) +} + +/// Pull the Github token out of the request's authorization header. +/// +/// Both the `Bearer ` and `token ` forms Github accepts are +/// understood, as is a bare token with no scheme. +fn bearer_token(rqctx: &RequestContext>) -> Option { + let value = header(rqctx, "authorization")?; + let token = match value.split_once(' ') { + Some((scheme, token)) + if scheme.eq_ignore_ascii_case("bearer") + || scheme.eq_ignore_ascii_case("token") => + { + token + } + _ => &value, + } + .trim(); + + (!token.is_empty()).then(|| token.to_owned()) +} + +/// Verify the token can see the redhawk repository. +async fn check_redhawk_access( + username: &str, + client: &reqwest::Client, + token: &str, + log: &Logger, +) -> Result<(), AuthError> { + let url = format!("https://api.github.com/repos/{REDHAWK_REPO}"); + let response = github_get(client, &url, token).await?; + + let status = response.status(); + if status.is_success() { + return Ok(()); + } + match status { + StatusCode::UNAUTHORIZED => { + info!(log, "github token rejected"; "username" => &username); + Err(AuthError::TokenRejected) + } + // Github reports repositories a token cannot see as absent rather than + // forbidden, so a 404 here means the same thing as a 403. + StatusCode::FORBIDDEN | StatusCode::NOT_FOUND => { + info!(log, "caller is not part of redhawk project"; + "username" => &username + ); + Err(AuthError::NoRedhawkProjectAccess) + } + other => { + let e = AuthError::GithubError(format!( + "unexpected response {other} from {url}" + )); + error!(log, "github error checking redhawk access: {e}"); + Err(e) + } + } +} + +/// Look up the Github username the token belongs to. +async fn github_username( + client: &reqwest::Client, + token: &str, + log: &Logger, +) -> Result { + let url = "https://api.github.com/user"; + let response = github_get(client, url, token).await?; + + let status = response.status(); + if !status.is_success() { + return Err(match status { + StatusCode::UNAUTHORIZED => { + info!(log, "github token rejected"); + AuthError::TokenRejected + } + other => { + let e = AuthError::GithubError(format!( + "unexpected response {other} from {url}" + )); + error!(log, "github error getting username: {e}"); + e + } + }); + } + + let user: GithubUser = response.json().await.map_err(|e| { + AuthError::GithubError(format!("decoding response from {url}: {e}")) + })?; + + // Github logins are case insensitive but reported in their original case. + // Downcasing here keeps one person from owning two sets of environments, + // and is required anyway to build an Oxide instance name out of it. + Ok(user.login.to_lowercase()) +} + +async fn github_get( + client: &reqwest::Client, + url: &str, + token: &str, +) -> Result { + client + .get(url) + .header(AUTHORIZATION, format!("Bearer {token}")) + .header(USER_AGENT, VW_USER_AGENT) + .header(ACCEPT, "application/vnd.github+json") + .send() + .await + .map_err(|e| AuthError::GithubError(format!("GET {url}: {e}"))) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn a_token_is_only_checked_with_github_once_per_ttl() { + let token = "ghp_a_token_used_for_a_whole_sync"; + assert!(cached_name(token).is_none(), "nothing is known yet"); + + remember(token, "rcgoodfellow"); + assert_eq!(cached_name(token).as_deref(), Some("rcgoodfellow")); + // Repeated lookups keep hitting: this is the whole point, since a + // sync asks once per file. + assert_eq!(cached_name(token).as_deref(), Some("rcgoodfellow")); + } + + #[test] + fn one_tokens_answer_is_not_anothers() { + remember("ghp_ferris", "ferris"); + remember("ghp_gorris", "gorris"); + + assert_eq!(cached_name("ghp_ferris").as_deref(), Some("ferris")); + assert_eq!(cached_name("ghp_gorris").as_deref(), Some("gorris")); + assert!(cached_name("ghp_never_seen").is_none()); + } + + #[test] + fn an_answer_stops_being_believed_once_it_is_old() { + let token = "ghp_a_token_since_revoked"; + + // Reach past `remember` to place an entry that has already expired, + // which is the state a revoked token's entry reaches on its own after + // a minute. + auth_cache().lock().expect("lock").insert( + cache_key(token), + CachedAuth { + name: "rcgoodfellow".to_owned(), + expires_at: Instant::now() - Duration::from_secs(1), + }, + ); + + assert!( + cached_name(token).is_none(), + "a stale answer must send the next request back to github", + ); + } + + #[test] + fn the_cache_is_not_keyed_by_the_credential_itself() { + // The key reaches a map that outlives the request. Keying it by the + // raw token would leave live credentials sitting in process memory in + // a form anything walking the map could read straight off. + let token = "ghp_a_real_looking_token"; + remember(token, "ferris"); + + let cache = auth_cache().lock().expect("lock"); + assert!( + !cache.keys().any(|key| key.as_slice() == token.as_bytes()), + "the token itself should not be a key", + ); + assert!(cache.contains_key(&cache_key(token))); + } +} diff --git a/vw-svc/src/db.rs b/vw-svc/src/db.rs new file mode 100644 index 0000000..dcceb26 --- /dev/null +++ b/vw-svc/src/db.rs @@ -0,0 +1,379 @@ +//! This module implements the vw service database +//! +//! The only state the vw service itself keeps track of is that of instances +//! within an environment. +//! +//! Environments live in a single redb table keyed by `"{user}/{name}"` with +//! the JSON encoding of an [`Environment`] as the value. Keying this way keeps +//! all of a user's environments contiguous in the table so listing them is a +//! prefix scan. + +use std::path::Path; +use std::sync::OnceLock; + +use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; +use vw_api_types_versions::latest::{ + Environment, EnvironmentImages, SshKeyPair, UserEnvironment, + UserEnvironmentPathParam, +}; + +use crate::reconciler::{InstanceKind, InstanceMap, UserInstance}; + +/// Environments keyed by `"{user}/{name}"`, valued by a JSON encoded +/// [`Environment`]. +const ENVIRONMENTS: TableDefinition<&str, &str> = + TableDefinition::new("environments"); + +/// Ssh keypairs keyed by `"{user}/{name}"`, valued by a JSON encoded +/// [`SshKeyPair`]. +/// +/// A separate table from the environments so a private key can never be +/// carried out by an endpoint that returns an [`Environment`]. +const SSH_KEYS: TableDefinition<&str, &str> = TableDefinition::new("ssh_keys"); + +/// The process-wide database handle, established by [`init`] before either API +/// server starts. +static DB: OnceLock = OnceLock::new(); + +/// Error conditions for opening the database. +#[derive(thiserror::Error, Debug)] +pub(crate) enum InitError { + #[error("opening database: {0}")] + Open(#[from] redb::DatabaseError), + #[error("beginning transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("committing transaction: {0}")] + Commit(#[from] redb::CommitError), + #[error("the database has already been initialized")] + AlreadyInitialized, +} + +/// Error conditions for listing user enviornments. +#[derive(thiserror::Error, Debug)] +pub(crate) enum ListError { + #[error("beginning read transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("reading environments table: {0}")] + Storage(#[from] redb::StorageError), + #[error("decoding environment '{key}': {source}")] + Decode { + key: String, + source: serde_json::Error, + }, +} + +/// Error conditions for creating an environment db entry. +#[derive(thiserror::Error, Debug)] +pub(crate) enum CreateError { + #[error("environment already exists")] + EnvironmentAlreadyExists, + #[error("encoding environment: {0}")] + Encode(#[from] serde_json::Error), + #[error("beginning write transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("writing environments table: {0}")] + Storage(#[from] redb::StorageError), + #[error("committing transaction: {0}")] + Commit(#[from] redb::CommitError), +} + +/// Error conditions for deleting an environment db entry. +#[derive(thiserror::Error, Debug)] +pub(crate) enum DeleteError { + #[error("environment does not exist")] + NoSuchEnvironment, + #[error("beginning write transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("writing environments table: {0}")] + Storage(#[from] redb::StorageError), + #[error("committing transaction: {0}")] + Commit(#[from] redb::CommitError), +} + +/// Error conditinos for retreiving environment status. +#[derive(thiserror::Error, Debug)] +pub(crate) enum GetError { + #[error("environment does not exist")] + NoSuchEnvironment, + #[error("beginning read transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("reading environments table: {0}")] + Storage(#[from] redb::StorageError), + #[error("decoding environment: {0}")] + Decode(#[from] serde_json::Error), +} + +/// Error conditinos for retreiving environment status. +#[derive(thiserror::Error, Debug)] +pub(crate) enum UpdateError { + #[error("environment does not exist")] + NoSuchEnvironment, + #[error("encoding environment: {0}")] + Encode(#[from] serde_json::Error), + #[error("beginning write transaction: {0}")] + Transaction(#[from] redb::TransactionError), + #[error("opening environments table: {0}")] + Table(#[from] redb::TableError), + #[error("writing environments table: {0}")] + Storage(#[from] redb::StorageError), + #[error("committing transaction: {0}")] + Commit(#[from] redb::CommitError), +} + +/// Open the database at `path`, creating it if it does not exist. +/// +/// This must be called once, before any of the accessors below are used. The +/// environments table is created here so that readers never have to contend +/// with a missing table on a fresh database. +pub(crate) fn init(path: impl AsRef) -> Result<(), InitError> { + let db = Database::create(path)?; + let tx = db.begin_write()?; + tx.open_table(ENVIRONMENTS)?; + tx.open_table(SSH_KEYS)?; + tx.commit()?; + DB.set(db).map_err(|_| InitError::AlreadyInitialized) +} + +/// The database handle established by [`init`]. +/// +/// Panics if [`init`] has not run. That is a service startup bug, not +/// something a request can provoke. +fn db() -> &'static Database { + DB.get().expect("database has not been initialized") +} + +/// The database key for an environment named `name` owned by `user`. +fn key(user: &str, name: &str) -> String { + format!("{user}/{name}") +} + +/// Every environment in the db, with the owning user. +pub(crate) fn list_all_environments() -> Result, ListError> +{ + let tx = db().begin_read()?; + let table = tx.open_table(ENVIRONMENTS)?; + + let mut environments = Vec::new(); + for entry in table.iter()? { + let (key, value) = entry?; + let key = key.value(); + // Keys are "{user}/{name}"; anything else did not come from here. + let Some((user, _)) = key.split_once('/') else { + continue; + }; + environments.push(UserEnvironment { + user: user.to_owned(), + environment: serde_json::from_str(value.value()).map_err( + |source| ListError::Decode { + key: key.to_owned(), + source, + }, + )?, + }); + } + + Ok(environments) +} + +/// Every environment decomposed into the individual instances that make it up. +/// +/// This is the reconciler's target state: an environment always wants all +/// three of its instances, whether or not any of them exist yet. +pub(crate) fn list_all_environment_instances() -> Result +{ + let mut instances = InstanceMap::new(); + + let tx = db().begin_read()?; + let keys = tx.open_table(SSH_KEYS)?; + + for UserEnvironment { user, environment } in list_all_environments()? { + // The public half is what gets attached to the instances. A missing + // entry is not fatal here; the instance simply comes up without a key + // and the reconciler says so. + let db_key = key(&user, &environment.name); + let public_key = match keys.get(db_key.as_str())? { + Some(value) => serde_json::from_str::(value.value()) + .map(|pair| pair.public_key) + .map_err(|source| ListError::Decode { + key: db_key.clone(), + source, + }) + .map(Some)?, + None => None, + }; + + for kind in InstanceKind::ALL { + let (recorded, image) = match kind { + InstanceKind::Vivado => ( + &environment.vivado_instance, + environment.images.as_ref().map(|i| &i.vivado), + ), + InstanceKind::Helios => ( + &environment.helios_instance, + environment.images.as_ref().map(|i| &i.helios), + ), + InstanceKind::Artifact => ( + &environment.artifact_instance, + environment.images.as_ref().map(|i| &i.artifact), + ), + }; + + // Instance names are unique per user/environment/kind, so this + // cannot collide with anything already inserted. + instances.insert_overwrite(UserInstance { + user: user.clone(), + environment: environment.name.clone(), + kind, + image: image.cloned(), + public_key: public_key.clone(), + oxide_instance: recorded.clone(), + }); + } + } + + Ok(instances) +} + +pub(crate) fn list_user_environments( + user: impl AsRef, +) -> Result, ListError> { + // Every one of this user's environments is keyed by this prefix, and redb + // iterates in key order, so the user's entries are the contiguous run + // starting at the first key greater than or equal to the prefix. + let prefix = format!("{}/", user.as_ref()); + + let tx = db().begin_read()?; + let table = tx.open_table(ENVIRONMENTS)?; + + let mut environments = Vec::new(); + for entry in table.range(prefix.as_str()..)? { + let (key, value) = entry?; + let key = key.value(); + if !key.starts_with(&prefix) { + break; + } + environments.push(serde_json::from_str(value.value()).map_err( + |source| ListError::Decode { + key: key.to_owned(), + source, + }, + )?); + } + + Ok(environments) +} + +pub(crate) fn create_environment( + env: UserEnvironmentPathParam, + images: Option, + ssh_key: &SshKeyPair, +) -> Result<(), CreateError> { + let key = key(&env.user, &env.name); + let value = serde_json::to_string(&Environment { + name: env.name, + images, + vivado_instance: None, + helios_instance: None, + artifact_instance: None, + })?; + let ssh_key = serde_json::to_string(ssh_key)?; + + // Both in one transaction: an environment without its keypair would be + // unreachable, and a keypair without its environment would never be + // cleaned up. + let tx = db().begin_write()?; + { + let mut environments = tx.open_table(ENVIRONMENTS)?; + if environments.get(key.as_str())?.is_some() { + // Dropping the transaction without committing aborts it. + return Err(CreateError::EnvironmentAlreadyExists); + } + environments.insert(key.as_str(), value.as_str())?; + + let mut keys = tx.open_table(SSH_KEYS)?; + keys.insert(key.as_str(), ssh_key.as_str())?; + } + tx.commit()?; + + Ok(()) +} + +/// The keypair that opens an environment's instances. +pub(crate) fn get_environment_keys( + env: UserEnvironmentPathParam, +) -> Result { + let key = key(&env.user, &env.name); + + let tx = db().begin_read()?; + let table = tx.open_table(SSH_KEYS)?; + let value = table + .get(key.as_str())? + .ok_or(GetError::NoSuchEnvironment)?; + + Ok(serde_json::from_str(value.value())?) +} + +pub(crate) fn delete_environment( + env: UserEnvironmentPathParam, +) -> Result<(), DeleteError> { + let key = key(&env.user, &env.name); + + let tx = db().begin_write()?; + { + let mut environments = tx.open_table(ENVIRONMENTS)?; + if environments.remove(key.as_str())?.is_none() { + return Err(DeleteError::NoSuchEnvironment); + } + // The keypair opens nothing once the environment is gone. + tx.open_table(SSH_KEYS)?.remove(key.as_str())?; + } + tx.commit()?; + + Ok(()) +} + +pub(crate) fn get_environment_status( + env: UserEnvironmentPathParam, +) -> Result { + let key = key(&env.user, &env.name); + + let tx = db().begin_read()?; + let table = tx.open_table(ENVIRONMENTS)?; + let value = table + .get(key.as_str())? + .ok_or(GetError::NoSuchEnvironment)?; + + Ok(serde_json::from_str(value.value())?) +} + +pub(crate) fn update_environment_status( + key: UserEnvironmentPathParam, + env: Environment, +) -> Result<(), UpdateError> { + let db_key = self::key(&key.user, &key.name); + let value = serde_json::to_string(&env)?; + + let tx = db().begin_write()?; + { + let mut table = tx.open_table(ENVIRONMENTS)?; + // Only an update: creating an environment goes through + // `create_environment` so that its images get resolved. + if table.get(db_key.as_str())?.is_none() { + return Err(UpdateError::NoSuchEnvironment); + } + table.insert(db_key.as_str(), value.as_str())?; + } + tx.commit()?; + + Ok(()) +} diff --git a/vw-svc/src/error.rs b/vw-svc/src/error.rs new file mode 100644 index 0000000..c25071b --- /dev/null +++ b/vw-svc/src/error.rs @@ -0,0 +1,171 @@ +//! Translations from the service's internal error types into the HTTP errors +//! the API surfaces. +//! +//! Database failures are all internal errors: the message goes into the +//! internal log rather than out to the caller, since it says more about the +//! service's storage than about the request. + +use dropshot::ClientErrorStatusCode; + +use crate::{auth, db, keys, oxide, relay}; + +impl From for dropshot::HttpError { + fn from(value: auth::AuthError) -> Self { + let message = value.to_string(); + match value { + // The caller has not established who they are. + auth::AuthError::NoAuthToken | auth::AuthError::TokenRejected => { + dropshot::HttpError::for_client_error( + None, + ClientErrorStatusCode::UNAUTHORIZED, + message, + ) + } + // The caller is known, but not entitled to this. Same status as + // lacking project access: they are who they say, and it is not + // enough. + auth::AuthError::NoRedhawkProjectAccess + | auth::AuthError::NotAnAdministrator(_) => { + dropshot::HttpError::for_client_error( + None, + ClientErrorStatusCode::FORBIDDEN, + message, + ) + } + // We could not reach a verdict because Github did not cooperate. + auth::AuthError::GithubError(_) => dropshot::HttpError { + status_code: dropshot::ErrorStatusCode::BAD_GATEWAY, + error_code: None, + external_message: String::from( + "unable to verify credentials with github", + ), + internal_message: message, + headers: None, + }, + } + } +} + +impl From for dropshot::HttpError { + fn from(value: db::ListError) -> Self { + dropshot::HttpError::for_internal_error(value.to_string()) + } +} + +impl From for dropshot::HttpError { + fn from(value: oxide::ImageError) -> Self { + let message = value.to_string(); + match value { + // The caller named an image that is not there, or asked for a kind + // the rack has no image for. Either way they can act on it. + oxide::ImageError::NoSuchImage(_) + | oxide::ImageError::NoMatchingImage(_) => { + dropshot::HttpError::for_bad_request(None, message) + } + // Something about this service or the rack behind it, not the + // request. + oxide::ImageError::List(_) => { + dropshot::HttpError::for_internal_error(message) + } + } + } +} + +impl From for dropshot::HttpError { + fn from(value: oxide::SessionError) -> Self { + // Either way the caller did nothing wrong: the service is missing its + // Oxide configuration, or cannot build a client from it. + dropshot::HttpError::for_internal_error(value.to_string()) + } +} + +impl From for dropshot::HttpError { + fn from(value: keys::KeyError) -> Self { + // Nothing the caller did; the service could not make a key. + dropshot::HttpError::for_internal_error(value.to_string()) + } +} + +impl From for dropshot::HttpError { + fn from(value: relay::RelayError) -> Self { + let message = value.to_string(); + match value { + // The caller named something that is not theirs or not there. + relay::RelayError::NoSuchEnvironment => { + dropshot::HttpError::for_not_found(None, message) + } + // The environment is real but not ready. Not an error on anyone's + // part — an environment spends its first minute like this — so it + // is worth a status a client can wait on rather than give up at. + relay::RelayError::NoInstance { .. } + | relay::RelayError::NoAddress { .. } => { + dropshot::HttpError::for_unavail(None, message) + } + // The instance said no, or could not be reached. Either way the + // caller's request was fine and the detail is in the log. + relay::RelayError::Agent { .. } => dropshot::HttpError { + status_code: dropshot::ErrorStatusCode::BAD_GATEWAY, + error_code: None, + external_message: message.clone(), + internal_message: message, + headers: None, + }, + relay::RelayError::Db(_) | relay::RelayError::Client(_) => { + dropshot::HttpError::for_internal_error(message) + } + } + } +} + +impl From for dropshot::HttpError { + fn from(value: db::CreateError) -> Self { + match value { + db::CreateError::EnvironmentAlreadyExists => { + dropshot::HttpError::for_client_error( + None, + ClientErrorStatusCode::CONFLICT, + value.to_string(), + ) + } + db::CreateError::Encode(_) + | db::CreateError::Transaction(_) + | db::CreateError::Table(_) + | db::CreateError::Storage(_) + | db::CreateError::Commit(_) => { + dropshot::HttpError::for_internal_error(value.to_string()) + } + } + } +} + +impl From for dropshot::HttpError { + fn from(value: db::DeleteError) -> Self { + match value { + db::DeleteError::NoSuchEnvironment => { + dropshot::HttpError::for_not_found(None, value.to_string()) + } + db::DeleteError::Transaction(_) + | db::DeleteError::Table(_) + | db::DeleteError::Storage(_) + | db::DeleteError::Commit(_) => { + dropshot::HttpError::for_internal_error(value.to_string()) + } + } + } +} + +impl From for dropshot::HttpError { + fn from(value: db::GetError) -> Self { + match value { + db::GetError::NoSuchEnvironment => { + dropshot::HttpError::for_not_found(None, value.to_string()) + } + db::GetError::Transaction(_) + | db::GetError::Table(_) + | db::GetError::Storage(_) + | db::GetError::Decode(_) => { + dropshot::HttpError::for_internal_error(value.to_string()) + } + } + } +} diff --git a/vw-svc/src/keys.rs b/vw-svc/src/keys.rs new file mode 100644 index 0000000..d38bac0 --- /dev/null +++ b/vw-svc/src/keys.rs @@ -0,0 +1,92 @@ +//! SSH keys for reaching the instances in an environment. +//! +//! An environment gets its own keypair when it is created. The private half +//! never leaves this service except through the key endpoint, which only the +//! environment's owner can reach; the public half is registered with the Oxide +//! silo and attached to every instance the environment is made of, so the +//! instances come up reachable rather than needing a key added by hand +//! afterwards. + +use ssh_key::{rand_core::OsRng, Algorithm, LineEnding, PrivateKey}; +use vw_api_types_versions::latest::SshKeyPair; + +/// What the key says it is for, so it is recognizable in `ssh-add -l` output +/// and in the Oxide silo's key list. +fn comment(user: &str, environment: &str) -> String { + format!("vw {user}/{environment}") +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum KeyError { + #[error("generating an ssh key failed")] + Generate(#[source] ssh_key::Error), + #[error("encoding an ssh key failed")] + Encode(#[source] ssh_key::Error), +} + +/// Generate a keypair for the environment `environment` owned by `user`. +/// +/// Ed25519 rather than RSA: the keys are short, every ssh client in use +/// understands them, and there is no key size to get wrong. +pub(crate) fn generate( + user: &str, + environment: &str, +) -> Result { + let mut key = PrivateKey::random(&mut OsRng, Algorithm::Ed25519) + .map_err(KeyError::Generate)?; + key.set_comment(comment(user, environment)); + + Ok(SshKeyPair { + // OpenSSH format, so it can be handed straight to `ssh -i` without + // conversion. + private_key: key + .to_openssh(LineEnding::LF) + .map_err(KeyError::Encode)? + .to_string(), + public_key: key.public_key().to_openssh().map_err(KeyError::Encode)?, + }) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn a_generated_key_is_usable_by_ssh() { + let pair = generate("ferris", "alpha").expect("generates"); + + // `ssh -i` and the Oxide silo both want OpenSSH encoding, not PEM. + assert!(pair + .private_key + .starts_with("-----BEGIN OPENSSH PRIVATE KEY-----")); + assert!(pair.private_key.ends_with('\n')); + assert!(pair.public_key.starts_with("ssh-ed25519 ")); + + // Round trip through the parser both halves will meet in the wild. + let parsed = + PrivateKey::from_openssh(&pair.private_key).expect("parses back"); + assert_eq!(parsed.algorithm(), Algorithm::Ed25519); + assert_eq!( + parsed.public_key().to_openssh().expect("encodes"), + pair.public_key, + ); + } + + #[test] + fn the_comment_says_which_environment_it_opens() { + let pair = generate("ferris", "alpha").expect("generates"); + assert!( + pair.public_key.ends_with(" vw ferris/alpha"), + "got {:?}", + pair.public_key, + ); + } + + #[test] + fn every_environment_gets_its_own_key() { + let one = generate("ferris", "alpha").expect("generates"); + let two = generate("ferris", "alpha").expect("generates"); + assert_ne!(one.private_key, two.private_key); + assert_ne!(one.public_key, two.public_key); + } +} diff --git a/vw-svc/src/main.rs b/vw-svc/src/main.rs new file mode 100644 index 0000000..0f5ef8e --- /dev/null +++ b/vw-svc/src/main.rs @@ -0,0 +1,269 @@ +use camino::Utf8PathBuf; +use clap::{Parser, Subcommand}; +use slog::{error, info, warn, Drain, Logger}; +use std::{ + io::stdout, + net::{IpAddr, Ipv6Addr, SocketAddr}, + sync::Arc, + time::Duration, +}; +use tokio::sync::Notify; + +mod admin_api; +mod artifacts; +mod auth; +mod db; +mod error; +mod keys; +mod oxide; +mod reconciler; +mod relay; +mod tls; +mod user_api; +mod wiring; + +pub struct Context { + server_args: ServerArgs, + /// Rung when an environment is created or deleted so the reconciler acts + /// on it right away instead of waiting out its interval. + reconcile: Arc, +} + +#[derive(Parser)] +#[command(name = "vw-svc")] +#[command(about = "vw service")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Run the server + /// + /// Boxed because it carries every one of the service's settings while the + /// other two variants carry nothing, and an enum is as large as its + /// largest variant — so emitting a spec would otherwise move a few hundred + /// bytes of server configuration around for no reason. + Serve(Box), + /// Emit the user OpenAPI spec + EmitUserSpec, + /// Emit the admin OpenAPI spec + EmitAdminSpec, +} + +#[derive(Parser, Clone)] +struct ServerArgs { + /// Server bind address + #[arg(long, default_value_t = IpAddr::V6(Ipv6Addr::UNSPECIFIED))] + address: IpAddr, + #[arg(long, default_value_t = 2727u16)] + user_api_port: u16, + #[arg(long, default_value_t = 2728u16)] + admin_api_port: u16, + /// Enable TLS + #[arg(long)] + tls: bool, + /// TLS certificate file path + #[arg(long, default_value = "cert.pem")] + cert_file: Utf8PathBuf, + /// TLS private key file path + #[arg(long, default_value = "key.pem")] + key_file: Utf8PathBuf, + /// Do not require a github token for API access + #[arg(long)] + no_auth: bool, + #[arg(long)] + admin_users: Vec, + /// Path to the environment database, created if it does not exist + #[arg(long, default_value = "vw-svc.redb")] + db_path: Utf8PathBuf, + + /// The Oxide API endpoint to use. + /// + /// Together with --oxide-token this is what connects the service to a + /// rack. With neither set the service keeps environment records but + /// provisions nothing. + #[arg(long, requires = "oxide_token")] + oxide_api_endpoint: Option, + + /// The token to use + #[arg(long, env = "OXIDE_TOKEN", requires = "oxide_api_endpoint")] + oxide_token: Option, + + /// The Oxide project instances are created in + #[arg(long, default_value = "redhawk")] + oxide_project: String, + + /// Seconds between reconciler passes + #[arg(long, default_value_t = 30)] + reconcile_interval: u64, + + /// Send vivado source to this address instead of to a rack instance. + /// + /// For running the whole stack on one machine: with no Oxide backend there + /// are no instances to look up, so there is nowhere for a sync to go. An + /// address here stands in for the instance the reconciler would otherwise + /// have recorded. + #[arg(long, value_name = "HOST:PORT")] + vivado_agent: Option, + + /// Send helios source to this address instead of to a rack instance. + #[arg(long, value_name = "HOST:PORT")] + helios_agent: Option, + + /// Ask this address for the environment's object store, rather than the + /// artifact instance the reconciler would have recorded. + #[arg(long, value_name = "HOST:PORT")] + artifact_agent: Option, +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + match cli.command { + Commands::Serve(args) => serve(*args).await, + Commands::EmitUserSpec => emit_user_spec(), + Commands::EmitAdminSpec => emit_admin_spec(), + }; +} + +async fn serve(args: ServerArgs) { + let log = logger(); + db::init(&args.db_path).expect("unable to open environment database"); + + if let Err(e) = oxide::init( + args.oxide_api_endpoint.as_deref(), + args.oxide_token.as_deref(), + &args.oxide_project, + ) { + error!(log, "oxide configuration error"; + slog_error_chain::InlineErrorChain::new(&e), + ); + std::process::exit(1); + } + if oxide::is_configured() { + // Not fatal either way: the rack may come back, and the reconciler + // opens its own session every pass. Say so loudly rather than looking + // healthy. + let endpoint = args.oxide_api_endpoint.as_deref().unwrap_or(""); + match oxide::session() { + Ok(session) => { + if let Err(e) = session.ping(&log).await { + warn!(log, "oxide api probe failed"; + "endpoint" => endpoint, + slog_error_chain::InlineErrorChain::new(&e), + ); + } + } + Err(e) => { + warn!(log, "cannot open a session with the oxide api"; + "endpoint" => endpoint, + slog_error_chain::InlineErrorChain::new(&e), + ); + } + } + } else { + warn!( + log, + "no oxide backend configured; environments will be recorded but \ + never provisioned. Pass --oxide-api-endpoint and --oxide-token \ + to reconcile instances." + ); + } + //let addr: IpAddr = args.address.parse().expect("unable to parse address"); + let user_sa = SocketAddr::new(args.address, args.user_api_port); + let admin_sa = SocketAddr::new(args.address, args.admin_api_port); + + // Resolve TLS before either server starts, so a bad certificate path is a + // startup failure with a message naming it rather than a half-up service. + // This also starts the watch that notices renewals. + let tls = match tls::config(&args, &log) { + Ok(tls) => tls, + Err(e) => { + error!(log, "tls configuration error"; + slog_error_chain::InlineErrorChain::new(&e), + ); + std::process::exit(1); + } + }; + if tls.is_some() { + info!(log, "serving tls"; + "cert_file" => %args.cert_file, + "key_file" => %args.key_file, + ); + } + + // Shared between the API servers and the reconciler: creating or deleting + // an environment rings it so a pass runs immediately. + let reconcile = Arc::new(Notify::new()); + + // The reconciler is only useful with a rack behind it. + if oxide::is_configured() { + let reconciler = reconciler::InstanceReconciler::new( + Duration::from_secs(args.reconcile_interval), + reconcile.clone(), + ); + let log = log.new(slog::o!("component" => "reconciler")); + info!(log, "starting reconciler"; + "interval_secs" => args.reconcile_interval, + ); + tokio::spawn(async move { reconciler.run(log).await }); + } + + // Environments outlive this service, so some of them may have been created + // while their instances were still coming up, or have had their object + // store rebuilt since. Put them right now rather than waiting for someone + // to synchronize source — an environment nobody has synced since the last + // restart would otherwise build images that went nowhere. + // + // In the background: an instance that is down should delay nothing, and + // the API can serve while this works through them. + { + let args = args.clone(); + let log = log.new(slog::o!("component" => "artifacts")); + tokio::spawn(async move { wiring::ensure_all(&args, &log).await }); + } + + // Both servers run for the life of the process, so they have to be driven + // concurrently. Awaiting them in sequence would leave the second one + // never started. If either stops, the whole service is done. + let user = user_api::start_server( + args.clone(), + log.clone(), + user_sa, + tls.clone(), + reconcile.clone(), + ); + let admin = + admin_api::start_server(args, log.clone(), admin_sa, tls, reconcile); + + tokio::try_join!( + async { user.await.map_err(|e| format!("user api stopped: {e}")) }, + async { admin.await.map_err(|e| format!("admin api stopped: {e}")) }, + ) + .expect("api server stopped"); +} + +fn emit_user_spec() { + let api = user_api::api_description(); + let spec = api.openapi("VW user API", vw_api::latest_version()); + let mut out = stdout(); + spec.write(&mut out).expect("write spec to stdout"); +} + +fn emit_admin_spec() { + let api = admin_api::api_description(); + let spec = api.openapi("VW admin API", vw_api::latest_version()); + let mut out = stdout(); + spec.write(&mut out).expect("write spec to stdout"); +} + +fn logger() -> Logger { + let drain = slog_bunyan::new(std::io::stdout()).build().fuse(); + let drain = slog_async::Async::new(drain) + .chan_size(0x8000) + .build() + .fuse(); + Logger::root(drain, slog::o!()) +} diff --git a/vw-svc/src/oxide.rs b/vw-svc/src/oxide.rs new file mode 100644 index 0000000..763d5c9 --- /dev/null +++ b/vw-svc/src/oxide.rs @@ -0,0 +1,1205 @@ +//! this module contains functionality for interacting with an Oxide Cloud +//! Computer. + +use std::sync::OnceLock; + +use futures::StreamExt; +use oxide::{ + types, ClientCurrentUserExt, ClientDisksExt, ClientImagesExt, + ClientInstancesExt, ClientSystemStatusExt, Error, +}; +use slog::{info, warn, Logger}; +use vw_api_types_versions::latest::{ + EnvironmentImages, ImageRef, OxideInstance, +}; + +use crate::reconciler::{InstanceKind, InstanceMap, UserInstance}; + +pub(crate) type OxideError = Error; + +/// How many items to ask for per page when listing. +/// +/// The Oxide API paginates and the client's `stream()` follows the page tokens +/// for us. Asking for everything in a single request instead — `limit` of +/// `u32::MAX` — leaves the control plane trying to satisfy a four-billion-item +/// query, which is a good way to have the request die at the transport layer +/// with nothing but "error sending request" to show for it. +const PAGE_SIZE: u32 = 100; + +/// How long to give a single Oxide API call, in seconds. +/// +/// The client's own default is 15 seconds, which creating an instance +/// comfortably exceeds — the control plane is still laying an image down on a +/// fresh disk. The request goes through regardless; it is only the client that +/// gives up, so a short timeout does not prevent the work, it just means the +/// reconciler never learns it succeeded and reports a failure for something +/// that worked. +const REQUEST_TIMEOUT_SECS: u64 = 300; + +/// How long to give the initial connection, in seconds. +/// +/// Set separately because it otherwise inherits [`REQUEST_TIMEOUT_SECS`], and +/// a rack that is simply not there should be reported in seconds rather than +/// minutes. +const CONNECT_TIMEOUT_SECS: u64 = 10; + +/// Boot disk of every instance the reconciler creates. +/// +/// One size for every kind, because it is the images rather than the work that +/// set the floor: the vivado image alone is 512 GiB and an instance's disk +/// cannot be smaller than the image laid down on it. Cpu and memory do vary — +/// see [`InstanceKind::shape`]. +const BOOT_DISK_GIB: u64 = 600; + +/// Instances are named `vwsvc-{user}-{env}-{kind}`, and this prefix is the +/// only thing that marks an Oxide instance as ours. +/// +/// Anything without it is somebody else's and is never touched, which is what +/// keeps a reconciler pass from deleting unrelated instances in the project. +pub(crate) const INSTANCE_PREFIX: &str = "vwsvc"; + +/// How to reach the Oxide API, recorded at startup. +/// +/// Deliberately just the credentials rather than a live client — see +/// [`session`]. +struct OxideConfig { + endpoint: String, + token: String, + project: String, +} + +/// `None` until [`init`] runs, and permanently `None` when the service was +/// started without Oxide credentials. +static OXIDE: OnceLock> = OnceLock::new(); + +#[derive(Debug, thiserror::Error)] +pub(crate) enum InitError { + #[error("the oxide configuration has already been initialized")] + AlreadyInitialized, +} + +/// Error conditions for opening a session against the Oxide API. +#[derive(Debug, thiserror::Error)] +pub(crate) enum SessionError { + #[error("this service has no oxide backend configured")] + NotConfigured, + #[error("building the oxide client failed")] + Client(#[source] oxide::OxideAuthError), +} + +/// Error conditions for choosing an image to boot an instance from. +#[derive(Debug, thiserror::Error)] +pub(crate) enum ImageError { + #[error("no image named '{0}' is visible to this service")] + NoSuchImage(String), + #[error( + "no image matching '{0}*' is visible to this service; \ + name one explicitly to use a different image" + )] + NoMatchingImage(String), + // No `{0}`: log sites report this through `InlineErrorChain`, which + // appends the source itself. See `PassError`. + // + // Boxed because the oxide client's error is large enough that carrying it + // inline makes every `Result` in this module the size of a failure that + // almost never happens. `RelayError` boxes its own for the same reason. + #[error("listing images failed")] + List(#[source] Box), +} + +impl From for ImageError { + fn from(value: OxideError) -> Self { + ImageError::List(Box::new(value)) + } +} + +/// Establish the connection to the Oxide API. +/// +/// `endpoint` and `token` are both `None` when the service is being run +/// without an Oxide backend, in which case every call in this module reports +/// that it is not configured and the reconciler does not run. +pub(crate) fn init( + endpoint: Option<&str>, + token: Option<&str>, + project: &str, +) -> Result<(), InitError> { + let config = match (endpoint, token) { + (Some(endpoint), Some(token)) => Some(OxideConfig { + endpoint: endpoint.to_owned(), + token: token.to_owned(), + project: project.to_owned(), + }), + _ => None, + }; + + OXIDE.set(config).map_err(|_| InitError::AlreadyInitialized) +} + +/// Open a session for one unit of work — a reconciler pass, or one API +/// request. +/// +/// Built fresh each time rather than kept alive for the life of the process. +/// The reconciler polls on an interval, so a long-lived client's connections +/// would spend nearly all their time idle and be dropped by the far end before +/// the next use. Picking one of those back up surfaces as: +/// +/// ```text +/// client error (SendRequest): connection error: peer closed connection +/// without sending TLS close_notify +/// ``` +/// +/// because rustls treats a close without `close_notify` as an error rather +/// than a clean EOF (curl and OpenSSL do not). At this call rate there is +/// nothing to gain from holding a connection open between passes, and a client +/// that outlives its work is just somewhere for that class of bug to live. +pub(crate) fn session() -> Result { + let config = config().ok_or(SessionError::NotConfigured)?; + + let client = oxide::Client::new_authenticated_config( + &oxide::ClientConfig::default() + .with_host_and_token(&config.endpoint, &config.token) + .with_timeout(REQUEST_TIMEOUT_SECS) + .with_connect_timeout(CONNECT_TIMEOUT_SECS), + ) + .map_err(SessionError::Client)?; + + Ok(Session { + client, + project: config.project.clone(), + }) +} + +/// A connection to the Oxide API, scoped to the project we manage. +pub(crate) struct Session { + client: oxide::Client, + project: String, +} + +/// Whether this service has an Oxide backend to reconcile against. +pub(crate) fn is_configured() -> bool { + config().is_some() +} + +fn config() -> Option<&'static OxideConfig> { + OXIDE + .get() + .expect("the oxide client has not been initialized") + .as_ref() +} + +impl Session { + /// Confirm the Oxide API is reachable and the configured credentials work. + pub(crate) async fn ping(&self, log: &Logger) -> Result<(), OxideError> { + self.client.ping().send().await?; + info!(log, "oxide api reachable"; "project" => &self.project); + Ok(()) + } + + /// Every instance in the project that this service manages. + pub(crate) async fn get_instances( + &self, + ) -> Result { + let instances = self + .client + .instance_list() + .project(self.project.as_str()) + .limit(4096) + .send() + .await? + .items + .clone(); + + let managed = ours(instances.iter().map(|instance| { + (instance.name.to_string(), instance.id, instance.run_state) + })); + + // The external address is what somebody needs in order to ssh in, and + // it is only available per instance rather than from the list. + let mut map = InstanceMap::new(); + for mut instance in managed { + let name = instance.oxide_instance_name(); + if instance.oxide_instance.is_some() { + let external = self.external_ip(&name).await?; + let internal = self.internal_ip(&name).await?; + if let Some(oxide) = instance.oxide_instance.as_mut() { + oxide.external_ip = external; + oxide.internal_ip = internal; + } + } + map.insert_overwrite(instance); + } + + Ok(map) + } + + /// The instance's address on the rack's own network. + /// + /// The primary interface's, since that is the one every instance has. A + /// dual-stack interface reports its v4 address: it is what the agent binds + /// and what a URL can name without bracketing. + async fn internal_ip( + &self, + name: &str, + ) -> Result, OxideError> { + let interfaces = self + .client + .instance_network_interface_list() + .instance(name) + .project(self.project.as_str()) + .limit(PAGE_SIZE) + .send() + .await? + .items + .clone(); + + let primary = interfaces + .iter() + .find(|interface| interface.primary) + .or_else(|| interfaces.first()); + + Ok(primary.map(|interface| match &interface.ip_stack { + types::PrivateIpStack::V4(v4) => std::net::IpAddr::V4(v4.ip), + types::PrivateIpStack::V6(v6) => std::net::IpAddr::V6(v6.ip), + types::PrivateIpStack::DualStack { v4, .. } => { + std::net::IpAddr::V4(v4.ip) + } + })) + } + + /// The address an instance can be reached on from outside the rack. + /// + /// SNAT addresses are skipped: they carry outbound traffic only, so one + /// would look like a way in without being one. + async fn external_ip( + &self, + name: &str, + ) -> Result, OxideError> { + let addresses = self + .client + .instance_external_ip_list() + .instance(name) + .project(self.project.as_str()) + .send() + .await? + .items + .clone(); + + Ok(addresses.iter().find_map(|address| match address { + types::ExternalIp::Ephemeral { ip, .. } + | types::ExternalIp::Floating { ip, .. } => Some(*ip), + types::ExternalIp::Snat { .. } => None, + })) + } + + /// Register every environment key the instances about to be created will + /// reference. + /// + /// Once per pass and one at a time, rather than from inside each create. + /// An environment's three instances share a single key, and creating them + /// concurrently had all three finding it absent and all three trying to + /// register it — the one that got there first won, and the other two + /// failed on a name that now existed, taking their instance creates down + /// with them. + pub(crate) async fn ensure_ssh_keys( + &self, + instances: &InstanceMap, + log: &Logger, + ) -> Result<(), OxideError> { + // Collapsed by key name, so an environment is considered once however + // many of its instances are being created. + let mut wanted = std::collections::BTreeMap::new(); + for instance in instances.iter() { + let Some(public_key) = instance.public_key.as_deref() else { + warn!(log, "environment has no ssh key recorded"; + "instance" => instance.oxide_instance_name(), + ); + continue; + }; + wanted.entry(instance.ssh_key_name()).or_insert(( + instance.user.clone(), + instance.environment.clone(), + public_key.to_owned(), + )); + } + if wanted.is_empty() { + return Ok(()); + } + + let existing = self.ssh_key_names().await?; + + for (name, (user, environment, public_key)) in wanted { + if existing.contains(&name) { + continue; + } + + info!(log, "registering environment ssh key"; "key" => &name); + let created = self + .client + .current_user_ssh_key_create() + .body(types::SshKeyCreate { + name: name.parse().map_err(bad_name)?, + description: format!("vw environment {user}/{environment}"), + public_key, + }) + .send() + .await; + + if let Err(e) = created { + // Another writer got there between the list and the create. + // The key being present is the outcome we wanted, so this is + // not a failure. + if already_exists(&e) { + info!(log, "environment ssh key was already registered"; + "key" => &name, + ); + continue; + } + return Err(e); + } + } + + Ok(()) + } + + /// The names of every ssh key on the silo user this service acts as. + async fn ssh_key_names( + &self, + ) -> Result, OxideError> { + let mut names = std::collections::BTreeSet::new(); + let mut keys = self + .client + .current_user_ssh_key_list() + .limit(PAGE_SIZE) + .stream(); + while let Some(key) = keys.next().await { + names.insert(key?.name.to_string()); + } + Ok(names) + } + + /// Delete silo ssh keys belonging to environments that are gone. + /// + /// Same reasoning as the disks: nothing else cleans these up, and a key + /// that opens instances which no longer exist is just clutter in the + /// silo's key list. + pub(crate) async fn reap_ssh_keys( + &self, + target: &InstanceMap, + log: &Logger, + ) -> Result<(), OxideError> { + // Every key an environment in the db still wants. + let wanted: std::collections::BTreeSet = + target.iter().map(|i| i.ssh_key_name()).collect(); + + let mut names = Vec::new(); + let mut keys = self + .client + .current_user_ssh_key_list() + .limit(PAGE_SIZE) + .stream(); + while let Some(key) = keys.next().await { + let name = key?.name.to_string(); + // Ours by the same prefix rule as everything else: this key list + // belongs to a silo user that may well have keys of their own. + if name.starts_with(&format!("{INSTANCE_PREFIX}-")) + && !wanted.contains(&name) + { + names.push(name); + } + } + + for name in names { + info!(log, "deleting orphaned ssh key"; "key" => &name); + let deleted = self + .client + .current_user_ssh_key_delete() + .ssh_key(name.as_str()) + .send() + .await; + + if let Err(e) = deleted { + if is_inconclusive(&e) { + warn!(log, "ssh key delete did not report back"; + "key" => &name, + slog_error_chain::InlineErrorChain::new(&e), + ); + continue; + } + return Err(e); + } + } + + Ok(()) + } + + /// Delete boot disks belonging to instances nothing wants any more. + /// + /// Deleting an instance leaves its boot disk behind, detached, so without + /// this every environment that goes away costs the rack a disk the size of + /// [`BOOT_DISK_GIB`] forever. + /// + /// Disks carry the same name as the instance they were built for, so the + /// same rule decides ownership: a disk whose name does not parse as one of + /// ours is somebody else's and is left alone. + pub(crate) async fn reap_disks( + &self, + target: &InstanceMap, + log: &Logger, + ) -> Result<(), OxideError> { + let mut disks = self + .client + .disk_list() + .project(self.project.as_str()) + .limit(PAGE_SIZE) + .stream(); + + while let Some(disk) = disks.next().await { + let disk = disk?; + let name = disk.name.to_string(); + + if !reapable(&name, &disk.state, target) { + continue; + } + + info!(log, "deleting orphaned boot disk"; "disk" => &name); + let deleted = self + .client + .disk_delete() + .disk(name.as_str()) + .project(self.project.as_str()) + .send() + .await; + + if let Err(e) = deleted { + // As with instances, a delete whose connection died may have + // gone through. One disk is also no reason to stop reaping the + // rest, so report and carry on either way. + if is_inconclusive(&e) { + warn!(log, "boot disk delete did not report back"; + "disk" => &name, + slog_error_chain::InlineErrorChain::new(&e), + ); + continue; + } + return Err(e); + } + } + + Ok(()) + } + + /// Resolve the images an environment's instances should boot from. + /// + /// A name given explicitly must exist. A kind left unset resolves to the + /// newest image whose name starts with that kind's prefix. + pub(crate) async fn resolve_images( + &self, + vivado: Option<&str>, + helios: Option<&str>, + artifact: Option<&str>, + ) -> Result { + let images = self.visible_images().await?; + + Ok(EnvironmentImages { + vivado: choose_image(&images, InstanceKind::Vivado, vivado)?, + helios: choose_image(&images, InstanceKind::Helios, helios)?, + artifact: choose_image(&images, InstanceKind::Artifact, artifact)?, + }) + } + + /// Every image the service can see, from both the project and the silo. + /// + /// Project images shadow silo images of the same name, matching how the + /// control plane resolves them. + async fn visible_images(&self) -> Result, ImageError> { + let mut images = Vec::new(); + + let mut silo = self.client.image_list().limit(PAGE_SIZE).stream(); + while let Some(image) = silo.next().await { + images.push(image?); + } + + let mut project = self + .client + .image_list() + .project(self.project.as_str()) + .limit(PAGE_SIZE) + .stream(); + while let Some(image) = project.next().await { + images.push(image?); + } + + Ok(images) + } + pub(crate) async fn create_instance( + &self, + instance: &UserInstance, + log: &Logger, + ) -> Result<(), OxideError> { + let name = instance.oxide_instance_name(); + + let Some(image) = instance.image.as_ref() else { + warn!(log, "cannot create instance without an image"; + "instance" => &name, + ); + return Ok(()); + }; + + // The boot disk is created along with the instance and is what the image + // is laid down on. + let boot_disk = types::InstanceDiskAttachment::Create { + description: format!("vw {} boot disk", instance.kind), + disk_backend: types::DiskBackend::Distributed( + types::DiskSource::Image { + image_id: image.id, + read_only: false, + }, + ), + name: name.parse().map_err(bad_name)?, + size: types::ByteCount(BOOT_DISK_GIB * 1024 * 1024 * 1024), + }; + + let shape = instance.kind.shape(); + + // Registered once per pass by `ensure_ssh_keys`, before any of this + // environment's instances are created. + let key_name = instance + .public_key + .as_ref() + .map(|_| instance.ssh_key_name()); + + let body = types::InstanceCreate { + name: name.parse().map_err(bad_name)?, + description: format!( + "vw {} instance for {}/{}", + instance.kind, instance.user, instance.environment + ), + hostname: instance.hostname().parse().map_err(bad_name)?, + ncpus: types::InstanceCpuCount(shape.vcpus), + memory: types::ByteCount(shape.memory_gib * 1024 * 1024 * 1024), + boot_disk: Some(boot_disk), + // Instances are reachable from outside the rack so that the vw client + // and its source-sync daemon can talk to them directly. + external_ips: vec![types::ExternalIpCreate::Ephemeral { + pool_selector: types::PoolSelector::Auto { ip_version: None }, + }], + start: true, + // Attached by name. Without this the instance comes up with no + // way in. + ssh_public_keys: Some( + key_name + .iter() + .map(|name| { + types::NameOrId::Name( + name.parse().expect("a name we built ourselves"), + ) + }) + .collect(), + ), + ..default_instance_create() + }; + + info!(log, "creating instance"; + "instance" => &name, + "image" => &image.name, + "ssh_key" => key_name.as_deref().unwrap_or("none"), + "vcpus" => shape.vcpus, + "memory_gib" => shape.memory_gib, + ); + self.client + .instance_create() + .project(self.project.as_str()) + .body(body) + .send() + .await?; + + Ok(()) + } + + pub(crate) async fn delete_instance( + &self, + instance: &UserInstance, + log: &Logger, + ) -> Result<(), OxideError> { + let name = instance.oxide_instance_name(); + + // An instance has to be stopped before the control plane will delete it. + // Stopping one that is already stopping or stopped is harmless, so the + // only states worth acting on are the ones that are still running. + if instance.is_running_or_starting() { + info!(log, "stopping instance before delete"; "instance" => &name); + self.client + .instance_stop() + .instance(name.as_str()) + .project(self.project.as_str()) + .send() + .await?; + // The stop is not instantaneous. Leave the delete for a later pass + // rather than blocking this one waiting for the state to settle. + return Ok(()); + } + if !instance.is_stopped() { + // Still on its way down, or in a state the control plane will not let + // us delete from. Try again next pass. + return Ok(()); + } + + info!(log, "deleting instance"; "instance" => &name); + self.client + .instance_delete() + .instance(name.as_str()) + .project(self.project.as_str()) + .send() + .await?; + + Ok(()) + } + + pub(crate) async fn ensure_instance_running( + &self, + instance: &UserInstance, + log: &Logger, + ) -> Result<(), OxideError> { + match run_action(instance) { + RunAction::Create => self.create_instance(instance, log).await, + RunAction::Wait => Ok(()), + RunAction::Start => { + let name = instance.oxide_instance_name(); + info!(log, "starting instance"; "instance" => &name); + self.client + .instance_start() + .instance(name.as_str()) + .project(self.project.as_str()) + .send() + .await?; + Ok(()) + } + } + } +} + +/// Narrow a project's instances down to the ones this service manages. +/// +/// NOTE the Oxide Cloud Computer does not have tags, so we need to encode +/// vw instance semantics in names. The format is +/// +/// vwsvc-{user name}-{env name}-{instance kind} +/// +/// where instance kind is currently one of vivado, helios or artifact. +/// +/// This is the only thing standing between a reconciler pass and somebody +/// else's instances: the project holds more than ours, and an instance that +/// does not parse back out to the shape above is dropped here rather than +/// carried forward as an instance with no target — which is what a pass +/// deletes. Nothing downstream re-checks, so the filter has to be right here. +fn ours( + instances: impl IntoIterator, +) -> InstanceMap { + let mut map = InstanceMap::new(); + for (name, id, state) in instances { + let Some(mut instance) = parse_instance_name(&name) else { + continue; + }; + instance.oxide_instance = Some(OxideInstance { + id: Some(id), + state, + // Filled in per instance by the caller; the list does not carry + // addresses. + external_ip: None, + internal_ip: None, + }); + // Two Oxide instances cannot share a name, so this cannot collide. + map.insert_overwrite(instance); + } + map +} + +/// What a pass should do with an instance it wants running. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RunAction { + /// Not on the rack. Make it. + Create, + /// Up, on its way up, or on its way down. Leave it for a later pass. + Wait, + /// Settled and stopped. Start it. + Start, +} + +/// Decide what to do with an instance, separately from doing it. +/// +/// Split out because the first branch is easy to get subtly wrong and +/// impossible to notice: `oxide_instance` holds both what the rack reported +/// and the marker written when a create is asked for, and reading the marker +/// as an existing instance makes every pass skip the create, report success, +/// and build nothing. +pub(crate) fn run_action(instance: &UserInstance) -> RunAction { + // Nothing on the rack yet, so there is nothing to start. + if !instance.exists_on_rack() { + return RunAction::Create; + } + // Starting an instance that is mid-transition is at best a no-op and at + // worst an error. + if instance.is_running_or_starting() { + return RunAction::Wait; + } + // Anything still shutting down is picked up once it has settled; only a + // fully stopped instance can be started. + if !instance.is_stopped() { + return RunAction::Wait; + } + RunAction::Start +} + +/// Whether the control plane refused a create because the thing is already +/// there. +/// +/// Two writers racing on the same name is not a failure when the name existing +/// is exactly the outcome wanted. Matched on the control plane's own error +/// code rather than the status, which is a plain `400 Bad Request` and would +/// otherwise swallow genuinely malformed requests: +/// +/// ```text +/// status: 400 Bad Request; value: Error { error_code: Some("ObjectAlreadyExists"), +/// message: "already exists: ssh-key \"vwsvc-rcgoodfellow-darmok\"", .. } +/// ``` +fn already_exists(error: &OxideError) -> bool { + match error { + Error::ErrorResponse(response) => { + response.error_code.as_deref() == Some("ObjectAlreadyExists") + } + _ => false, + } +} + +/// Whether an error leaves the outcome of a request genuinely unknown. +/// +/// A request whose connection died on the way to or from the rack may well +/// have been carried out anyway — the control plane takes longer to delete an +/// instance than something in the path is willing to hold a connection open +/// for, so the work happens and the answer never arrives. Treating that as a +/// failure is wrong twice over: it reports a problem where there is none, and +/// it abandons the rest of the pass over something the next pass will see the +/// truth about. +/// +/// A response that did arrive is a different matter. If the control plane said +/// no, it meant it. +/// +/// The match is written out rather than using a wildcard so that a new variant +/// in the client has to be classified rather than silently assumed benign. +pub(crate) fn is_inconclusive(error: &OxideError) -> bool { + match error { + // Nothing came back, so the request's fate is unknown. + Error::CommunicationError(_) + | Error::ResponseBodyError(_) + | Error::InvalidUpgrade(_) => true, + + // The rack answered. Whatever it said, it is the truth. + Error::ErrorResponse(_) + | Error::UnexpectedResponse(_) + | Error::InvalidResponsePayload(_, _) + // Ours to fix, and retrying will not help. + | Error::InvalidRequest(_) + | Error::Custom(_) => false, + } +} + +/// Whether a disk should be deleted on this pass. +/// +/// Three things all have to hold, and the first is the one that matters: the +/// project holds disks belonging to other people's instances, and deleting one +/// of those destroys work this service never created. +fn reapable( + name: &str, + state: &types::DiskState, + target: &InstanceMap, +) -> bool { + // Ours, by the same naming rule the instances use. + parse_instance_name(name).is_some() + // Not wanted. This also covers the window during creation, when the + // disk exists before the instance it belongs to does. + && target.get(name).is_none() + // Deletable. Anything still attached belongs to an instance on its way + // out, and a later pass gets it once the control plane has finished. + && matches!(state, types::DiskState::Detached) +} + +/// Recover the environment an Oxide instance belongs to from its name. +/// +/// Returns `None` for anything that is not one of ours. +/// +/// Parsed from the right, because the username is the one field that may +/// contain a `-`: Github hands out names like `foo-bar` and we do not get to +/// choose them. The kind and the environment name are both guaranteed +/// hyphen-free — the kind by being one of a fixed set, the environment name by +/// [`crate::reconciler::validate_environment_name`] — so whatever sits between +/// the prefix and those two is the user, hyphens and all. +pub(crate) fn parse_instance_name(name: &str) -> Option { + let rest = name.strip_prefix(INSTANCE_PREFIX)?.strip_prefix('-')?; + + let (rest, kind) = rest.rsplit_once('-')?; + let kind = kind.parse().ok()?; + let (user, environment) = rest.rsplit_once('-')?; + + if user.is_empty() || environment.is_empty() { + return None; + } + + Some(UserInstance { + user: user.to_owned(), + environment: environment.to_owned(), + kind, + // The image an existing instance was built from, and the key it was + // given, are recorded in the db rather than recoverable from the + // instance itself. + image: None, + public_key: None, + oxide_instance: None, + }) +} + +/// Pick the image for `kind`, either the one named or the newest match. +fn choose_image( + images: &[types::Image], + kind: InstanceKind, + requested: Option<&str>, +) -> Result { + if let Some(name) = requested { + return images + .iter() + .find(|image| image.name.as_str() == name) + .map(image_ref) + .ok_or_else(|| ImageError::NoSuchImage(name.to_owned())); + } + + // Newest by creation time rather than by the date in the name: the + // timestamp is authoritative and always present, whereas the name suffix + // is a convention. + images + .iter() + .filter(|image| image.name.as_str().starts_with(kind.image_prefix())) + .max_by_key(|image| image.time_created) + .map(image_ref) + .ok_or_else(|| { + ImageError::NoMatchingImage(kind.image_prefix().to_owned()) + }) +} + +fn image_ref(image: &types::Image) -> ImageRef { + ImageRef { + id: image.id, + name: image.name.to_string(), + } +} + +/// The fields of an `InstanceCreate` the reconciler does not set. +fn default_instance_create() -> types::InstanceCreate { + types::InstanceCreate { + anti_affinity_groups: Vec::new(), + auto_restart_policy: None, + boot_disk: None, + cpu_platform: None, + description: String::new(), + disks: Vec::new(), + enable_jumbo_frames: false, + external_ips: Vec::new(), + hostname: "placeholder".parse().expect("valid hostname"), + memory: types::ByteCount(0), + multicast_groups: Vec::new(), + name: "placeholder".parse().expect("valid instance name"), + ncpus: types::InstanceCpuCount(0), + network_interfaces: + types::InstanceNetworkInterfaceAttachment::DefaultIpv4, + ssh_public_keys: None, + start: true, + user_data: String::new(), + } +} + +/// An instance name the control plane will not accept. +/// +/// User and environment names are validated on the way in, so reaching this +/// means the naming scheme itself is wrong rather than the caller's input. +fn bad_name(e: impl std::fmt::Display) -> OxideError { + OxideError::InvalidRequest(format!("invalid instance name: {e}")) +} + +#[cfg(test)] +mod test { + use super::*; + use uuid::Uuid; + + fn listed(name: &str) -> (String, Uuid, types::InstanceState) { + ( + name.to_owned(), + Uuid::new_v4(), + types::InstanceState::Running, + ) + } + + #[test] + fn only_instances_we_named_are_managed() { + // A realistic project: ours mixed in with everything else that + // happens to live there. Only ours may come out the other side, + // because whatever does is a candidate for deletion. + let map = ours([ + listed("vwsvc-ferris-alpha-vivado"), + listed("vwsvc-ferris-alpha-helios"), + listed("vwsvc-foo-bar-beta-artifact"), + // Not ours, and deleting any of these would be somebody's bad day. + listed("build-runner-3"), + listed("gimlet-dev"), + listed("vwsvc"), + listed("vwsvc-ferris"), + listed("vwsvc-ferris-alpha"), + listed("vwsvc-ferris-alpha-mystery"), + listed("notvwsvc-ferris-alpha-vivado"), + listed("vwsvcferris-alpha-vivado"), + ]); + + let mut names: Vec = + map.iter().map(|i| i.oxide_instance_name()).collect(); + names.sort(); + assert_eq!( + names, + [ + "vwsvc-ferris-alpha-helios", + "vwsvc-ferris-alpha-vivado", + "vwsvc-foo-bar-beta-artifact", + ] + ); + } + + #[test] + fn a_managed_instance_keeps_its_identity_and_state() { + let id = Uuid::new_v4(); + let map = ours([( + String::from("vwsvc-foo-bar-alpha-vivado"), + id, + types::InstanceState::Stopped, + )]); + + let instance = map.iter().next().expect("one instance"); + assert_eq!(instance.user, "foo-bar"); + assert_eq!(instance.environment, "alpha"); + assert_eq!(instance.kind, InstanceKind::Vivado); + + // The name has to reconstruct exactly, because that is what a delete + // is aimed at. A lossy round trip would target the wrong instance. + assert_eq!( + instance.oxide_instance_name(), + "vwsvc-foo-bar-alpha-vivado" + ); + + let oxide = instance.oxide_instance.as_ref().expect("carries state"); + assert_eq!(oxide.id, Some(id)); + assert_eq!(oxide.state, types::InstanceState::Stopped); + } + + /// A target map holding exactly the named instances. + fn wanted(names: &[&str]) -> InstanceMap { + let mut map = InstanceMap::new(); + for name in names { + map.insert_overwrite( + parse_instance_name(name).expect("a well formed name"), + ); + } + map + } + + #[test] + fn only_our_disks_are_ever_reaped() { + // Names taken from a real project: ours alongside boot disks that + // belong to instances this service knows nothing about. Reaping one of + // those would destroy somebody else's machine. + let target = wanted(&[]); + for theirs in [ + "katie-test-redhawk-dev-20260724035113-48889b", + "rhbs-noble-cloud-a6640b", + "vhdl-sim-jammy-server-6c074e", + ] { + assert!( + !reapable(theirs, &types::DiskState::Detached, &target), + "'{theirs}' is not ours and must be left alone", + ); + } + + for ours in [ + "vwsvc-rcgoodfellow-darmok-vivado", + "vwsvc-rcgoodfellow-darmok-helios", + "vwsvc-rcgoodfellow-darmok-artifact", + ] { + assert!( + reapable(ours, &types::DiskState::Detached, &target), + "'{ours}' is ours, unwanted and detached, so it should go", + ); + } + } + + #[test] + fn a_disk_its_environment_still_wants_is_kept() { + let target = wanted(&["vwsvc-rcgoodfellow-darmok-vivado"]); + + assert!(!reapable( + "vwsvc-rcgoodfellow-darmok-vivado", + &types::DiskState::Detached, + &target, + )); + // A sibling whose environment was deleted is still fair game. + assert!(reapable( + "vwsvc-rcgoodfellow-darmok-helios", + &types::DiskState::Detached, + &target, + )); + } + + #[test] + fn a_disk_still_in_use_is_left_for_a_later_pass() { + let target = wanted(&[]); + let name = "vwsvc-rcgoodfellow-darmok-vivado"; + let instance = Uuid::new_v4(); + + // Only a detached disk can be deleted; the rest are mid-transition. + for busy in [ + types::DiskState::Attached(instance), + types::DiskState::Attaching(instance), + types::DiskState::Detaching(instance), + types::DiskState::Creating, + types::DiskState::Finalizing, + ] { + assert!( + !reapable(name, &busy, &target), + "a disk in {busy:?} cannot be deleted yet", + ); + } + } + + #[test] + fn a_request_that_never_answered_is_not_a_failure() { + // The rack takes longer to delete an instance than something in the + // path will hold a connection open for, so the work happens and the + // answer never arrives. Calling that a failure reports a problem that + // is not there and abandons the rest of the pass. + // + // `reqwest::Error` has no public constructor, so the transport + // variants cannot be built here; `is_inconclusive` matches every + // variant explicitly instead, which makes the compiler insist that any + // new one gets classified. What is testable is the other side of the + // rule: an answer that did arrive is always conclusive. + assert!(!is_inconclusive(&OxideError::InvalidRequest( + "malformed".into() + ))); + assert!(!is_inconclusive(&OxideError::Custom("nope".into()))); + } + + #[test] + fn a_name_that_is_already_taken_is_not_a_failure() { + // An environment's three instances share one ssh key, and two writers + // racing to register it is not a problem when all that was wanted is + // for it to exist. Matched on the control plane's error code rather + // than the status, which is a plain 400 that a malformed request also + // carries — as the other two cases here check. + assert!(!already_exists(&OxideError::InvalidRequest("bad".into()))); + assert!(!already_exists(&OxideError::Custom("nope".into()))); + } + + fn recorded(state: Option) -> UserInstance { + let mut instance = parse_instance_name("vwsvc-ferris-alpha-vivado") + .expect("a well formed name"); + instance.oxide_instance = state.map(|state| OxideInstance { + id: Some(Uuid::new_v4()), + state, + external_ip: None, + internal_ip: None, + }); + instance + } + + #[test] + fn an_instance_the_rack_has_not_got_is_created() { + // Nothing recorded at all. + assert_eq!(run_action(&recorded(None)), RunAction::Create); + + // And the case that actually bit: this service's own marker for a + // create it asked for. It carries `Creating`, so anything treating it + // as a live instance decides it is already on its way up and never + // builds it — every pass, without an error to show for it. + let mut pending = recorded(None); + pending.oxide_instance = Some(OxideInstance { + id: None, + state: types::InstanceState::Creating, + external_ip: None, + internal_ip: None, + }); + assert_eq!(run_action(&pending), RunAction::Create); + } + + #[test] + fn an_instance_in_motion_is_left_alone() { + for state in [ + types::InstanceState::Creating, + types::InstanceState::Starting, + types::InstanceState::Running, + types::InstanceState::Rebooting, + types::InstanceState::Migrating, + types::InstanceState::Repairing, + types::InstanceState::Stopping, + ] { + assert_eq!( + run_action(&recorded(Some(state))), + RunAction::Wait, + "{state} needs no action", + ); + } + } + + #[test] + fn a_stopped_instance_is_started() { + assert_eq!( + run_action(&recorded(Some(types::InstanceState::Stopped))), + RunAction::Start, + ); + } + + #[test] + fn an_instance_names_itself_after_its_kind_and_environment() { + // The Oxide instance name has to be unique project-wide and so drags + // the owner and a prefix along; the shell prompt does not need any of + // it. + for (kind, expected) in [ + (InstanceKind::Vivado, "vivado-darmok"), + (InstanceKind::Helios, "helios-darmok"), + (InstanceKind::Artifact, "artifact-darmok"), + ] { + let mut instance = + parse_instance_name("vwsvc-rcgoodfellow-darmok-vivado") + .expect("a well formed name"); + instance.kind = kind; + + assert_eq!(instance.hostname(), expected); + // No dot: cloud-init would read one as an FQDN separator and keep + // only the first label, dropping the environment from the name the + // box calls itself. + assert!( + !expected.contains('.'), + "{expected} would be truncated to its first label", + ); + // And the control plane has to accept it, which is the part a + // typo would only reveal at create time. + assert!( + expected.parse::().is_ok(), + "{expected} is not a hostname the control plane will take", + ); + } + } + + #[test] + fn a_hyphenated_owner_does_not_reach_the_hostname() { + // Github names may carry hyphens, which are fine in a hostname label + // but would put the owner into a name nobody inside the environment + // needs to read. + let instance = parse_instance_name("vwsvc-foo-bar-darmok-vivado") + .expect("a well formed name"); + assert_eq!(instance.user, "foo-bar"); + assert_eq!(instance.hostname(), "vivado-darmok"); + } +} diff --git a/vw-svc/src/oxide.rs.tmp b/vw-svc/src/oxide.rs.tmp new file mode 100644 index 0000000..e69de29 diff --git a/vw-svc/src/reconciler.rs b/vw-svc/src/reconciler.rs new file mode 100644 index 0000000..b74e3b4 --- /dev/null +++ b/vw-svc/src/reconciler.rs @@ -0,0 +1,989 @@ +//! This module implements the vw instance reconciler. The reconciler looks +//! at the environments in the db and ensures and reconciles their target +//! state with what's actually running on an Oxide Cloud Computer. +//! +//! A pass is a diff: the db says which instances should exist, the Oxide API +//! says which do, and the difference becomes a list of things to create, +//! destroy, or start. Nothing here waits for an instance to finish changing +//! state — a pass acts on what it can and leaves the rest for the next one, +//! so a slow boot or a slow shutdown never blocks reconciliation of anything +//! else. + +use std::{fmt::Display, str::FromStr, time::Duration}; + +use crate::{ + db::{self, list_all_environment_instances, ListError}, + oxide as ox, +}; +use daft::Diffable; +use futures::{stream::FuturesUnordered, StreamExt}; +use iddqd::{id_upcast, IdOrdItem, IdOrdMap}; +use slog::{error, info, warn, Logger}; +use slog_error_chain::InlineErrorChain; +use tokio::sync::Notify; +use vw_api_types_versions::latest::{ + ImageRef, OxideInstance, UserEnvironmentPathParam, +}; + +pub(crate) struct InstanceReconciler { + /// How long to wait between passes when nothing asks for one sooner. + interval: Duration, + /// Rung by the API when an environment is created or deleted, so a pass + /// runs immediately instead of waiting out the interval. + wake: std::sync::Arc, +} + +impl InstanceReconciler { + pub(crate) fn new( + interval: Duration, + wake: std::sync::Arc, + ) -> Self { + Self { interval, wake } + } + + /// Reconcile forever, one pass at a time. + /// + /// A failed pass is logged and retried on the next tick rather than + /// stopping the loop: the causes are transient often enough (an API + /// blip, an instance mid-transition) that giving up would be worse than + /// trying again. + pub(crate) async fn run(&self, log: Logger) { + loop { + self.run_once(&log).await; + tokio::select! { + _ = tokio::time::sleep(self.interval) => {} + _ = self.wake.notified() => {} + } + } + } + + async fn run_once(&self, log: &Logger) { + if let Err(e) = self.pass(log).await { + error!(log, "reconciliation error"; InlineErrorChain::new(&e)); + } + } + + async fn pass(&self, log: &Logger) -> Result<(), PassError> { + // One client for this pass, discarded when it ends. + let session = ox::session()?; + + let target = list_all_environment_instances()?; + let current = session.get_instances().await?; + let plan = self.plan(&target, ¤t, log); + + // Before acting, not after: the writes describe state read at the top + // of this pass, and `execute` can spend a long time waiting on the + // control plane. Deferring them would leave the environment looking + // untouched for the whole of it. + plan.write_status(log); + plan.execute(&session, log).await?; + + // Deleting an instance does not take its boot disk with it, so the + // disks have to be reconciled too or every environment that goes away + // leaves 600GiB behind. Done against the target rather than against + // what was just deleted, so disks orphaned by an earlier crash or a + // half-finished pass get swept up as well. + session.reap_disks(&target, log).await?; + session.reap_ssh_keys(&target, log).await?; + + Ok(()) + } + + fn plan( + &self, + target: &InstanceMap, + current: &InstanceMap, + log: &Logger, + ) -> PassAction { + // daft diffs `before.diff(&after)`, so the rack goes on the left and + // the db on the right. That makes `added` the instances the db wants + // that the rack does not have, and `removed` the ones on the rack that + // nothing wants any more. + let diff = current.diff(target); + + // These come off the db side, so they carry the image to build from. + let mut to_create = InstanceMap::new(); + for instance in diff.added.iter() { + to_create.insert_overwrite((*instance).clone()); + } + + // These come off the rack side, so they carry the live state that says + // whether to stop an instance or delete it. + let mut to_destroy = InstanceMap::new(); + for instance in diff.removed.iter() { + to_destroy.insert_overwrite((*instance).clone()); + } + + // Anything in both places should be running, and the db's idea of its + // state should match the rack's. + let mut ensure_running = InstanceMap::new(); + let mut status_updates = Vec::new(); + for pair in diff.common.iter() { + let (current, target) = (pair.before(), pair.after()); + + // Take the db's record for the image, and the rack's for the + // state: acting on what the db last wrote would mean starting + // instances that are already up. + let mut instance = (*target).clone(); + instance.oxide_instance = current.oxide_instance.clone(); + ensure_running.insert_overwrite(instance); + + if !same_state(&target.oxide_instance, ¤t.oxide_instance) { + status_updates.push(StatusUpdate { + user: current.user.clone(), + environment: current.environment.clone(), + kind: current.kind, + oxide_instance: current.oxide_instance.clone(), + }); + } + } + + for instance in diff.added.iter() { + match instance.oxide_instance.as_ref() { + // The db names an instance id the rack no longer has. Clear + // it, or the db goes on advertising something that does not + // resolve. + Some(recorded) if recorded.id.is_some() => { + status_updates.push(StatusUpdate { + user: instance.user.clone(), + environment: instance.environment.clone(), + kind: instance.kind, + oxide_instance: None, + }); + } + // A create already asked for and not yet seen to land. Leave + // the marker be; it is the only sign of life there is until + // the rack starts reporting the instance. + Some(_) => {} + // About to be asked for. Record that now rather than after the + // request comes back: the control plane takes long enough that + // an environment would otherwise sit there looking like + // nothing had happened. + None => { + status_updates.push(StatusUpdate { + user: instance.user.clone(), + environment: instance.environment.clone(), + kind: instance.kind, + oxide_instance: Some(OxideInstance { + id: None, + state: oxide::types::InstanceState::Creating, + external_ip: None, + internal_ip: None, + }), + }); + } + } + } + + if !to_create.is_empty() + || !to_destroy.is_empty() + || !status_updates.is_empty() + { + info!(log, "reconciliation plan"; + "create" => to_create.len(), + "destroy" => to_destroy.len(), + "ensure_running" => ensure_running.len(), + "status_updates" => status_updates.len(), + ); + } + + PassAction { + to_create, + to_destroy, + ensure_running, + status_updates, + } + } +} + +/// Reject an environment name that would not survive the instance naming +/// scheme. +/// +/// Instance names are `vwsvc-{user}-{env}-{kind}`, and they are taken back +/// apart from the right so that a username may contain `-`. That only works if +/// the environment name does not — otherwise the split lands in the wrong +/// place and one environment can be mistaken for another. The remaining rules +/// are what the control plane accepts for a `Name`. +pub(crate) fn validate_environment_name(name: &str) -> Result<(), String> { + if name.is_empty() { + return Err(String::from("environment name cannot be empty")); + } + if name.contains('-') { + return Err(format!( + "'{name}' cannot contain '-'; it separates the parts of the \ + underlying instance name" + )); + } + if !name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) + { + return Err(format!( + "'{name}' may only contain lowercase letters and digits" + )); + } + if !name.starts_with(|c: char| c.is_ascii_lowercase()) { + return Err(format!("'{name}' must start with a lowercase letter")); + } + Ok(()) +} + +/// Reject a username that cannot be part of an Oxide instance name. +/// +/// Unlike an environment name this is not the caller's to choose — it comes +/// from Github — so the rules are as loose as the control plane allows: `-` is +/// fine, since names are parsed from the right. What is left over is a Github +/// name the control plane would refuse, which is worth saying plainly rather +/// than letting the reconciler fail on it later. +pub(crate) fn validate_user_name(name: &str) -> Result<(), String> { + if name.is_empty() { + return Err(String::from("username cannot be empty")); + } + if !name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + return Err(format!( + "github username '{name}' contains characters that cannot appear \ + in an instance name" + )); + } + if !name.starts_with(|c: char| c.is_ascii_lowercase()) { + return Err(format!( + "github username '{name}' must start with a letter to be part of \ + an instance name" + )); + } + if name.ends_with('-') { + return Err(format!("github username '{name}' cannot end with '-'")); + } + Ok(()) +} + +/// Whether the db's record of an instance already matches the rack's. +fn same_state( + recorded: &Option, + actual: &Option, +) -> bool { + match (recorded, actual) { + (Some(recorded), Some(actual)) => { + recorded.id == actual.id + && recorded.state == actual.state + // The address turns up some time after the instance does, and + // it is the part somebody actually needs. + && recorded.external_ip == actual.external_ip + && recorded.internal_ip == actual.internal_ip + } + (None, None) => true, + _ => false, + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Diffable)] +pub(crate) enum InstanceKind { + Vivado, + Helios, + Artifact, +} + +impl InstanceKind { + /// Every kind, so a whole environment can be enumerated. + pub(crate) const ALL: [InstanceKind; 3] = + [Self::Vivado, Self::Helios, Self::Artifact]; + + /// Images for this kind are named with this prefix followed by a date, and + /// an environment that does not name an image explicitly gets the newest + /// one that matches. + /// + /// These are vw's own images, built by `redhawk-dev-image`, each of which + /// has an agent already installed and enabled. A stock OS image will boot + /// and answer ssh, but nothing will ever reach it through this service. + pub(crate) fn image_prefix(&self) -> &'static str { + match self { + Self::Vivado => "vw-vivado-", + Self::Helios => "vw-helios-", + Self::Artifact => "vw-artifact-", + } + } + + /// How much machine this kind of instance gets. + /// + /// Synthesis and a kernel build both take whatever they are given, so the + /// two build instances get the largest shape an environment is worth. The + /// artifact instance compiles nothing: it runs an object store, and the + /// work it does is moving bytes between a socket and a disk. Sizing it + /// like a build machine only takes cores away from environments that + /// would use them. + pub(crate) fn shape(&self) -> Shape { + match self { + Self::Vivado | Self::Helios => Shape { + vcpus: 16, + memory_gib: 32, + }, + Self::Artifact => Shape { + vcpus: 4, + memory_gib: 16, + }, + } + } +} + +/// The cpu and memory an instance is created with. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Shape { + pub(crate) vcpus: u16, + pub(crate) memory_gib: u64, +} + +impl Display for InstanceKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Vivado => write!(f, "vivado"), + Self::Helios => write!(f, "helios"), + Self::Artifact => write!(f, "artifact"), + } + } +} + +impl FromStr for InstanceKind { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "vivado" => Ok(Self::Vivado), + "helios" => Ok(Self::Helios), + "artifact" => Ok(Self::Artifact), + _ => Err(()), + } + } +} + +#[derive(Clone, Debug, Diffable)] +pub(crate) struct UserInstance { + pub(crate) user: String, + pub(crate) environment: String, + pub(crate) kind: InstanceKind, + /// The image this instance should boot from. + /// + /// `None` for instances discovered on the rack, whose image was chosen + /// when their environment was created, and for environments recorded + /// without an Oxide backend to resolve images against. + pub(crate) image: Option, + /// The public half of the environment's ssh key, attached to the instance + /// when it is created so it comes up reachable. + /// + /// `None` for instances discovered on the rack, where the key is recorded + /// in the db rather than on the instance. + pub(crate) public_key: Option, + /// What the Oxide API says about this instance, or what the db last + /// recorded, depending on which side of the diff it came from. + pub(crate) oxide_instance: Option, +} + +impl UserInstance { + /// The name of the Oxide silo ssh key shared by an environment's + /// instances. + /// + /// One key per environment rather than per instance, so a user has a + /// single key to fetch and use against all three. + pub(crate) fn ssh_key_name(&self) -> String { + format!("{}-{}-{}", ox::INSTANCE_PREFIX, self.user, self.environment) + } + + /// The hostname the instance sees itself as. + /// + /// Separate from the Oxide instance name, which has to be unique across + /// the whole project and so carries the owner and a prefix. Inside the + /// environment none of that is news, and a shell prompt reading + /// `ubuntu@vwsvc-rcgoodfellow-darmok-vivado` is a lot of it. + /// + /// Joined with `-` rather than `.` on purpose. A dotted name is a valid + /// hostname, but cloud-init reads the dot as an FQDN separator and sets + /// the static hostname to the first label alone — asking for + /// `vivado.darmok` gets a box that calls itself `vivado`, and which + /// environment it belongs to disappears from the prompt. A hyphen carries + /// no such meaning, so the whole name survives. + pub(crate) fn hostname(&self) -> String { + format!("{}-{}", self.kind, self.environment) + } + + pub(crate) fn oxide_instance_name(&self) -> String { + format!( + "{}-{}-{}-{}", + ox::INSTANCE_PREFIX, + self.user, + self.environment, + self.kind + ) + } + + /// Whether the rack has actually told us about this instance. + /// + /// A record carrying no id is this service's own marker for a create it + /// asked for and has not yet seen land — it says what was wanted, not what + /// exists. Anything deciding whether to create must ask this rather than + /// whether `oxide_instance` is set. + pub(crate) fn exists_on_rack(&self) -> bool { + self.oxide_instance + .as_ref() + .is_some_and(|instance| instance.id.is_some()) + } + + /// Whether this instance is up or on its way up, in which case starting it + /// again is not something to do. + pub(crate) fn is_running_or_starting(&self) -> bool { + use oxide::types::InstanceState; + matches!( + self.oxide_instance.as_ref().map(|i| &i.state), + Some( + InstanceState::Running + | InstanceState::Starting + | InstanceState::Creating + | InstanceState::Rebooting + | InstanceState::Migrating + | InstanceState::Repairing + ) + ) + } + + /// Whether this instance has settled in a state it can be started or + /// deleted from. + pub(crate) fn is_stopped(&self) -> bool { + use oxide::types::InstanceState; + matches!( + self.oxide_instance.as_ref().map(|i| &i.state), + Some(InstanceState::Stopped | InstanceState::Failed) + ) + } +} + +impl IdOrdItem for UserInstance { + type Key<'a> = String; + + fn key(&self) -> Self::Key<'_> { + // The Oxide instance name is unique across the rack by construction, + // which makes it the natural identity for both sides of the diff. + self.oxide_instance_name() + } + + id_upcast!(); +} + +pub(crate) type InstanceMap = IdOrdMap; + +/// A db record that no longer matches the rack. +struct StatusUpdate { + user: String, + environment: String, + kind: InstanceKind, + oxide_instance: Option, +} + +pub struct PassAction { + to_create: InstanceMap, + to_destroy: InstanceMap, + ensure_running: InstanceMap, + status_updates: Vec, +} + +impl PassAction { + async fn execute( + &self, + session: &ox::Session, + log: &Logger, + ) -> Result<(), PassError> { + // Each `async fn` has its own anonymous type, so futures from + // different functions cannot share a `Vec` without being boxed. + // `FuturesUnordered` of boxed futures lets them all run together and + // report back as they finish. + // Before anything concurrent: an environment's instances share one + // ssh key, and racing to register it is how two of the three end up + // failing. + session.ensure_ssh_keys(&self.to_create, log).await?; + + let mut tasks = FuturesUnordered::new(); + for inst in self.to_create.iter() { + tasks.push(boxed(labeled( + "create", + inst, + session.create_instance(inst, log), + ))); + } + for inst in self.ensure_running.iter() { + tasks.push(boxed(labeled( + "ensure_running", + inst, + session.ensure_instance_running(inst, log), + ))); + } + for inst in self.to_destroy.iter() { + tasks.push(boxed(labeled( + "destroy", + inst, + session.delete_instance(inst, log), + ))); + } + + // One instance failing says nothing about the others, so let every + // task finish and report the first failure only once they have. The + // next pass retries whatever did not take. + let mut first_error = None; + while let Some((operation, instance, result)) = tasks.next().await { + let Err(e) = result else { continue }; + + if ox::is_inconclusive(&e) { + // The rack may well have done what was asked and simply never + // said so. The next pass reads the actual state, so let it + // settle there rather than calling this a failure and + // abandoning the rest of the pass. + warn!(log, "instance operation did not report back"; + "operation" => operation, + "instance" => &instance, + InlineErrorChain::new(&e), + ); + continue; + } + + error!(log, "instance operation failed"; + "operation" => operation, + "instance" => &instance, + InlineErrorChain::new(&e), + ); + first_error.get_or_insert(e); + } + + match first_error { + Some(e) => Err(e.into()), + None => Ok(()), + } + } + + /// Bring the db's record of instance state back in line with the rack. + /// + /// Failures here are logged rather than propagated: the rack is the source + /// of truth and the next pass will try again, so a write that does not + /// land is not a reason to abandon the rest of the update. + fn write_status(&self, log: &Logger) { + for update in &self.status_updates { + let key = UserEnvironmentPathParam { + user: update.user.clone(), + name: update.environment.clone(), + }; + let mut environment = match db::get_environment_status(key.clone()) + { + Ok(environment) => environment, + Err(e) => { + warn!(log, "cannot read environment to update status"; + "user" => &update.user, + "environment" => &update.environment, + InlineErrorChain::new(&e), + ); + continue; + } + }; + + let slot = match update.kind { + InstanceKind::Vivado => &mut environment.vivado_instance, + InstanceKind::Helios => &mut environment.helios_instance, + InstanceKind::Artifact => &mut environment.artifact_instance, + }; + *slot = update.oxide_instance.clone(); + + if let Err(e) = db::update_environment_status(key, environment) { + warn!(log, "cannot record instance state"; + "user" => &update.user, + "environment" => &update.environment, + "kind" => %update.kind, + InlineErrorChain::new(&e), + ); + } + } + } +} + +/// What a task was doing and to which instance, carried alongside its result. +/// +/// Without this a failed pass says only that *an* operation failed, which is +/// no help when several instances are in flight at once. +type TaskResult = (&'static str, String, Result<(), ox::OxideError>); + +/// Tag a task with the operation and instance it belongs to. +async fn labeled<'a>( + operation: &'static str, + instance: &UserInstance, + future: impl std::future::Future> + + Send + + 'a, +) -> TaskResult { + let name = instance.oxide_instance_name(); + (operation, name, future.await) +} + +/// Box a future so differently-typed futures can share one collection. +fn boxed<'a>( + future: impl std::future::Future + Send + 'a, +) -> std::pin::Pin + Send + 'a>> +{ + Box::pin(future) +} + +#[derive(Debug, thiserror::Error)] +enum PassError { + // These deliberately do not interpolate their source: every log site + // reports them through `InlineErrorChain`, which appends the whole chain + // of causes itself. A wrapper that also embedded its source would print + // the cause twice. + #[error("listing environments from the db failed")] + DbList(#[from] ListError), + + #[error("talking to the oxide api failed")] + Oxide(#[from] ox::OxideError), + + #[error("connecting to the oxide api failed")] + Session(#[from] ox::SessionError), +} + +#[cfg(test)] +mod test { + use super::*; + use oxide::types::InstanceState; + use uuid::Uuid; + + fn instance( + user: &str, + environment: &str, + kind: InstanceKind, + state: Option, + ) -> UserInstance { + UserInstance { + user: user.to_owned(), + environment: environment.to_owned(), + kind, + image: None, + public_key: None, + oxide_instance: state.map(|state| OxideInstance { + id: Some(Uuid::nil()), + state, + external_ip: None, + internal_ip: None, + }), + } + } + + fn map(instances: impl IntoIterator) -> InstanceMap { + let mut map = InstanceMap::new(); + for instance in instances { + map.insert_overwrite(instance); + } + map + } + + fn reconciler() -> InstanceReconciler { + InstanceReconciler::new( + Duration::from_secs(1), + std::sync::Arc::new(Notify::new()), + ) + } + + fn log() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + #[test] + fn wanted_but_absent_instances_are_created() { + let target = + map([instance("ferris", "alpha", InstanceKind::Vivado, None)]); + let plan = reconciler().plan(&target, &InstanceMap::new(), &log()); + + assert_eq!(plan.to_create.len(), 1); + assert!(plan.to_destroy.is_empty()); + assert!(plan.ensure_running.is_empty()); + } + + #[test] + fn instances_nothing_wants_are_destroyed() { + let current = map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Running), + )]); + let plan = reconciler().plan(&InstanceMap::new(), ¤t, &log()); + + assert!(plan.to_create.is_empty()); + assert_eq!(plan.to_destroy.len(), 1); + } + + #[test] + fn instances_on_both_sides_are_kept_running() { + let target = + map([instance("ferris", "alpha", InstanceKind::Vivado, None)]); + let current = map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Stopped), + )]); + let plan = reconciler().plan(&target, ¤t, &log()); + + assert!(plan.to_create.is_empty()); + assert!(plan.to_destroy.is_empty()); + assert_eq!(plan.ensure_running.len(), 1); + + // The plan carries the live state, not the db's stale idea of it, so + // the executor can tell a stopped instance from a running one. + let instance = plan.ensure_running.iter().next().unwrap(); + assert!(instance.is_stopped()); + } + + #[test] + fn a_state_the_db_has_not_caught_up_with_is_recorded() { + let target = map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Starting), + )]); + let current = map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Running), + )]); + let plan = reconciler().plan(&target, ¤t, &log()); + + assert_eq!(plan.status_updates.len(), 1); + assert_eq!( + plan.status_updates[0] + .oxide_instance + .as_ref() + .map(|i| &i.state), + Some(&InstanceState::Running), + ); + } + + #[test] + fn a_state_already_matching_the_rack_is_left_alone() { + let both = || { + map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Running), + )]) + }; + let plan = reconciler().plan(&both(), &both(), &log()); + + assert!(plan.status_updates.is_empty()); + } + + #[test] + fn an_instance_the_rack_has_lost_has_its_record_cleared() { + // The db still names an instance id, but nothing on the rack answers + // to it any more. + let target = map([instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Running), + )]); + let plan = reconciler().plan(&target, &InstanceMap::new(), &log()); + + assert_eq!(plan.to_create.len(), 1); + assert_eq!(plan.status_updates.len(), 1); + assert!(plan.status_updates[0].oxide_instance.is_none()); + } + + #[test] + fn every_instance_of_an_environment_gets_its_own_entry() { + // The whole point of keying on the instance name: three instances of + // one environment coexist, where keying on user or environment alone + // would collapse them into one. + let map = map(InstanceKind::ALL + .map(|kind| instance("ferris", "alpha", kind, None))); + + assert_eq!(map.len(), 3); + } + + #[test] + fn the_artifact_instance_is_not_sized_like_a_build_machine() { + // It runs an object store and nothing else. Giving it a builder's + // shape costs every environment cores and memory that only the two + // instances doing the compiling can use. + let artifact = InstanceKind::Artifact.shape(); + + for building in [InstanceKind::Vivado, InstanceKind::Helios] { + let shape = building.shape(); + assert!( + artifact.vcpus < shape.vcpus, + "artifact has as many cpus as {building}", + ); + assert!( + artifact.memory_gib < shape.memory_gib, + "artifact has as much memory as {building}", + ); + } + } + + #[test] + fn instance_names_round_trip() { + for kind in InstanceKind::ALL { + let original = instance("ferris", "alpha", kind, None); + let name = original.oxide_instance_name(); + assert_eq!(name, format!("vwsvc-ferris-alpha-{kind}")); + + let parsed = + crate::oxide::parse_instance_name(&name).expect("parses back"); + assert_eq!(parsed.user, original.user); + assert_eq!(parsed.environment, original.environment); + assert_eq!(parsed.kind, original.kind); + } + } + + #[test] + fn instances_that_are_not_ours_are_ignored() { + // Nothing here may be mistaken for a vw instance, or a reconciler + // pass would delete somebody else's work. + for name in [ + "some-other-instance", + "vwsvc-ferris-alpha", + "vwsvc-ferris-alpha-vivado-extra", + "vwsvc-ferris-alpha-mystery", + "notvwsvc-ferris-alpha-vivado", + ] { + assert!( + crate::oxide::parse_instance_name(name).is_none(), + "'{name}' should not be treated as a vw instance", + ); + } + } + + #[test] + fn environment_names_that_would_not_survive_the_scheme_are_rejected() { + for good in ["alpha", "env2", "a", "x9y9"] { + assert!( + validate_environment_name(good).is_ok(), + "'{good}' should be valid", + ); + } + for bad in ["", "my-env", "My", "my_env", "9lives", "a.b"] { + assert!( + validate_environment_name(bad).is_err(), + "'{bad}' should be invalid", + ); + } + } + + #[test] + fn hyphenated_github_usernames_are_allowed() { + // Github hands these out and we do not get to choose them, so they + // must work rather than lock somebody out. + for good in ["foo-bar", "rcgoodfellow", "a-b-c", "x9"] { + assert!( + validate_user_name(good).is_ok(), + "'{good}' should be valid", + ); + } + for bad in ["", "Foo", "foo_bar", "9lives", "foo-"] { + assert!( + validate_user_name(bad).is_err(), + "'{bad}' should be invalid", + ); + } + } + + #[test] + fn a_hyphenated_username_still_round_trips() { + // Parsed from the right, so the hyphens land in the user and nowhere + // else. Getting this wrong makes the instance unrecognizable, and the + // reconciler would then recreate it on every pass. + let original = instance("foo-bar", "alpha", InstanceKind::Vivado, None); + let name = original.oxide_instance_name(); + assert_eq!(name, "vwsvc-foo-bar-alpha-vivado"); + + let parsed = + crate::oxide::parse_instance_name(&name).expect("parses back"); + assert_eq!(parsed.user, "foo-bar"); + assert_eq!(parsed.environment, "alpha"); + assert_eq!(parsed.kind, InstanceKind::Vivado); + } + + #[test] + fn a_create_is_recorded_before_it_is_requested() { + // The control plane takes long enough over a create that an + // environment would otherwise sit at "none" for the whole of it, + // looking like nothing had happened. + let target = + map([instance("ferris", "alpha", InstanceKind::Vivado, None)]); + let plan = reconciler().plan(&target, &InstanceMap::new(), &log()); + + assert_eq!(plan.to_create.len(), 1); + assert_eq!(plan.status_updates.len(), 1); + + let recorded = plan.status_updates[0] + .oxide_instance + .as_ref() + .expect("a marker to show for it"); + assert_eq!(recorded.state, InstanceState::Creating); + // No id yet: the rack has not answered, and inventing one would name + // an instance that does not exist. + assert!(recorded.id.is_none()); + } + + #[test] + fn a_pending_create_is_not_mistaken_for_a_lost_instance() { + // Second pass on an instance the rack has not started reporting yet. + // Clearing the marker here would drop the user back to "none" and undo + // the whole point of writing it. + let mut pending = + instance("ferris", "alpha", InstanceKind::Vivado, None); + pending.oxide_instance = Some(OxideInstance { + id: None, + state: InstanceState::Creating, + external_ip: None, + internal_ip: None, + }); + let target = map([pending]); + + let plan = reconciler().plan(&target, &InstanceMap::new(), &log()); + + assert_eq!(plan.to_create.len(), 1, "still worth asking again"); + assert!( + plan.status_updates.is_empty(), + "the marker should be left alone", + ); + } + + #[test] + fn a_pending_marker_is_not_an_instance_on_the_rack() { + // This is the distinction that decides whether a create happens at + // all. Reading the marker as an existing instance made every pass skip + // the create and report success, so the instance was never built and + // nothing ever complained. + let mut pending = + instance("ferris", "alpha", InstanceKind::Vivado, None); + pending.oxide_instance = Some(OxideInstance { + id: None, + state: InstanceState::Creating, + external_ip: None, + internal_ip: None, + }); + assert!(!pending.exists_on_rack()); + // ... even though it does look like it is on its way up. + assert!(pending.is_running_or_starting()); + + // An id only ever comes from the rack, so one means it is really + // there. + let real = instance( + "ferris", + "alpha", + InstanceKind::Vivado, + Some(InstanceState::Creating), + ); + assert!(real.exists_on_rack()); + + // And nothing recorded at all is plainly absent. + let absent = instance("ferris", "alpha", InstanceKind::Vivado, None); + assert!(!absent.exists_on_rack()); + } +} diff --git a/vw-svc/src/relay.rs b/vw-svc/src/relay.rs new file mode 100644 index 0000000..a07dabd --- /dev/null +++ b/vw-svc/src/relay.rs @@ -0,0 +1,416 @@ +//! Passing source through to the instance it belongs on. +//! +//! Nothing is kept here. A developer's machine holds the tree, the instance +//! holds a copy, and this service only decides who is allowed to talk to whom +//! and forwards the bytes. That is deliberate: a copy stored here would be a +//! third place for the tree to be subtly wrong, and there is nothing it could +//! recover that the developer's own working tree cannot. +//! +//! The instance is reached on its VPC address rather than its external one. +//! Both would work, but the internal path is the rack's own fabric — orders of +//! magnitude more bandwidth, and it never leaves the building. + +use vw_api_types_versions::latest::{TargetKind, UserEnvironmentPathParam}; + +use crate::{db, reconciler::InstanceKind}; + +/// The port an agent listens on. +/// +/// Fixed rather than discovered: the agents are started by this service's own +/// provisioning, so there is nothing to negotiate. +pub(crate) const AGENT_PORT: u16 = 2729; + +/// Where an agent lives on the rack's network. +fn agent_url(address: std::net::IpAddr) -> String { + match address { + std::net::IpAddr::V4(v4) => format!("http://{v4}:{AGENT_PORT}"), + std::net::IpAddr::V6(v6) => format!("http://[{v6}]:{AGENT_PORT}"), + } +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum RelayError { + #[error("environment does not exist")] + NoSuchEnvironment, + #[error("the {kind} instance for this environment does not exist yet")] + NoInstance { kind: TargetKind }, + #[error( + "the {kind} instance has no address on the rack's network yet; it is \ + probably still coming up" + )] + NoAddress { kind: TargetKind }, + #[error("reading the environment")] + Db(#[from] db::GetError), + #[error("building a client for the instance")] + Client(#[source] vw_api_client::Error), + #[error("talking to the {kind} instance")] + Agent { + kind: TargetKind, + #[source] + source: Box< + vw_api_client::agent::Error, + >, + }, +} + +/// A connection to the agent serving one half of one environment. +pub(crate) struct Agent { + pub(crate) client: vw_api_client::agent::Client, + pub(crate) environment: String, + pub(crate) kind: TargetKind, +} + +impl Agent { + /// Find the instance serving `kind` for `user`'s environment `name`. + /// + /// Everything this needs is already recorded by the reconciler, so no call + /// to the rack is made to work out where to send things. + pub(crate) fn resolve( + user: &str, + name: &str, + kind: TargetKind, + args: &crate::ServerArgs, + ) -> Result { + // A development override stands in for the instance lookup, so the + // whole path can be exercised on one machine with no rack behind it. + let override_address = match kind { + TargetKind::Vivado => args.vivado_agent.as_deref(), + TargetKind::Helios => args.helios_agent.as_deref(), + }; + if let Some(address) = override_address { + return Agent::at(&format!("http://{address}"), name, kind); + } + + let environment = + db::get_environment_status(UserEnvironmentPathParam { + user: user.to_owned(), + name: name.to_owned(), + }) + .map_err(|e| match e { + db::GetError::NoSuchEnvironment => { + RelayError::NoSuchEnvironment + } + other => RelayError::Db(other), + })?; + + let instance = match InstanceKind::from(kind) { + InstanceKind::Vivado => environment.vivado_instance, + InstanceKind::Helios => environment.helios_instance, + InstanceKind::Artifact => None, + } + .ok_or(RelayError::NoInstance { kind })?; + + // An instance that has been asked for but not yet built has a record + // with no address on it. That is worth saying plainly — it is the + // ordinary case in the first minute of an environment's life, not a + // failure. + let address = + instance.internal_ip.ok_or(RelayError::NoAddress { kind })?; + + Agent::at(&agent_url(address), name, kind) + } + + /// Find the instance that runs this environment's object store. + /// + /// Separate from [`Agent::resolve`] because the artifact instance is not + /// something source is ever synchronized to — it has no `TargetKind` — but + /// it is the one machine that knows where finished artifacts go. + pub(crate) fn resolve_artifact( + user: &str, + name: &str, + args: &crate::ServerArgs, + ) -> Result { + if let Some(address) = args.artifact_agent.as_deref() { + return Agent::at( + &format!("http://{address}"), + name, + TargetKind::Vivado, + ); + } + + let environment = + db::get_environment_status(UserEnvironmentPathParam { + user: user.to_owned(), + name: name.to_owned(), + }) + .map_err(|e| match e { + db::GetError::NoSuchEnvironment => { + RelayError::NoSuchEnvironment + } + other => RelayError::Db(other), + })?; + + let instance = + environment + .artifact_instance + .ok_or(RelayError::NoInstance { + kind: TargetKind::Vivado, + })?; + let address = instance.internal_ip.ok_or(RelayError::NoAddress { + kind: TargetKind::Vivado, + })?; + + Agent::at( + &agent_url(address), + name, + // Only used for error text; there is no artifact target kind and + // inventing one would put it in the public API for no reason. + TargetKind::Vivado, + ) + } + + /// Where this environment's artifacts go, and the key that opens it. + /// + /// The endpoint is filled in here rather than by the instance that minted + /// the key: that instance cannot know which of its addresses another one + /// can reach it on, and this service has both in the same record. + pub(crate) async fn object_store( + &self, + address: std::net::IpAddr, + kind: TargetKind, + ) -> Result { + let mut credentials = self + .client + .get_object_store(&self.environment, Some(&kind)) + .await + .map_err(|e| self.failed(e))? + .into_inner(); + + let port = credentials.port; + credentials.endpoint = match address { + std::net::IpAddr::V4(v4) => format!("http://{v4}:{port}"), + std::net::IpAddr::V6(v6) => format!("http://[{v6}]:{port}"), + }; + + Ok(credentials) + } + + /// Where this instance currently believes its artifacts go. + /// + /// An instance that has never been told answers that it has not, which is + /// the answer that matters at startup. + pub(crate) async fn artifact_target( + &self, + ) -> Result { + Ok(self + .client + .get_artifact_target(&self.environment) + .await + .map_err(|e| self.failed(e))? + .into_inner()) + } + + /// Tell this instance where the artifacts it builds should go. + pub(crate) async fn set_artifact_target( + &self, + credentials: &vw_api_types_versions::latest::S3Credentials, + ) -> Result<(), RelayError> { + self.client + .put_artifact_target(&self.environment, credentials) + .await + .map_err(|e| self.failed(e))?; + Ok(()) + } + + fn at( + base_url: &str, + environment: &str, + kind: TargetKind, + ) -> Result { + Ok(Agent { + client: vw_api_client::agent_client(base_url) + .map_err(RelayError::Client)?, + environment: environment.to_owned(), + kind, + }) + } + + /// Hand the instance the credentials a build fetches dependencies with. + /// + /// The caller's own token, passed straight through from the request that + /// carried it. This service keeps no copy and has no credentials of its + /// own to lend, which is the point: an instance can reach exactly what the + /// developer using it can reach. + /// + /// Does nothing when there is no token to pass on, which is the case under + /// `--no-auth`. A development service has no credentials to relay and + /// failing every sync over their absence would make that mode useless. + pub(crate) async fn give_credentials( + &self, + caller: &crate::auth::AuthorizedCaller, + log: &slog::Logger, + ) -> Result<(), RelayError> { + let Some(token) = caller.token.as_deref() else { + slog::debug!(log, "no credentials to relay"; + "environment" => &self.environment, + "kind" => %self.kind, + ); + return Ok(()); + }; + + self.client + .put_credentials( + &self.environment, + &vw_api_types_versions::latest::Credentials { + user: caller.name.clone(), + token: token.to_owned(), + }, + ) + .await + .map_err(|e| self.failed(e))?; + + Ok(()) + } + + /// Open a vivado session on the instance and join it to `client`. + /// + /// Frames are passed through untouched in both directions. This service + /// has already decided the only thing it is in a position to decide — + /// whether this caller owns this environment — and the conversation that + /// follows is between the developer's machine and the worker. Reading it + /// would buy nothing and add a place for it to be misunderstood. + pub(crate) async fn join_vivado_session( + &self, + client: dropshot::WebsocketConnection, + query: &vw_api_types_versions::latest::VivadoSessionQuery, + ) -> Result<(), RelayError> { + let upgraded = self + .client + .vivado_session( + &self.environment, + Some(query.info_with_stack), + query.part.as_deref(), + query.variant.as_deref(), + Some(query.verbose), + ) + .await + .map_err(|e| self.failed(e))? + .into_inner(); + + join(client, upgraded).await; + + Ok(()) + } + + /// Build the driver on the instance, joined to `client`. + pub(crate) async fn join_driver_build( + &self, + client: dropshot::WebsocketConnection, + query: &vw_api_types_versions::latest::DriverBuildQuery, + ) -> Result<(), RelayError> { + let upgraded = self + .client + .driver_build( + &self.environment, + query.args.as_deref(), + Some(query.release), + ) + .await + .map_err(|e| self.failed(e))? + .into_inner(); + + join(client, upgraded).await; + + Ok(()) + } + + /// Run this environment's testbenches on the instance, joined to `client`. + /// + /// Relayed the same way a vivado session is, and for the same reason: what + /// crosses is a conversation between the developer's machine and the + /// instance, and this service's only business with it was deciding whether + /// to allow it at all. + pub(crate) async fn join_bench_session( + &self, + client: dropshot::WebsocketConnection, + query: &vw_api_types_versions::latest::BenchQuery, + ) -> Result<(), RelayError> { + let upgraded = self + .client + .bench_session( + &self.environment, + query.concurrency, + query.filter.as_deref(), + query.ignore.as_deref(), + query.standard.as_deref(), + ) + .await + .map_err(|e| self.failed(e))? + .into_inner(); + + join(client, upgraded).await; + + Ok(()) + } + + /// Wrap an error from the agent so it says which instance failed. + pub(crate) fn failed( + &self, + source: vw_api_client::agent::Error, + ) -> RelayError { + RelayError::Agent { + kind: self.kind, + source: Box::new(source), + } + } +} + +/// Pass frames between a developer and an instance until one of them stops. +/// +/// Untouched in both directions. Reading them would buy nothing — the two ends +/// share a protocol this service has no part in — and would add a place for it +/// to be misunderstood. +/// +/// Either side ending ends the session. A developer who interrupted a build +/// wants it torn down rather than left running; an instance whose worker has +/// died has nothing more to say. +async fn join( + client: dropshot::WebsocketConnection, + instance: reqwest::Upgraded, +) { + use futures::{SinkExt, StreamExt}; + use tokio_tungstenite::tungstenite::protocol::Role; + use tokio_tungstenite::WebSocketStream; + + let instance = + WebSocketStream::from_raw_socket(instance, Role::Client, None).await; + let developer = WebSocketStream::from_raw_socket( + client.into_inner(), + Role::Server, + None, + ) + .await; + + let (mut to_instance, mut from_instance) = instance.split(); + let (mut to_developer, mut from_developer) = developer.split(); + + let outbound = async { + while let Some(Ok(frame)) = from_developer.next().await { + if to_instance.send(frame).await.is_err() { + break; + } + } + let _ = to_instance.close().await; + }; + let inbound = async { + while let Some(Ok(frame)) = from_instance.next().await { + if to_developer.send(frame).await.is_err() { + break; + } + } + let _ = to_developer.close().await; + }; + + tokio::pin!(outbound); + tokio::pin!(inbound); + futures::future::select(outbound, inbound).await; +} + +impl From for InstanceKind { + fn from(value: TargetKind) -> Self { + match value { + TargetKind::Vivado => InstanceKind::Vivado, + TargetKind::Helios => InstanceKind::Helios, + } + } +} diff --git a/vw-svc/src/tls.rs b/vw-svc/src/tls.rs new file mode 100644 index 0000000..ee6b1ac --- /dev/null +++ b/vw-svc/src/tls.rs @@ -0,0 +1,350 @@ +//! TLS configuration for the API servers, and keeping it current. +//! +//! In production the certificate comes from Let's Encrypt, which means it is +//! replaced every couple of months by a `certbot renew` that runs from a timer +//! with nobody watching. Restarting to pick one up would be the easy answer +//! and the wrong one: this service relays the connections a build runs over, +//! so a restart ends somebody's synthesis run, REPL session or download partway +//! through, for a certificate that had two weeks left on it. +//! +//! So the certificate files are watched instead, and the running servers are +//! handed the replacement between handshakes. Nothing on either side of an +//! established connection notices, and there is nothing to run but certbot. + +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use camino::{Utf8Path, Utf8PathBuf}; +use dropshot::{ConfigTls, HttpServer, ServerContext}; +use slog::{error, info, warn, Logger}; +use slog_error_chain::InlineErrorChain; +use tokio::sync::watch; + +use crate::ServerArgs; + +/// How often the certificate files are checked for replacement. +/// +/// A renewal happens twice a year and nothing is waiting on it, so this is set +/// by what it costs rather than by how soon it must be noticed: two `stat` +/// calls, and a parse only when they say something changed. +const POLL: Duration = Duration::from_secs(60); + +/// Error conditions for assembling the TLS configuration. +#[derive(Debug, thiserror::Error)] +pub(crate) enum TlsError { + #[error("certificate file {0} does not exist")] + NoCertFile(Utf8PathBuf), + #[error("key file {0} does not exist")] + NoKeyFile(Utf8PathBuf), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), + #[error("parsing {0}")] + Parse(Utf8PathBuf, #[source] std::io::Error), + #[error("{0} contains no certificate")] + NoCertificates(Utf8PathBuf), + #[error("{0} contains no private key")] + NoPrivateKey(Utf8PathBuf), + #[error( + "{cert_file} and {key_file} are not a certificate and key this can serve" + )] + Unusable { + cert_file: Utf8PathBuf, + key_file: Utf8PathBuf, + #[source] + source: rustls::Error, + }, +} + +/// The TLS the API servers run with, and a channel carrying its replacements. +/// +/// Cloned per server: both serve the same certificate, and each holds its own +/// acceptor that has to be told about a new one separately. +#[derive(Clone)] +pub(crate) struct Tls { + /// What a server starts with. + initial: ConfigTls, + /// Certificates that replaced it, as they are noticed. + updates: watch::Receiver, +} + +/// The TLS configuration both API servers should run with, or `None` when the +/// service was not asked to serve HTTPS. +/// +/// The certificate and key are read and parsed here rather than left to +/// dropshot, so that anything wrong with them is a startup failure naming the +/// path at fault instead of turning up later as a handshake failure. Starting +/// the watch here too means the load that runs on every renewal is this one. +pub(crate) fn config( + args: &ServerArgs, + log: &Logger, +) -> Result, TlsError> { + if !args.tls { + return Ok(None); + } + if !args.cert_file.exists() { + return Err(TlsError::NoCertFile(args.cert_file.clone())); + } + if !args.key_file.exists() { + return Err(TlsError::NoKeyFile(args.key_file.clone())); + } + + let initial = load(&args.cert_file, &args.key_file)?; + + let (tx, updates) = watch::channel(initial.clone()); + tokio::spawn(watch_for_renewals( + args.cert_file.clone(), + args.key_file.clone(), + tx, + log.new(slog::o!("component" => "tls")), + )); + + Ok(Some(Tls { initial, updates })) +} + +/// Read the certificate and key, and build what a server can be handed. +/// +/// [`ConfigTls::Dynamic`] rather than [`ConfigTls::AsFile`] on purpose. Given +/// paths, a server reads and parses them itself, and on the refresh path +/// dropshot does that behind an `unwrap`. Given an already-built configuration +/// there is nothing left for it to fail at. +/// +/// The parsing is done here rather than by handing dropshot the bytes for the +/// same reason: its conversion returns an error for a file it cannot read, but +/// *panics* on one it can read and cannot use — an empty certificate, or a key +/// belonging to a different certificate. Both are ordinary things to see for a +/// moment during a renewal. A panic in the task that watches for renewals does +/// not stop the service; it stops the watching, silently and for good, which +/// is worse. +/// +/// The result is otherwise exactly what dropshot builds, ALPN included, so a +/// renewed certificate changes nothing about how connections are negotiated. +fn load( + cert_file: &Utf8Path, + key_file: &Utf8Path, +) -> Result { + let cert_pem = std::fs::read(cert_file) + .map_err(|e| TlsError::Read(cert_file.to_owned(), e))?; + let key_pem = std::fs::read(key_file) + .map_err(|e| TlsError::Read(key_file.to_owned(), e))?; + + let certs = rustls_pemfile::certs(&mut cert_pem.as_slice()) + .collect::, _>>() + .map_err(|e| TlsError::Parse(cert_file.to_owned(), e))?; + if certs.is_empty() { + return Err(TlsError::NoCertificates(cert_file.to_owned())); + } + + // Any of the three encodings a private key comes in, where dropshot takes + // only PKCS#8. Certbot writes PKCS#8, so this is about not failing + // mysteriously on a key that came from somewhere else. + let key = rustls_pemfile::private_key(&mut key_pem.as_slice()) + .map_err(|e| TlsError::Parse(key_file.to_owned(), e))? + .ok_or_else(|| TlsError::NoPrivateKey(key_file.to_owned()))?; + + // Checks that the key belongs to the certificate, which is what catches a + // renewal read halfway through. + let mut raw = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|source| TlsError::Unusable { + cert_file: cert_file.to_owned(), + key_file: key_file.to_owned(), + source, + })?; + raw.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + + Ok(ConfigTls::Dynamic(raw)) +} + +/// Notice when certbot has replaced the certificate, and pass it on. +/// +/// Polled rather than watched for filesystem events because of the shape of +/// the thing being watched. Certbot keeps every certificate it has issued in +/// `archive/` and points a symlink in `live/` at the current one, so the +/// configured path is a symlink that is never itself modified — it is +/// replaced, and the writes land in a directory nobody configured. A `stat` a +/// minute is immune to all of that, and a renewal noticed a minute late is a +/// renewal noticed on time. +async fn watch_for_renewals( + cert_file: Utf8PathBuf, + key_file: Utf8PathBuf, + tx: watch::Sender, + log: Logger, +) { + let mut current = stamp(&cert_file, &key_file); + + loop { + tokio::time::sleep(POLL).await; + + let latest = stamp(&cert_file, &key_file); + if latest == current { + continue; + } + + match load(&cert_file, &key_file) { + Ok(config) => { + info!(log, "certificate replaced"; + "cert_file" => %cert_file, + "key_file" => %key_file, + ); + current = latest; + // Fails only once every server has gone, by which point the + // service is on its way down and has no use for this. + let _ = tx.send(config); + } + Err(e) => { + // `current` is deliberately left behind, so the next pass sees + // the files as still changed and tries again. The likeliest + // cause is catching a renewal mid-flight, with the certificate + // replaced and the key not yet — which resolves itself within + // the minute. + warn!( + log, + "cannot serve the replaced certificate, keeping the \ + current one"; + InlineErrorChain::new(&e), + ); + } + } + } +} + +/// What is compared to decide the files have been replaced. +/// +/// Size and modification time of each, followed through the symlinks, which is +/// all that is needed: a renewal writes a different certificate at a later +/// time. Hashing the contents would be more precise about a case that does not +/// arise, and this is only ever the decision to look closer. +/// +/// A file that cannot be stat'd reads as `None` rather than an error. A +/// certificate that has briefly gone missing is not a reason to bring the +/// service down, and it compares unequal when it comes back. +fn stamp( + cert_file: &Utf8Path, + key_file: &Utf8Path, +) -> [Option<(u64, SystemTime)>; 2] { + [cert_file, key_file].map(|path| { + let meta = std::fs::metadata(path).ok()?; + Some((meta.len(), meta.modified().ok()?)) + }) +} + +/// The configuration a server should start with. +pub(crate) fn initial(tls: Option<&Tls>) -> Option { + tls.map(|tls| tls.initial.clone()) +} + +/// Hand `server` every certificate that replaces the one it started with. +/// +/// Returns immediately; the following is a task that lives as long as the +/// service does. Does nothing when there is no TLS, so that a caller does not +/// have to ask twice whether there is any. +pub(crate) fn follow_renewals( + server: Arc>, + tls: Option, + log: Logger, +) { + let Some(Tls { mut updates, .. }) = tls else { + return; + }; + + tokio::spawn(async move { + // Ends when the watch is dropped, which is when the service is done. + while updates.changed().await.is_ok() { + let config = updates.borrow_and_update().clone(); + match server.refresh_tls(&config).await { + // Connections already up are untouched and keep the old + // certificate for as long as they live; everything from here + // on gets the new one. + Ok(()) => info!(log, "now serving the replaced certificate"), + // Only reachable on a server built without TLS, and nothing + // subscribes without one. + Err(e) => { + error!(log, "cannot install the replaced certificate"; + "error" => e, + ) + } + } + } + }); +} + +/// The URL scheme the servers answer on, for log messages. +pub(crate) fn scheme(args: &ServerArgs) -> &'static str { + if args.tls { + "https" + } else { + "http" + } +} + +#[cfg(test)] +mod test { + use super::*; + + /// Every way a certificate can be unusable, written where `load` will find + /// it. + fn write(contents: &[u8]) -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8").to_owned(); + let path = root.join("pem"); + std::fs::write(&path, contents).expect("write"); + (dir, path) + } + + #[test] + fn an_unusable_certificate_is_an_error_and_not_a_panic() { + // This is load-bearing rather than tidy. `load` runs on a timer in a + // task nobody is watching, and a panic there does not stop the + // service — it stops renewals, silently, until somebody restarts. Each + // of these is a state a certificate directory really passes through + // while certbot is writing to it. + for (what, contents) in [ + ("an empty file", &b""[..]), + ("a file that is not PEM at all", b"not a certificate"), + ( + "a PEM header and nothing else", + b"-----BEGIN CERTIFICATE-----", + ), + ( + "a truncated certificate", + b"-----BEGIN CERTIFICATE-----\nMIIB\n", + ), + ] { + let (_dir, path) = write(contents); + + let loaded = load(&path, &path); + + assert!(loaded.is_err(), "{what} should be refused, not accepted"); + } + } + + #[test] + fn a_certificate_that_is_not_there_is_an_error() { + // The window while certbot has unlinked one symlink and not yet + // written the next. + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8"); + let missing = root.join("nothing.pem"); + + assert!(load(&missing, &missing).is_err()); + } + + #[test] + fn nothing_new_leaves_the_stamp_alone() { + // The whole poll rests on this: an untouched file must compare equal, + // or every pass would reload a certificate that had not changed. + let (_dir, path) = write(b"whatever"); + + assert_eq!(stamp(&path, &path), stamp(&path, &path)); + } + + #[test] + fn a_missing_file_stamps_rather_than_fails() { + let dir = tempfile::TempDir::new().expect("scratch"); + let root = Utf8Path::from_path(dir.path()).expect("utf8"); + let missing = root.join("nothing.pem"); + + assert_eq!(stamp(&missing, &missing), [None, None]); + } +} diff --git a/vw-svc/src/user_api.rs b/vw-svc/src/user_api.rs new file mode 100644 index 0000000..6f60c67 --- /dev/null +++ b/vw-svc/src/user_api.rs @@ -0,0 +1,916 @@ +//! This module implements the user api trait `[vw_api::VwUserApi]` +use crate::{ + auth, db, keys, oxide, + reconciler::{validate_environment_name, validate_user_name}, + relay, +}; +use dropshot::{ApiDescription, BuildError, ConfigDropshot}; +use slog::{error, info, o}; +use slog_error_chain::InlineErrorChain; +use std::{net::SocketAddr, sync::Arc}; +use tokio::sync::Notify; +use vw_api::VwUserApi; +use vw_api_types_versions::v1::UserEnvironmentPathParam; + +use crate::{Context, ServerArgs}; + +pub struct UserApi {} +impl VwUserApi for UserApi { + type Context = Arc; + + async fn get_environments( + rqctx: dropshot::RequestContext, + ) -> Result< + dropshot::HttpResponseOk< + dropshot::ResultsPage, + >, + dropshot::HttpError, + > { + let caller = auth::authorize_caller(rqctx).await?; + let environments = db::list_user_environments(&caller.name)?; + // This endpoint takes no pagination parameters, so the caller's + // environments are always returned as one complete page. If a limit + // and page selector are ever added to the endpoint, this becomes a + // `ResultsPage::new` call with a selector keyed on environment name. + Ok(dropshot::HttpResponseOk(dropshot::ResultsPage { + next_page: None, + items: environments, + })) + } + + async fn create_environment( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + body: dropshot::TypedBody< + vw_api_types_versions::latest::EnvironmentCreate, + >, + ) -> Result< + dropshot::HttpResponseCreated< + vw_api_types_versions::latest::SshKeyPair, + >, + dropshot::HttpError, + > { + let reconcile = rqctx.context().reconcile.clone(); + let log = rqctx.log.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + let requested = body.into_inner(); + + // Both halves become part of an Oxide instance name, so both have to + // survive being parsed back out of one. + validate_environment_name(&name).map_err(|e| { + info!(log, "rejecting environment name"; "name" => &name); + dropshot::HttpError::for_bad_request(None, e) + })?; + validate_user_name(&caller.name).map_err(|e| { + info!(log, "rejecting caller name"; "user" => &caller.name); + dropshot::HttpError::for_bad_request(None, e) + })?; + + // Pin the images now rather than at reconcile time, so publishing a + // newer image never changes what an existing environment boots. This + // is also where an explicitly named image gets validated. + let images = if oxide::is_configured() { + let session = oxide::session()?; + Some( + session + .resolve_images( + requested.vivado_image.as_deref(), + requested.helios_image.as_deref(), + requested.artifact_image.as_deref(), + ) + .await + .inspect_err(|e| { + info!(log, "cannot resolve environment images"; + "environment" => &name, + "error" => %e, + ); + })?, + ) + } else { + // No rack to resolve against, so the environment is a bare + // record. Accepting an image the service can neither validate nor + // ever use would look like it took effect, so say so instead. + if requested.vivado_image.is_some() + || requested.helios_image.is_some() + || requested.artifact_image.is_some() + { + info!(log, "image requested with no oxide backend"; + "environment" => &name, + ); + return Err(dropshot::HttpError::for_bad_request( + None, + String::from( + "this service has no oxide backend configured, so it \ + cannot resolve or honor an image", + ), + )); + } + None + }; + + let key = UserEnvironmentPathParam { + user: caller.name.clone(), + name, + }; + // Every environment gets its own keypair, generated here and kept + // alongside it. Without one the instances come up with no way in. + let ssh_key = + keys::generate(&key.user, &key.name).inspect_err(|e| { + error!(log, "cannot generate an ssh key"; + "user" => &key.user, + "environment" => &key.name, + InlineErrorChain::new(e), + ); + })?; + + db::create_environment(key, images, &ssh_key)?; + + // Provision it now rather than on the next tick. + reconcile.notify_one(); + + // Handed back so the caller can save it straight away; the same pair + // stays available from the keys endpoint. + Ok(dropshot::HttpResponseCreated(ssh_key)) + } + + async fn sync_plan( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::TargetPathParam, + >, + body: dropshot::TypedBody, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let target = path_params.into_inner(); + let manifest = body.into_inner(); + + let agent = relay::Agent::resolve( + &caller.name, + &target.name, + target.kind, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + // Every sync begins here, which makes this the place to hand the + // instance the credentials its build will fetch dependencies with. + // Sent each time rather than once: an instance rebuilt underneath us + // comes back with none, and the failure that causes shows up much + // later as a build that cannot reach a private repository. + agent + .give_credentials(&caller, &log) + .await + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + // And tell it where the artifacts it builds should go, if it has not + // been told since this service last restarted. Only vivado builds + // images, and only the artifact instance knows the key — this service + // is the one place that can see both. + crate::wiring::ensure( + &caller.name, + &target.name, + target.kind, + &args, + &agent, + &log, + ) + .await; + + let plan = agent + .client + .sync_plan(&agent.environment, &manifest) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + Ok(dropshot::HttpResponseOk(plan)) + } + + async fn sync_blob( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::TargetBlobPathParam, + >, + body: dropshot::UntypedBody, + ) -> Result + { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let params = path_params.into_inner(); + let target = vw_api_types_versions::latest::TargetPathParam { + name: params.name.clone(), + kind: params.kind, + }; + + let agent = relay::Agent::resolve( + &caller.name, + ¶ms.name, + params.kind, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + agent + .client + .sync_blob( + &agent.environment, + params.digest.0.as_str(), + body.as_bytes().to_vec(), + ) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + Ok(dropshot::HttpResponseUpdatedNoContent()) + } + + async fn sync_commit( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::TargetPathParam, + >, + body: dropshot::TypedBody, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let target = path_params.into_inner(); + let manifest = body.into_inner(); + + let agent = relay::Agent::resolve( + &caller.name, + &target.name, + target.kind, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + let result = agent + .client + .sync_commit(&agent.environment, &manifest) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + info!(log, "relayed a source sync"; + "environment" => &target.name, + "target" => %target.kind, + "created" => result.created, + "updated" => result.updated, + "deleted" => result.deleted, + "unchanged" => result.unchanged, + ); + + Ok(dropshot::HttpResponseOk(result)) + } + + async fn sync_clear( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::TargetPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let target = path_params.into_inner(); + + let agent = relay::Agent::resolve( + &caller.name, + &target.name, + target.kind, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + let result = agent + .client + .sync_clear(&agent.environment) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + info!(log, "cleared a source tree"; + "environment" => &target.name, + "target" => %target.kind, + "deleted" => result.deleted, + ); + + Ok(dropshot::HttpResponseOk(result)) + } + + async fn clean_build_output( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::TargetPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let target = path_params.into_inner(); + + let agent = relay::Agent::resolve( + &caller.name, + &target.name, + target.kind, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + let cleaned = agent + .client + .clean_build_output(&agent.environment) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + info!(log, "removed build output"; + "environment" => &target.name, + "target" => %target.kind, + "bytes" => cleaned.bytes, + ); + + Ok(dropshot::HttpResponseOk(cleaned)) + } + + async fn driver_build( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + query: dropshot::Query, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + let query = query.into_inner(); + + // Helios, not vivado: the driver's target is native there and its + // pinned toolchain is installed there. + let target = vw_api_types_versions::latest::TargetPathParam { + name: name.clone(), + kind: vw_api_types_versions::latest::TargetKind::Helios, + }; + let agent = relay::Agent::resolve( + &caller.name, + &name, + vw_api_types_versions::latest::TargetKind::Helios, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + // A build fetches the driver's dependencies from github like any + // other, so the instance needs the caller's credentials first. + agent + .give_credentials(&caller, &log) + .await + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + info!(log, "building the driver"; + "environment" => &name, + "user" => &caller.name, + "release" => query.release, + ); + + let result = agent.join_driver_build(websock, &query).await; + + match &result { + Ok(()) => info!(log, "driver build ended"; + "environment" => &name, + ), + Err(e) => log_relay_failure(&log, &target, e), + } + + result.map_err(Into::into) + } + + async fn bench_session( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + query: dropshot::Query, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + let query = query.into_inner(); + + let target = vw_api_types_versions::latest::TargetPathParam { + name: name.clone(), + kind: vw_api_types_versions::latest::TargetKind::Vivado, + }; + + let agent = relay::Agent::resolve( + &caller.name, + &name, + vw_api_types_versions::latest::TargetKind::Vivado, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + // A bench build fetches the workspace's dependencies like any other, + // so the instance needs the caller's credentials before it starts. + agent + .give_credentials(&caller, &log) + .await + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + info!(log, "running testbenches"; + "environment" => &name, + "user" => &caller.name, + "filter" => query.filter.as_deref().unwrap_or("-"), + ); + + let result = agent.join_bench_session(websock, &query).await; + + match &result { + Ok(()) => info!(log, "testbench run ended"; + "environment" => &name, + ), + Err(e) => log_relay_failure(&log, &target, e), + } + + result.map_err(Into::into) + } + + async fn vivado_session( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + query: dropshot::Query< + vw_api_types_versions::latest::VivadoSessionQuery, + >, + websock: dropshot::WebsocketConnection, + ) -> dropshot::WebsocketChannelResult { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + let query = query.into_inner(); + + let target = vw_api_types_versions::latest::TargetPathParam { + name: name.clone(), + kind: vw_api_types_versions::latest::TargetKind::Vivado, + }; + + let agent = relay::Agent::resolve( + &caller.name, + &name, + vw_api_types_versions::latest::TargetKind::Vivado, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + // The worker will want to fetch this build's dependencies, and the + // credentials for that are the caller's. Sent before the session + // opens, because once it does this service is only moving frames. + agent + .give_credentials(&caller, &log) + .await + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + info!(log, "joining a vivado session"; + "environment" => &name, + "user" => &caller.name, + "variant" => query.variant.as_deref().unwrap_or("-"), + ); + + let result = agent.join_vivado_session(websock, &query).await; + + match &result { + Ok(()) => info!(log, "vivado session ended"; + "environment" => &name, + ), + Err(e) => log_relay_failure(&log, &target, e), + } + + result.map_err(Into::into) + } + + async fn generated_manifest( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + + let target = vw_api_types_versions::latest::TargetPathParam { + name: name.clone(), + kind: vw_api_types_versions::latest::TargetKind::Vivado, + }; + let agent = relay::Agent::resolve( + &caller.name, + &name, + vw_api_types_versions::latest::TargetKind::Vivado, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + let manifest = agent + .client + .generated_manifest(&agent.environment) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + info!(log, "reported generated ip"; + "environment" => &name, + "files" => manifest.entries.len(), + ); + + Ok(dropshot::HttpResponseOk(manifest)) + } + + async fn generated_file( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + query: dropshot::Query< + vw_api_types_versions::latest::GeneratedFileQuery, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + let wanted = query.into_inner().path; + + let target = vw_api_types_versions::latest::TargetPathParam { + name: name.clone(), + kind: vw_api_types_versions::latest::TargetKind::Vivado, + }; + let agent = relay::Agent::resolve( + &caller.name, + &name, + vw_api_types_versions::latest::TargetKind::Vivado, + &args, + ) + .inspect_err(|e| log_relay_failure(&log, &target, e))?; + + let contents = agent + .client + .generated_file(&agent.environment, &wanted) + .await + .map_err(|e| agent.failed(e)) + .inspect_err(|e| log_relay_failure(&log, &target, e))? + .into_inner(); + + // Small text files, read whole rather than streamed: a wrapper is a + // few kilobytes and there is nothing to gain from frames. + let bytes = futures::TryStreamExt::try_fold( + contents.into_inner(), + Vec::new(), + |mut collected, chunk| async move { + collected.extend_from_slice(&chunk); + Ok(collected) + }, + ) + .await + .map_err(|e| { + dropshot::HttpError::for_internal_error(format!( + "reading a generated file from the instance: {e}" + )) + })?; + + Ok(dropshot::HttpResponseOk( + dropshot::Body::with_content(bytes).into(), + )) + } + + async fn get_artifacts( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk>, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + + let mut found = Vec::new(); + // Every kind that has a bucket, so one listing answers "what did this + // environment build" rather than "what did one half of it build". + for kind in [ + vw_api_types_versions::latest::TargetKind::Vivado, + vw_api_types_versions::latest::TargetKind::Helios, + ] { + let credentials = match crate::wiring::store_for( + &caller.name, + &name, + kind, + &args, + ) + .await + { + Ok(credentials) => credentials, + Err(e) => { + error!(log, "cannot reach the object store"; + "environment" => &name, + "kind" => %kind, + InlineErrorChain::new(&e), + ); + return Err(e.into()); + } + }; + + match crate::artifacts::list(&credentials, kind).await { + Ok(mut artifacts) => found.append(&mut artifacts), + Err(e) => { + error!(log, "cannot list artifacts"; + "environment" => &name, + "kind" => %kind, + InlineErrorChain::new(&e), + ); + return Err(e.into()); + } + } + } + + found.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(dropshot::HttpResponseOk(found)) + } + + async fn clear_artifacts( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk< + vw_api_types_versions::latest::ArtifactsCleared, + >, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let name = path_params.into_inner().name; + + let mut cleared = + vw_api_types_versions::latest::ArtifactsCleared::default(); + + for kind in [ + vw_api_types_versions::latest::TargetKind::Vivado, + vw_api_types_versions::latest::TargetKind::Helios, + ] { + let credentials = + crate::wiring::store_for(&caller.name, &name, kind, &args) + .await + .inspect_err(|e| { + error!(log, "cannot reach the object store"; + "environment" => &name, + "kind" => %kind, + InlineErrorChain::new(e), + ); + })?; + + let (removed, bytes) = crate::artifacts::clear(&credentials) + .await + .inspect_err(|e| { + error!(log, "cannot clear artifacts"; + "environment" => &name, + "kind" => %kind, + InlineErrorChain::new(e), + ); + })?; + + cleared.removed += removed; + cleared.bytes += bytes; + } + + // Said plainly and after the fact: this is not recoverable, and the + // record of who emptied what is the only thing left of it. + info!(log, "cleared an environment's artifacts"; + "environment" => &name, + "user" => &caller.name, + "removed" => cleared.removed, + "bytes" => cleared.bytes, + ); + + Ok(dropshot::HttpResponseOk(cleared)) + } + + async fn get_artifact( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::ArtifactPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let log = rqctx.log.clone(); + let args = rqctx.context().server_args.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let wanted = path_params.into_inner(); + + let credentials = crate::wiring::store_for( + &caller.name, + &wanted.name, + wanted.kind, + &args, + ) + .await + .inspect_err(|e| { + error!(log, "cannot reach the object store"; + "environment" => &wanted.name, + InlineErrorChain::new(e), + ); + })?; + + let stream = crate::artifacts::fetch(&credentials, &wanted.artifact) + .await + .inspect_err(|e| { + error!(log, "cannot fetch an artifact"; + "environment" => &wanted.name, + "artifact" => &wanted.artifact, + InlineErrorChain::new(e), + ); + })?; + + info!(log, "streaming an artifact"; + "environment" => &wanted.name, + "kind" => %wanted.kind, + "artifact" => &wanted.artifact, + ); + + // Passed through frame by frame as the store produces them, so an + // image of any size costs this service no more memory than a small one. + let body = dropshot::Body::wrap(http_body_util::StreamBody::new( + futures::TryStreamExt::map_ok(stream, http_body::Frame::data), + )); + + Ok(dropshot::HttpResponseOk(body.into())) + } + + async fn get_environment_keys( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + // Scoped to the caller like every other endpoint here, so the private + // key only ever goes back to the environment's owner. + let caller = auth::authorize_caller(rqctx).await?; + let key = UserEnvironmentPathParam { + user: caller.name.clone(), + name: path_params.into_inner().name, + }; + Ok(dropshot::HttpResponseOk(db::get_environment_keys(key)?)) + } + + async fn get_environment( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result< + dropshot::HttpResponseOk, + dropshot::HttpError, + > { + let caller = auth::authorize_caller(rqctx).await?; + let key = UserEnvironmentPathParam { + user: caller.name.clone(), + name: path_params.into_inner().name, + }; + let env = db::get_environment_status(key)?; + Ok(dropshot::HttpResponseOk(env)) + } + + async fn delete_environment( + rqctx: dropshot::RequestContext, + path_params: dropshot::Path< + vw_api_types_versions::latest::EnvironmentPathParam, + >, + ) -> Result { + let reconcile = rqctx.context().reconcile.clone(); + let caller = auth::authorize_caller(rqctx).await?; + let key = UserEnvironmentPathParam { + user: caller.name.clone(), + name: path_params.into_inner().name, + }; + db::delete_environment(key)?; + + // Tear the instances down now rather than on the next tick. + reconcile.notify_one(); + + Ok(dropshot::HttpResponseDeleted()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum StartServerError { + #[error("Server build error {0}")] + ServerBuildError(#[from] BuildError), + #[error("Unexpected server exit {0}")] + ServerExit(String), +} + +pub async fn start_server( + server_args: ServerArgs, + log: slog::Logger, + bind_address: SocketAddr, + tls: Option, + reconcile: Arc, +) -> Result<(), StartServerError> { + let scheme = crate::tls::scheme(&server_args); + let context = Arc::new(Context { + server_args, + reconcile, + }); + let cfg = ConfigDropshot { + bind_address, + default_request_body_max_bytes: usize::MAX, + ..Default::default() + }; + let lg = log.new(o!("component" => "user_api")); + let api = api_description(); + + // Shared rather than owned so that the certificate can be replaced under a + // server that is already running. Dropping the last handle shuts the + // server down, so this one outlives the follower task below. + let server = Arc::new( + dropshot::ServerBuilder::new(api, context, lg.clone()) + .config(cfg) + .tls(crate::tls::initial(tls.as_ref())) + .start()?, + ); + + info!(lg, "listening on {scheme}://{}", server.local_addr()); + + crate::tls::follow_renewals(server.clone(), tls, lg.clone()); + + server + .wait_for_shutdown() + .await + .map_err(StartServerError::ServerExit) +} + +pub fn api_description() -> ApiDescription> { + vw_api::vw_user_api_mod::api_description::().unwrap() +} + +/// Say why a sync could not be passed on. +/// +/// Every one of these is worth a line: an instance that is not up yet, an +/// address that has not appeared, an agent that refused. None of it is visible +/// from the response alone, which only says the sync did not happen. +fn log_relay_failure( + log: &slog::Logger, + target: &vw_api_types_versions::latest::TargetPathParam, + error: &relay::RelayError, +) { + slog::warn!(log, "cannot relay a source sync"; + "environment" => &target.name, + "target" => %target.kind, + InlineErrorChain::new(error), + ); +} diff --git a/vw-svc/src/wiring.rs b/vw-svc/src/wiring.rs new file mode 100644 index 0000000..037f3ed --- /dev/null +++ b/vw-svc/src/wiring.rs @@ -0,0 +1,185 @@ +//! Telling each instance where the artifacts it builds should go. +//! +//! Only this service can do it. The instance that holds the object store mints +//! the key but cannot know which address its neighbours reach it on; the +//! instances that fill the store cannot know which of their neighbours holds +//! it. This service sees both, and its whole job here is to introduce them. +//! +//! Done in two places, for two different reasons. On startup, so an +//! environment created while its instances were still coming up — or one whose +//! store has been rebuilt since — is put right without anyone having to do +//! anything. And on each first sync, so an environment created after startup +//! does not have to wait for the next one. + +use slog::{error, info, Logger}; +use vw_api_types_versions::latest::TargetKind; + +use crate::{db, relay, ServerArgs}; + +/// The key and bucket for one of an environment's artifact stores. +/// +/// Used both to configure an instance and to read the store back out on a +/// developer's behalf, which is the same question asked by two callers. +pub(crate) async fn store_for( + user: &str, + name: &str, + kind: TargetKind, + args: &ServerArgs, +) -> Result< + vw_api_types_versions::latest::S3Credentials, + crate::artifacts::ArtifactError, +> { + let artifact = relay::Agent::resolve_artifact(user, name, args) + .map_err(|_| crate::artifacts::ArtifactError::NoStore)?; + let address = artifact_address(user, name, args) + .ok_or(crate::artifacts::ArtifactError::NoStore)?; + + artifact + .object_store(address, kind) + .await + .map_err(|_| crate::artifacts::ArtifactError::NoStore) +} + +/// The kinds of instance that build something worth keeping. +const KINDS: [TargetKind; 2] = [TargetKind::Vivado, TargetKind::Helios]; + +/// Make sure one instance knows where its artifacts go. +/// +/// Cheap when there is nothing to do: the instance is asked what it currently +/// believes, and only told again if that differs from the truth. Which means a +/// store rebuilt with a new key is noticed and corrected, rather than left +/// pointing somewhere that no longer accepts it. +pub(crate) async fn ensure( + user: &str, + name: &str, + kind: TargetKind, + args: &ServerArgs, + instance: &relay::Agent, + log: &Logger, +) { + let artifact = match relay::Agent::resolve_artifact(user, name, args) { + Ok(artifact) => artifact, + Err(e) => { + info!(log, "no artifact instance to store artifacts on yet"; + "environment" => name, + "detail" => %e, + ); + return; + } + }; + + let Some(address) = artifact_address(user, name, args) else { + return; + }; + + let wanted = match artifact.object_store(address, kind).await { + Ok(wanted) => wanted, + Err(e) => { + info!(log, "cannot read the environment's object store yet"; + "environment" => name, + "detail" => %e, + ); + return; + } + }; + + // Already right, so say nothing and do nothing. This is the ordinary case + // on every restart after the first. + if let Ok(current) = instance.artifact_target().await { + if current.endpoint == wanted.endpoint + && current.bucket == wanted.bucket + && current.access_key_id == wanted.access_key_id + { + return; + } + } + + if let Err(e) = instance.set_artifact_target(&wanted).await { + error!(log, "cannot tell an instance where artifacts go"; + "environment" => name, + "kind" => %kind, + slog_error_chain::InlineErrorChain::new(&e), + ); + return; + } + + info!(log, "artifacts wired up"; + "environment" => name, + "kind" => %kind, + "bucket" => &wanted.bucket, + ); +} + +/// Put every environment right, once. +/// +/// Run at startup. An environment whose instances were still coming up when it +/// was created has nowhere to put artifacts until someone tells it, and until +/// now the only thing that ever told it was a source synchronization — so an +/// environment nobody had synced since the service last started would build +/// images that went nowhere. +/// +/// Failures are per environment and only logged. One environment whose +/// instances are down should not stop the rest being configured, and the sync +/// path will catch it later regardless. +pub(crate) async fn ensure_all(args: &ServerArgs, log: &Logger) { + let environments = match db::list_all_environments() { + Ok(environments) => environments, + Err(e) => { + error!(log, "cannot list environments to configure artifacts for"; + slog_error_chain::InlineErrorChain::new(&e), + ); + return; + } + }; + + if environments.is_empty() { + return; + } + + info!(log, "checking that every environment can store artifacts"; + "environments" => environments.len(), + ); + + for entry in environments { + let (user, name) = (&entry.user, &entry.environment.name); + for kind in KINDS { + let instance = match relay::Agent::resolve(user, name, kind, args) { + Ok(instance) => instance, + Err(e) => { + // Ordinary while an environment is still being built. + info!(log, "instance not ready to be configured"; + "environment" => name, + "kind" => %kind, + "detail" => %e, + ); + continue; + } + }; + + ensure(user, name, kind, args, &instance, log).await; + } + } +} + +/// The artifact instance's address on the rack's network. +pub(crate) fn artifact_address( + user: &str, + name: &str, + args: &ServerArgs, +) -> Option { + // A development override is an address the other instances can reach too, + // since with no rack behind this everything is on one machine. + if let Some(address) = args.artifact_agent.as_deref() { + return address.split(':').next().and_then(|host| host.parse().ok()); + } + + db::get_environment_status( + vw_api_types_versions::latest::UserEnvironmentPathParam { + user: user.to_owned(), + name: name.to_owned(), + }, + ) + .ok() + .and_then(|environment| environment.artifact_instance) + .and_then(|instance| instance.internal_ip) +} diff --git a/vw-svc/tests/admin_api.rs b/vw-svc/tests/admin_api.rs new file mode 100644 index 0000000..1debe7b --- /dev/null +++ b/vw-svc/tests/admin_api.rs @@ -0,0 +1,270 @@ +// The admin API, which exists to do the two things the user API deliberately +// cannot: see every environment on the rack, and delete one that belongs to +// somebody else. +// +// That reach is the whole point and also the whole risk, so most of what is +// checked here is who is allowed to use it. As in `user_api.rs`, the service +// runs with authorization disabled and takes the `x-vw-user` header at face +// value — which is exactly what makes it possible to arrive as somebody who is +// not an administrator and confirm the door is shut. + +use std::net::TcpListener; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use reqwest::StatusCode; +use tempfile::TempDir; + +const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); + +/// The administrator the service is started with. +const ADMIN: &str = "picard"; + +/// Somebody who uses the service but does not run it. +const DEVELOPER: &str = "barclay"; + +/// A running service, reachable on both of its APIs. +struct TestServer { + child: Child, + users: String, + admin: String, + client: reqwest::Client, +} + +impl TestServer { + async fn start(db_path: &Path) -> TestServer { + let user_port = free_port(); + let admin_port = free_port(); + let child = Command::new(env!("CARGO_BIN_EXE_vw-svc")) + .arg("serve") + .args(["--address", "127.0.0.1"]) + .args(["--user-api-port", &user_port.to_string()]) + .args(["--admin-api-port", &admin_port.to_string()]) + .args(["--db-path", db_path.to_str().expect("utf8 database path")]) + .args(["--admin-users", ADMIN]) + .arg("--no-auth") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn vw-svc"); + + let mut server = TestServer { + child, + users: format!("http://127.0.0.1:{user_port}"), + admin: format!("http://127.0.0.1:{admin_port}"), + client: reqwest::Client::new(), + }; + server.wait_until_ready().await; + server + } + + async fn wait_until_ready(&mut self) { + let deadline = Instant::now() + STARTUP_TIMEOUT; + loop { + if self + .environments(ADMIN) + .await + .is_ok_and(|response| response.status() == StatusCode::OK) + { + return; + } + if let Some(status) = + self.child.try_wait().expect("check on vw-svc") + { + panic!("vw-svc exited during startup: {status}"); + } + assert!( + Instant::now() < deadline, + "vw-svc never started answering on {}", + self.admin, + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + /// Ask the admin API for every environment. + async fn environments( + &self, + caller: &str, + ) -> reqwest::Result { + self.client + .get(format!("{}/environments", self.admin)) + .header("x-vw-user", caller) + .send() + .await + } + + /// Delete somebody's environment through the admin API. + async fn delete( + &self, + caller: &str, + user: &str, + name: &str, + ) -> reqwest::Response { + self.client + .delete(format!("{}/environment/{user}/{name}", self.admin)) + .header("x-vw-user", caller) + .send() + .await + .expect("delete environment") + } + + /// Create an environment the ordinary way, as `user`. + async fn create(&self, user: &str, name: &str) { + let response = self + .client + .put(format!("{}/environment/{name}", self.users)) + .header("x-vw-user", user) + .json(&serde_json::json!({})) + .send() + .await + .expect("create environment"); + assert_eq!(response.status(), StatusCode::CREATED); + } + + /// What `user` can see of their own environments, through the user API. + async fn own_environments(&self, user: &str) -> Vec { + let response = self + .client + .get(format!("{}/environments", self.users)) + .header("x-vw-user", user) + .send() + .await + .expect("list own environments"); + assert_eq!(response.status(), StatusCode::OK); + + let page: serde_json::Value = + response.json().await.expect("decode listing"); + page["items"] + .as_array() + .expect("items") + .iter() + .map(|item| item["name"].as_str().expect("name").to_owned()) + .collect() + } + + /// Every environment the admin API reports, as `user/name`. + async fn everything(&self) -> Vec { + let response = self.environments(ADMIN).await.expect("list"); + assert_eq!(response.status(), StatusCode::OK); + + let page: serde_json::Value = + response.json().await.expect("decode listing"); + let mut found: Vec = page["items"] + .as_array() + .expect("items") + .iter() + .map(|item| { + format!( + "{}/{}", + item["user"].as_str().expect("user"), + item["environment"]["name"].as_str().expect("name"), + ) + }) + .collect(); + found.sort(); + found + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("read ephemeral port") + .port() +} + +async fn server() -> (TempDir, TestServer) { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + (dir, server) +} + +#[tokio::test] +async fn an_administrator_sees_every_environment() { + let (_dir, server) = server().await; + server.create(DEVELOPER, "darmok").await; + server.create(DEVELOPER, "jalad").await; + server.create("laforge", "tanagra").await; + + // The user API shows one developer their own two and nothing else; this + // is the endpoint that exists because that is not enough for whoever runs + // the rack. + assert_eq!(server.own_environments(DEVELOPER).await.len(), 2); + + assert_eq!( + server.everything().await, + ["barclay/darmok", "barclay/jalad", "laforge/tanagra"], + ); +} + +#[tokio::test] +async fn a_developer_is_not_an_administrator() { + let (_dir, server) = server().await; + server.create(DEVELOPER, "darmok").await; + + let refused = server.environments(DEVELOPER).await.expect("list"); + + // Forbidden rather than unauthorized: they are who they say they are, and + // it is not enough. + assert_eq!(refused.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn an_administrator_can_delete_somebody_elses_environment() { + // The reason this API exists: reclaiming a rack should not require the + // developer who filled it to still be around. + let (_dir, server) = server().await; + server.create(DEVELOPER, "darmok").await; + server.create(DEVELOPER, "jalad").await; + + let deleted = server.delete(ADMIN, DEVELOPER, "darmok").await; + + assert_eq!(deleted.status(), StatusCode::NO_CONTENT); + assert_eq!(server.everything().await, ["barclay/jalad"]); + // And the developer sees the same thing, since there is one record of it. + assert_eq!(server.own_environments(DEVELOPER).await, ["jalad"]); +} + +#[tokio::test] +async fn a_developer_cannot_delete_anything_through_the_admin_api() { + let (_dir, server) = server().await; + server.create(DEVELOPER, "darmok").await; + server.create("laforge", "tanagra").await; + + let refused = server.delete(DEVELOPER, "laforge", "tanagra").await; + + assert_eq!(refused.status(), StatusCode::FORBIDDEN); + // Nothing happened, which is the part that matters. + assert_eq!( + server.everything().await, + ["barclay/darmok", "laforge/tanagra"], + ); +} + +#[tokio::test] +async fn deleting_an_environment_that_is_not_there_is_not_found() { + let (_dir, server) = server().await; + server.create(DEVELOPER, "darmok").await; + + let missing = server.delete(ADMIN, DEVELOPER, "tanagra").await; + + assert_eq!(missing.status(), StatusCode::NOT_FOUND); + assert_eq!(server.everything().await, ["barclay/darmok"]); +} + +#[tokio::test] +async fn a_service_with_no_environments_lists_none() { + let (_dir, server) = server().await; + + assert!(server.everything().await.is_empty()); +} diff --git a/vw-svc/tests/user_api.rs b/vw-svc/tests/user_api.rs new file mode 100644 index 0000000..9592b61 --- /dev/null +++ b/vw-svc/tests/user_api.rs @@ -0,0 +1,498 @@ +// Integration tests for the vw-svc user API. +// +// Each test spawns the service as a child process against a scratch database +// with `--no-auth`, so these run anywhere without Github credentials. With +// authorization disabled the service takes the `x-vw-user` header at face +// value as the caller's identity, which is what lets these tests exercise the +// per-user behavior of the API without talking to Github. + +use std::net::TcpListener; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use dropshot::ResultsPage; +use reqwest::StatusCode; +use tempfile::TempDir; +use vw_api_types_versions::latest::{Environment, SshKeyPair}; + +/// How long to wait for a freshly spawned service to accept requests. +const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); + +/// A running vw-svc backed by a scratch database. +/// +/// Dropping the server kills the child process. +struct TestServer { + child: Child, + base_url: String, + client: reqwest::Client, +} + +impl TestServer { + /// Spawn the service against the database at `db_path`, creating it if it + /// does not exist, and wait for the user API to start answering. + async fn start(db_path: &Path) -> TestServer { + let user_port = free_port(); + let child = Command::new(env!("CARGO_BIN_EXE_vw-svc")) + .arg("serve") + .args(["--address", "127.0.0.1"]) + .args(["--user-api-port", &user_port.to_string()]) + .args(["--admin-api-port", &free_port().to_string()]) + .args(["--db-path", db_path.to_str().expect("utf8 database path")]) + .arg("--no-auth") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn vw-svc"); + + let mut server = TestServer { + child, + base_url: format!("http://127.0.0.1:{user_port}"), + client: reqwest::Client::new(), + }; + server.wait_until_ready().await; + server + } + + async fn wait_until_ready(&mut self) { + let deadline = Instant::now() + STARTUP_TIMEOUT; + loop { + if self + .client + .get(format!("{}/environments", self.base_url)) + .send() + .await + .is_ok() + { + return; + } + if let Some(status) = + self.child.try_wait().expect("check on vw-svc process") + { + panic!("vw-svc exited during startup: {status}"); + } + assert!( + Instant::now() < deadline, + "vw-svc never started listening on {}", + self.base_url, + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + /// Send a request as `user`. Authorization is disabled, so the service + /// reads the caller's identity straight out of this header. + fn request( + &self, + method: reqwest::Method, + user: &str, + path: &str, + ) -> reqwest::RequestBuilder { + self.client + .request(method, format!("{}{path}", self.base_url)) + .header("x-vw-user", user) + } + + async fn list(&self, user: &str) -> reqwest::Response { + self.request(reqwest::Method::GET, user, "/environments") + .send() + .await + .expect("list environments") + } + + async fn create(&self, user: &str, name: &str) -> reqwest::Response { + // No image overrides: the service has no Oxide backend in these + // tests, so it records the environment without resolving any. + self.request( + reqwest::Method::PUT, + user, + &format!("/environment/{name}"), + ) + .json(&serde_json::json!({})) + .send() + .await + .expect("create environment") + } + + async fn get(&self, user: &str, name: &str) -> reqwest::Response { + self.request( + reqwest::Method::GET, + user, + &format!("/environment/{name}"), + ) + .send() + .await + .expect("get environment") + } + + async fn keys(&self, user: &str, name: &str) -> reqwest::Response { + self.request( + reqwest::Method::GET, + user, + &format!("/environment/{name}/keys"), + ) + .send() + .await + .expect("get environment keys") + } + + async fn delete(&self, user: &str, name: &str) -> reqwest::Response { + self.request( + reqwest::Method::DELETE, + user, + &format!("/environment/{name}"), + ) + .send() + .await + .expect("delete environment") + } + + /// The names of `user`'s environments, in the order the API returned them. + /// + /// The endpoint takes no pagination parameters, so a complete listing is + /// always one page with no next page token. + async fn environment_names(&self, user: &str) -> Vec { + let response = self.list(user).await; + assert_eq!(response.status(), StatusCode::OK); + let page: ResultsPage = + response.json().await.expect("decode environments page"); + assert_eq!( + page.next_page, None, + "an unpaginated listing should not offer a next page", + ); + page.items.into_iter().map(|env| env.name).collect() + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// A port nothing is listening on, obtained by binding an ephemeral port and +/// immediately releasing it. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("read ephemeral port") + .port() +} + +#[tokio::test] +async fn environment_lifecycle() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert!(server.environment_names("ferris").await.is_empty()); + + let response = server.create("ferris", "alpha").await; + assert_eq!(response.status(), StatusCode::CREATED); + + let response = server.get("ferris", "alpha").await; + assert_eq!(response.status(), StatusCode::OK); + let env: Environment = response.json().await.expect("decode environment"); + assert_eq!(env.name, "alpha"); + // A new environment has no instances behind it yet. + assert!(env.vivado_instance.is_none()); + assert!(env.helios_instance.is_none()); + assert!(env.artifact_instance.is_none()); + + assert_eq!(server.environment_names("ferris").await, ["alpha"]); + + let response = server.delete("ferris", "alpha").await; + assert_eq!(response.status(), StatusCode::NO_CONTENT); + + assert!(server.environment_names("ferris").await.is_empty()); +} + +#[tokio::test] +async fn creating_the_same_environment_twice_conflicts() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CONFLICT + ); + + // The conflicting create left the original entry alone. + assert_eq!(server.environment_names("ferris").await, ["alpha"]); +} + +#[tokio::test] +async fn missing_environments_are_not_found() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.get("ferris", "nonesuch").await.status(), + StatusCode::NOT_FOUND + ); + assert_eq!( + server.delete("ferris", "nonesuch").await.status(), + StatusCode::NOT_FOUND + ); + + // Deleting an environment consumes it, so a second delete is a 404 too. + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + assert_eq!( + server.delete("ferris", "alpha").await.status(), + StatusCode::NO_CONTENT + ); + assert_eq!( + server.delete("ferris", "alpha").await.status(), + StatusCode::NOT_FOUND + ); +} + +#[tokio::test] +async fn environments_are_scoped_to_their_owner() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + for (user, name) in + [("ferris", "alpha"), ("ferris", "beta"), ("gorris", "alpha")] + { + assert_eq!( + server.create(user, name).await.status(), + StatusCode::CREATED + ); + } + + assert_eq!(server.environment_names("ferris").await, ["alpha", "beta"]); + assert_eq!(server.environment_names("gorris").await, ["alpha"]); + + // Same environment name, different owners: deleting one leaves the other. + assert_eq!( + server.delete("ferris", "alpha").await.status(), + StatusCode::NO_CONTENT + ); + assert_eq!(server.environment_names("ferris").await, ["beta"]); + assert_eq!(server.environment_names("gorris").await, ["alpha"]); + + // One user's name being a prefix of another's must not blur the two + // listings together, which is the failure mode of the prefix scan that + // backs this endpoint. + assert_eq!( + server.create("f", "solo").await.status(), + StatusCode::CREATED + ); + assert_eq!(server.environment_names("f").await, ["solo"]); + assert_eq!(server.environment_names("ferris").await, ["beta"]); +} + +#[tokio::test] +async fn environments_outlive_the_service() { + let dir = TempDir::new().expect("scratch directory"); + let db_path = dir.path().join("vw-svc.redb"); + + { + let server = TestServer::start(&db_path).await; + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + } + + let server = TestServer::start(&db_path).await; + assert_eq!(server.environment_names("ferris").await, ["alpha"]); +} + +#[tokio::test] +async fn a_full_listing_is_a_single_page() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + let expected: Vec = (0..25).map(|i| format!("env{i:02}")).collect(); + for name in &expected { + assert_eq!( + server.create("ferris", name).await.status(), + StatusCode::CREATED + ); + } + + // `environment_names` asserts the page carries no next page token. + assert_eq!(server.environment_names("ferris").await, expected); +} + +#[tokio::test] +async fn names_that_break_the_instance_naming_scheme_are_rejected() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + // Oxide instances are named "vwsvc-{user}-{env}-{kind}" and split back + // apart on `-`, so an environment name carrying one would not survive the + // round trip. The rest are what the control plane accepts for a name. + // An empty name is not in this list because the router rejects + // `PUT /environment/` as a 404 before a handler ever sees it. + for rejected in ["my-env", "MyEnv", "my_env", "9lives"] { + assert_eq!( + server.create("ferris", rejected).await.status(), + StatusCode::BAD_REQUEST, + "expected '{rejected}' to be rejected", + ); + } + + for accepted in ["alpha", "env2", "a"] { + assert_eq!( + server.create("ferris", accepted).await.status(), + StatusCode::CREATED, + "expected '{accepted}' to be accepted", + ); + } +} + +#[tokio::test] +async fn environments_have_no_images_without_an_oxide_backend() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + + // With no rack to resolve against there is nothing to pin, and the + // environment is a bare record the reconciler will never provision. + let response = server.get("ferris", "alpha").await; + let env: Environment = response.json().await.expect("decode environment"); + assert!(env.images.is_none()); +} + +#[tokio::test] +async fn naming_an_image_without_an_oxide_backend_is_rejected() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + // Accepting an image the service can neither validate nor ever use would + // look like it took effect. + let response = server + .request(reqwest::Method::PUT, "ferris", "/environment/alpha") + .json(&serde_json::json!({ "vivado_image": "vw-vivado-20260101" })) + .send() + .await + .expect("create environment"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(server.environment_names("ferris").await.is_empty()); +} + +#[tokio::test] +async fn an_environment_comes_with_a_key_that_opens_it() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + // The create hands the key back directly, so a client can save it without + // a second round trip. + let created = server.create("ferris", "alpha").await; + assert_eq!(created.status(), StatusCode::CREATED); + let from_create: SshKeyPair = + created.json().await.expect("decode created keys"); + + let response = server.keys("ferris", "alpha").await; + assert_eq!(response.status(), StatusCode::OK); + let keys: SshKeyPair = response.json().await.expect("decode keys"); + + // And it is the same pair, not a second one generated on the way out. + assert_eq!(from_create.private_key, keys.private_key); + assert_eq!(from_create.public_key, keys.public_key); + + // The shapes ssh itself insists on: OpenSSH private key encoding, and a + // public key line an authorized_keys file would accept. + assert!(keys + .private_key + .starts_with("-----BEGIN OPENSSH PRIVATE KEY-----")); + assert!(keys.public_key.starts_with("ssh-ed25519 ")); + // Named after the environment it opens, so it is recognizable in an agent. + assert!(keys.public_key.trim_end().ends_with("vw ferris/alpha")); +} + +#[tokio::test] +async fn a_private_key_never_rides_along_with_an_environment() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + + // The key lives in its own table precisely so that fetching or listing an + // environment cannot carry it out. + for body in [ + server.get("ferris", "alpha").await.text().await.unwrap(), + server.list("ferris").await.text().await.unwrap(), + ] { + assert!( + !body.contains("PRIVATE KEY"), + "an environment response leaked a private key: {body}", + ); + } +} + +#[tokio::test] +async fn one_users_key_is_not_another_users_to_fetch() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + + // Same environment name, different caller: keys are scoped like every + // other endpoint here. + assert_eq!( + server.keys("gorris", "alpha").await.status(), + StatusCode::NOT_FOUND + ); +} + +#[tokio::test] +async fn deleting_an_environment_takes_its_key_with_it() { + let dir = TempDir::new().expect("scratch directory"); + let server = TestServer::start(&dir.path().join("vw-svc.redb")).await; + + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + assert_eq!( + server.keys("ferris", "alpha").await.status(), + StatusCode::OK + ); + + assert_eq!( + server.delete("ferris", "alpha").await.status(), + StatusCode::NO_CONTENT + ); + assert_eq!( + server.keys("ferris", "alpha").await.status(), + StatusCode::NOT_FOUND, + "a key that opens nothing should not outlive its environment", + ); + + // A fresh environment of the same name gets a fresh key rather than + // inheriting the old one. + assert_eq!( + server.create("ferris", "alpha").await.status(), + StatusCode::CREATED + ); + let keys: SshKeyPair = server + .keys("ferris", "alpha") + .await + .json() + .await + .expect("decode keys"); + assert!(keys.public_key.starts_with("ssh-ed25519 ")); +} diff --git a/vw-sync-api/Cargo.toml b/vw-sync-api/Cargo.toml new file mode 100644 index 0000000..6048c5d --- /dev/null +++ b/vw-sync-api/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vw-sync-api" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The API a vw build instance exposes to receive source" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools"] + +[dependencies] +vw-api-types-versions = { path = "../vw-api-types/versions" } +dropshot.workspace = true +dropshot-api-manager-types = "0.7.2" +schemars.workspace = true +serde.workspace = true diff --git a/vw-sync-api/src/lib.rs b/vw-sync-api/src/lib.rs new file mode 100644 index 0000000..423fc51 --- /dev/null +++ b/vw-sync-api/src/lib.rs @@ -0,0 +1,307 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The API a build instance exposes so that `vw-svc` can put source on it. +//! +//! Three calls make a synchronization: say what the tree should look like and +//! find out what is missing, deliver exactly that, then ask for the tree to be +//! made to match. Each is idempotent — a blob is named by its own content, and +//! a commit describes a destination rather than a change — so a retry after a +//! dropped connection costs at worst a repeated upload. +//! +//! This is not reachable from a developer's machine. `vw-svc` holds the only +//! route to it, over the rack's internal network, and relays the client's calls +//! after deciding whether the caller owns the environment in question. + +use dropshot::{ + api_description, FreeformBody, HttpError, HttpResponseOk, + HttpResponseUpdatedNoContent, Path, Query, RequestContext, TypedBody, + UntypedBody, WebsocketChannelResult, WebsocketConnection, +}; +use dropshot_api_manager_types::api_versions; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use vw_api_types_versions::latest; + +api_versions!([ + // WHEN CHANGING THE API (part 1 of 2): + // + // +- Pick a new semver and define it in the list below. The list MUST + // | remain sorted, which generally means that your version should go at + // | the very top. + // | + // | Duplicate this line, uncomment the *second* copy, update that copy for + // | your new API version, and leave the first copy commented out as an + // | example for the next person. + // v + // (next_int, IDENT), + (1, INITIAL), +]); + +/// Which environment a request is for. +/// +/// An agent serves exactly one, and checks this against the one it was +/// started with. The name never becomes part of a filesystem path — the tree +/// and content store are fixed at startup — so a request for the wrong +/// environment is answered rather than acted on. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentPathParam { + pub environment: String, +} + +/// Which piece of content is being delivered. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct BlobPathParam { + pub environment: String, + /// The digest of the content in the body, which is verified on arrival + /// rather than taken at face value. + pub digest: latest::Digest, +} + +/// Source synchronization for one build instance. +#[api_description] +pub trait VwSyncApi { + type Context; + + /// Report what content is missing before a tree can be made to match. + /// + /// Content already held anywhere in the instance's tree is not asked for, + /// whatever path it currently sits under, so a rename or a directory move + /// costs nothing over the wire. + #[endpoint { + method = POST, + path = "/environment/{environment}/sync/plan", + }] + async fn sync_plan( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + /// Deliver one piece of content. + /// + /// Rejected if the body does not hash to the digest in the path: the + /// digest is how every later lookup finds this content, so storing it + /// under a name it does not have would be worse than not storing it. + #[endpoint { + method = PUT, + path = "/environment/{environment}/sync/blob/{digest}", + }] + async fn sync_blob( + rqctx: RequestContext, + path_params: Path, + body: UntypedBody, + ) -> Result; + + /// Make the instance's tree match the manifest. + /// + /// The manifest is the complete desired state, so this adds, replaces and + /// removes as needed. Anything a build produced is invisible to it. + #[endpoint { + method = POST, + path = "/environment/{environment}/sync/commit", + }] + async fn sync_commit( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + /// Discard the source tree and everything delivered towards it. + /// + /// The instance is left as though it had never been synchronized: no + /// source, and no record of what content it holds. Build output is + /// untouched, as it is for a commit. + /// + /// This is not part of an ordinary sync, which needs no help — a commit + /// replaces whatever differs from the manifest. It is what a sender uses + /// when it does not believe the instance's account of what it has, so that + /// the sync that follows sends everything rather than asking first. + /// + /// Answers with the result of committing an empty manifest, so the count + /// of what was removed is the `deleted` field. + #[endpoint { + method = DELETE, + path = "/environment/{environment}/sync", + }] + async fn sync_clear( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Put the credentials a build needs to fetch its dependencies in place. + /// + /// Written as a `.netrc`, which is what git, cargo and the rest already + /// know how to read, so nothing on the instance needs teaching about where + /// its credentials come from. + /// + /// These belong to whoever is synchronizing, and `vw-svc` sends them with + /// every sync rather than once: an instance rebuilt underneath us comes + /// back with no credentials at all, and the alternative is builds that + /// fail to fetch until something notices. + #[endpoint { + method = PUT, + path = "/environment/{environment}/credentials", + }] + async fn put_credentials( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result; + + /// Remove everything a build wrote on this instance. + /// + /// The opposite of what synchronization does: `target/` is the one thing a + /// sync will never send and never delete, which is exactly why removing it + /// needs saying explicitly. Source is untouched, so the next build starts + /// over without anything having to be pushed again. + #[endpoint { + method = DELETE, + path = "/environment/{environment}/build-output", + }] + async fn clean_build_output( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Where this instance currently believes its artifacts should go. + /// + /// Answers `404` when it has never been told. Lets the service notice an + /// instance that needs configuring — one created before there was a store, + /// or whose store has since been rebuilt with a new key — without pushing + /// credentials at every instance on every restart. + #[endpoint { + method = GET, + path = "/environment/{environment}/artifact-target", + }] + async fn get_artifact_target( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// The VHDL vivado generated for this environment's IP. + /// + /// A `POST` because it finishes the job first: vivado writes an + /// instantiation template per standalone IP, and turning those into + /// black-box entities is a mechanical step that happens in Rust after the + /// vivado pass. On a local run that happens on the developer's machine; on + /// a remote one there is nobody there to do it, so it happens here, where + /// the templates are. + /// + /// Answers with paths relative to the workspace, so the far end can put + /// each file exactly where its own tools will look for it. + #[endpoint { + method = POST, + path = "/environment/{environment}/generated", + }] + async fn generated_manifest( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// One generated file's contents. + #[endpoint { + method = GET, + path = "/environment/{environment}/generated/file", + }] + async fn generated_file( + rqctx: RequestContext, + path_params: Path, + query: Query, + ) -> Result, HttpError>; + + /// The key that opens this instance's object store. + /// + /// Answered only by the instance that runs the store. The admin credential + /// that minted this key never leaves that machine — this is the one thing + /// it hands out, and `vw-svc` passes it to the instance that has artifacts + /// to upload. + #[endpoint { + method = GET, + path = "/environment/{environment}/object-store", + }] + async fn get_object_store( + rqctx: RequestContext, + path_params: Path, + query: Query, + ) -> Result, HttpError>; + + /// Tell this instance where to put the artifacts it builds. + /// + /// Sent by `vw-svc`, which got it from the instance that runs the store — + /// this one cannot ask directly, since it has no way to know which of its + /// neighbours holds it. Remembered on disk, so a reboot between two builds + /// does not lose the answer. + #[endpoint { + method = PUT, + path = "/environment/{environment}/artifact-target", + }] + async fn put_artifact_target( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result; + + /// Build the driver on this instance. + /// + /// A websocket because a build takes minutes and produces output the whole + /// time, and because a developer who interrupts one should not leave cargo + /// running on a machine nobody is watching. + /// + /// Cargo is spawned rather than linked: the driver pins its toolchain in + /// `rust-toolchain.toml`, which the rustup shim honours and a linked cargo + /// would not, so linking it would quietly build a kernel module with the + /// wrong compiler. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{environment}/driver/build", + }] + async fn driver_build( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// Run this workspace's testbenches on this instance. + /// + /// A websocket for the same reason a vivado session is one: a batch takes + /// minutes and finishes one bench at a time, and a developer watching it + /// should see each result land rather than a verdict at the end. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{environment}/bench/session", + }] + async fn bench_session( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// Drive a vivado worker on this instance. + /// + /// A websocket rather than a request and a reply because a build is not + /// one of those. It is a conversation that runs for minutes, produces + /// output the whole time, and has to show that output as it happens — + /// waiting for a synthesis run to finish before saying anything would make + /// the remote flow useless for the thing people actually do with it. + /// + /// What crosses the socket is the protocol `vw-eda` already uses to talk + /// to a local worker: commands in, output chunks and results back. The + /// worker is spawned when the socket opens and torn down when it closes, + /// so no state survives between runs — the same guarantee running vivado + /// locally gives, and what the checkpoint machinery in the htcl library + /// already relies on for speed. + #[channel { + protocol = WEBSOCKETS, + path = "/environment/{environment}/vivado/session", + }] + async fn vivado_session( + rqctx: RequestContext, + path_params: Path, + query: Query, + websock: WebsocketConnection, + ) -> WebsocketChannelResult; +} diff --git a/vw-sync/Cargo.toml b/vw-sync/Cargo.toml new file mode 100644 index 0000000..eb26af0 --- /dev/null +++ b/vw-sync/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "vw-sync" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Content-addressed source tree synchronization for vw" +keywords = ["vhdl", "workspace", "dependency-management"] +categories = ["development-tools"] + +[dependencies] +vw-api-types-versions = { path = "../vw-api-types/versions" } +camino.workspace = true +thiserror.workspace = true +blake3 = "1" +ignore = "0.4" + +[dev-dependencies] +tempfile.workspace = true diff --git a/vw-sync/examples/scan.rs b/vw-sync/examples/scan.rs new file mode 100644 index 0000000..bfec2d7 --- /dev/null +++ b/vw-sync/examples/scan.rs @@ -0,0 +1,41 @@ +//! Show what a workspace would synchronize, without synchronizing it. +//! +//! ```text +//! cargo run -p vw-sync --example scan -- ~/sketch/metroid +//! ``` +//! +//! Useful for answering "why is my sync so large" before blaming the network. +//! The answer is nearly always an ignore rule that is not doing what it looks +//! like it does. + +fn main() { + let Some(root) = std::env::args().nth(1) else { + eprintln!("usage: scan "); + std::process::exit(1); + }; + let root = camino::Utf8PathBuf::from(root); + + let manifest = match vw_sync::scan(&root) { + Ok(manifest) => manifest, + Err(e) => { + eprintln!("error: {e}"); + std::process::exit(1); + } + }; + + let bytes: u64 = manifest + .entries + .iter() + .filter_map(|entry| std::fs::metadata(root.join(&entry.path)).ok()) + .map(|meta| meta.len()) + .sum(); + + println!( + "{} files, {:.1} MiB", + manifest.entries.len(), + bytes as f64 / (1024.0 * 1024.0), + ); + for entry in &manifest.entries { + println!(" {}", entry.path); + } +} diff --git a/vw-sync/src/lib.rs b/vw-sync/src/lib.rs new file mode 100644 index 0000000..12008af --- /dev/null +++ b/vw-sync/src/lib.rs @@ -0,0 +1,47 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Content-addressed synchronization of a source tree. +//! +//! The same engine runs at both ends. A sender scans a directory into a +//! [`TreeManifest`] describing every file it wants to exist; a receiver +//! answers with the content it does not already hold, takes delivery of that +//! into a [`Store`], and then makes its own directory match the manifest. +//! +//! Three properties are worth stating up front, because the rest follows from +//! them. +//! +//! **Whole files, not diffs.** Build sources are small, and content addressing +//! already collapses the unchanged ones. Computing and applying deltas would +//! cost more round trips than it saves bytes — the same conclusion Bazel and +//! Buck2 reached for their remote execution protocols. +//! +//! **Complete manifests, not changesets.** An environment may be synchronized +//! from a different machine tomorrow, and a changeset computed against one +//! machine's state means nothing against another's. A complete manifest makes +//! "make it look like this" the literal semantics, and deletions and renames +//! fall out of it. +//! +//! **Generated files are invisible.** The scan honours `.gitignore` +//! hierarchically and excludes build output outright, so neither end sends, +//! receives, or deletes anything a build produced. Both ends apply the same +//! rules — the receiver reads them from the tree it was handed, so they cannot +//! drift apart. +//! +//! [`TreeManifest`]: vw_api_types_versions::latest::TreeManifest + +mod scan; +mod store; +mod tree; + +pub use scan::{scan, ScanError, ALWAYS_IGNORED, BUILD_OUTPUT}; +pub use store::{Store, StoreError}; +pub use tree::{apply, clean, clear, missing, ApplyError, Cleaned}; + +use vw_api_types_versions::latest::Digest; + +/// The digest of some bytes. +pub fn digest_bytes(bytes: &[u8]) -> Digest { + Digest(blake3::hash(bytes).to_hex().to_string()) +} diff --git a/vw-sync/src/scan.rs b/vw-sync/src/scan.rs new file mode 100644 index 0000000..b28851e --- /dev/null +++ b/vw-sync/src/scan.rs @@ -0,0 +1,110 @@ +//! Turning a directory into a manifest. + +use camino::{Utf8Path, Utf8PathBuf}; +use ignore::WalkBuilder; +use vw_api_types_versions::latest::{FileEntry, TreeManifest}; + +/// Directories never synchronized, whatever the ignore files say. +/// +/// A vw workspace puts every generated artifact under a `target` directory — +/// vivado's synthesis output at the root, cargo's under each crate — and both +/// are already in a `.gitignore`. This is a floor underneath that, so a +/// one-line edit to an ignore file cannot turn a keystroke into a transfer of +/// somebody's entire synthesis run. +/// +/// `.git` is here for the same reason rather than for size: the receiver has +/// no use for history, and a half-copied object store is worse than none. +pub const ALWAYS_IGNORED: [&str; 2] = [BUILD_OUTPUT, ".git"]; + +/// The directory a build writes its output to. +/// +/// Named once here because two things depend on knowing it: synchronization, +/// which must never send or delete it, and `vw clean`, whose entire job is to +/// delete it. Those are opposite behaviours over the same directory, and them +/// disagreeing about which directory would be a bad afternoon either way. +pub const BUILD_OUTPUT: &str = "target"; + +#[derive(Debug, thiserror::Error)] +pub enum ScanError { + #[error("walking {0}")] + Walk(Utf8PathBuf, #[source] ignore::Error), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), + #[error("{0} is not valid utf-8, which a manifest path has to be")] + NotUtf8(std::path::PathBuf), +} + +/// Describe every file under `root` that should be synchronized. +/// +/// Honours `.gitignore` hierarchically, so a nested crate's own ignore rules +/// apply to its subtree the way git would read them, on top of +/// [`ALWAYS_IGNORED`]. +/// +/// Entries come back sorted by path. That is not cosmetic: a manifest is +/// compared and hashed by consumers, and a walk order that varies with the +/// filesystem would make identical trees look different. +pub fn scan(root: &Utf8Path) -> Result { + let mut entries = Vec::new(); + + let mut walker = WalkBuilder::new(root); + walker + // Read .gitignore files, including nested ones, but do not require the + // tree to be a git repository — a synchronized copy on the receiver is + // not one, and it still has to reach the same answer. + .git_ignore(true) + .git_global(false) + .git_exclude(false) + .require_git(false) + .hidden(false) + .parents(false) + // One predicate covering every name: `filter_entry` keeps only the + // last closure it is given, so a loop calling it per name would + // silently apply just the final one. + .filter_entry(|entry| { + !ALWAYS_IGNORED + .iter() + .any(|name| entry.file_name() == std::ffi::OsStr::new(name)) + }); + + for entry in walker.build() { + let entry = entry.map_err(|e| ScanError::Walk(root.to_owned(), e))?; + + // Directories are implied by the paths of the files in them, and + // anything that is neither a file nor a directory — a socket, a fifo — + // has no meaning on the far end. + if !entry.file_type().is_some_and(|kind| kind.is_file()) { + continue; + } + + let path = Utf8Path::from_path(entry.path()) + .ok_or_else(|| ScanError::NotUtf8(entry.path().to_owned()))?; + let relative = path + .strip_prefix(root) + .map_err(|_| ScanError::NotUtf8(path.as_std_path().to_owned()))?; + + let contents = std::fs::read(path) + .map_err(|e| ScanError::Read(path.to_owned(), e))?; + + entries.push(FileEntry { + path: relative.as_str().to_owned(), + digest: crate::digest_bytes(&contents), + executable: is_executable(path), + }); + } + + entries.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(TreeManifest { entries }) +} + +#[cfg(unix)] +fn is_executable(path: &Utf8Path) -> bool { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path) + .map(|meta| meta.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +#[cfg(not(unix))] +fn is_executable(_path: &Utf8Path) -> bool { + false +} diff --git a/vw-sync/src/store.rs b/vw-sync/src/store.rs new file mode 100644 index 0000000..96a29a7 --- /dev/null +++ b/vw-sync/src/store.rs @@ -0,0 +1,106 @@ +//! Where delivered content waits between arriving and being put in place. + +use camino::{Utf8Path, Utf8PathBuf}; +use vw_api_types_versions::latest::Digest; + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("'{0}' is not a well formed digest")] + MalformedDigest(Digest), + #[error("content for {0} does not match its digest")] + DigestMismatch(Digest), + #[error("creating {0}")] + CreateDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("reading {0}")] + Read(Utf8PathBuf, #[source] std::io::Error), + #[error("emptying {0}")] + Empty(Utf8PathBuf, #[source] std::io::Error), +} + +/// Content held by digest, waiting to be placed into a tree. +/// +/// Only a staging area. A commit copies out of it into the working tree, and +/// nothing reads through it afterwards — so it can be emptied at any time and +/// the next sync will simply re-deliver what it needs. +pub struct Store { + root: Utf8PathBuf, +} + +impl Store { + pub fn new(root: impl Into) -> Store { + Store { root: root.into() } + } + + /// Whether this content is already held. + pub fn has(&self, digest: &Digest) -> bool { + self.path(digest).is_ok_and(|path| path.is_file()) + } + + /// Take delivery of content. + /// + /// The digest is verified rather than trusted. It names the file the + /// content is written to, and every later lookup goes by digest, so + /// accepting a mismatch would poison the store with content that is wrong + /// under a name that looks right. + pub fn put( + &self, + digest: &Digest, + contents: &[u8], + ) -> Result<(), StoreError> { + if crate::digest_bytes(contents) != *digest { + return Err(StoreError::DigestMismatch(digest.clone())); + } + + let path = self.path(digest)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| StoreError::CreateDir(parent.to_owned(), e))?; + } + std::fs::write(&path, contents) + .map_err(|e| StoreError::Write(path.clone(), e))?; + + Ok(()) + } + + /// Discard everything held. + /// + /// Nothing is lost that a sender cannot deliver again — that is what makes + /// this safe to offer. A sender that no longer trusts what this store + /// claims to have empties it, and the next plan asks for the whole tree. + pub fn empty(&self) -> Result<(), StoreError> { + match std::fs::remove_dir_all(&self.root) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(StoreError::Empty(self.root.clone(), e)), + } + } + + pub fn get(&self, digest: &Digest) -> Result, StoreError> { + let path = self.path(digest)?; + std::fs::read(&path).map_err(|e| StoreError::Read(path, e)) + } + + /// Where content with this digest lives. + /// + /// Sharded by the first two hex characters, so a store with a lot of + /// content does not become one enormous directory. + /// + /// The digest is checked for shape first. It arrives over the wire and is + /// about to become a path, so anything that is not 64 hex characters — + /// `../../etc/whatever` being the interesting case — is refused here + /// rather than allowed to escape the store. + fn path(&self, digest: &Digest) -> Result { + if !digest.is_well_formed() { + return Err(StoreError::MalformedDigest(digest.clone())); + } + let (shard, rest) = digest.0.split_at(2); + Ok(self.root.join(shard).join(rest)) + } + + /// Where this store keeps its content. + pub fn root(&self) -> &Utf8Path { + &self.root + } +} diff --git a/vw-sync/src/tree.rs b/vw-sync/src/tree.rs new file mode 100644 index 0000000..990c79b --- /dev/null +++ b/vw-sync/src/tree.rs @@ -0,0 +1,378 @@ +//! Making a directory match a manifest. + +use std::collections::BTreeMap; + +use camino::{Utf8Path, Utf8PathBuf}; +use vw_api_types_versions::latest::{ + CommitResult, Digest, FileEntry, SyncPlan, TreeManifest, +}; + +use crate::{scan, Store, StoreError}; + +#[derive(Debug, thiserror::Error)] +pub enum ApplyError { + #[error("'{0}' is not a path a manifest may name")] + UnsafePath(String), + #[error("scanning the tree at {0}")] + Scan(Utf8PathBuf, #[source] crate::ScanError), + #[error("no content for {digest}, wanted at {path}")] + MissingContent { path: String, digest: Digest }, + #[error(transparent)] + Store(#[from] StoreError), + #[error("creating {0}")] + CreateDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("removing {0}")] + Remove(Utf8PathBuf, #[source] std::io::Error), +} + +/// The content a manifest needs that is nowhere to be found. +/// +/// Content already sitting somewhere in the tree is not reported, whatever +/// path it is under. A rename or a directory restructure therefore costs +/// nothing over the wire — the bytes are already here, and applying the +/// manifest copies them into their new place locally. +pub fn missing( + root: &Utf8Path, + store: &Store, + manifest: &TreeManifest, +) -> Result { + let held = held_content(root)?; + + let mut missing: Vec = manifest + .entries + .iter() + .map(|entry| &entry.digest) + .filter(|digest| !held.contains_key(*digest) && !store.has(digest)) + .cloned() + .collect(); + + // One request per digest, however many paths want it. + missing.sort(); + missing.dedup(); + + Ok(SyncPlan { missing }) +} + +/// Make the tree at `root` match `manifest`. +/// +/// Writes and updates first, then removes what the manifest does not mention. +/// That order is what makes a rename work: the content of the old path is +/// still there to be copied to the new one when it is needed. +/// +/// Only files the scan can see are candidates for removal, so anything a build +/// produced — everything under a `target` directory, anything a `.gitignore` +/// covers — is left alone. The receiver reads those rules from the tree it was +/// handed, which is why they cannot disagree with the sender's. +pub fn apply( + root: &Utf8Path, + store: &Store, + manifest: &TreeManifest, +) -> Result { + for entry in &manifest.entries { + check_path(&entry.path)?; + } + + let held = held_content(root)?; + let existing = + scan::scan(root).map_err(|e| ApplyError::Scan(root.to_owned(), e))?; + let current: BTreeMap<&str, &FileEntry> = existing + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + + let mut result = CommitResult::default(); + + for entry in &manifest.entries { + let path = root.join(&entry.path); + + match current.get(entry.path.as_str()) { + Some(have) + if have.digest == entry.digest + && have.executable == entry.executable => + { + result.unchanged += 1; + continue; + } + Some(_) => result.updated += 1, + None => result.created += 1, + } + + let contents = content_for(entry, store, &held)?; + write_file(&path, &contents, entry.executable)?; + } + + // Everything the manifest does not ask for. + let wanted: BTreeMap<&str, ()> = manifest + .entries + .iter() + .map(|entry| (entry.path.as_str(), ())) + .collect(); + for entry in &existing.entries { + if wanted.contains_key(entry.path.as_str()) { + continue; + } + let path = root.join(&entry.path); + std::fs::remove_file(&path).map_err(|e| ApplyError::Remove(path, e))?; + result.deleted += 1; + } + + if result.deleted > 0 { + prune_empty_dirs(root)?; + } + + Ok(result) +} + +/// Discard everything synchronization has put here. +/// +/// The tree is made to match an empty manifest and the content store is +/// emptied. Between them that removes every trace of what a sender last said, +/// while leaving anything a build produced exactly where it was — the same +/// rules decide what may be deleted here as anywhere else. +/// +/// Nothing needs this to stay correct. A commit already replaces whatever +/// differs from the manifest, so an ordinary sync is enough to fix a tree that +/// is merely out of date. This is for the case where the receiver's account of +/// itself is the thing in doubt: with nothing held and nothing in the tree, +/// there is no account left to be wrong, and the sync that follows sends the +/// whole tree because it genuinely is all missing. +pub fn clear( + root: &Utf8Path, + store: &Store, +) -> Result { + store.empty()?; + apply(root, store, &TreeManifest::default()) +} + +/// Content the tree already holds, indexed by digest rather than by path. +/// +/// By digest because that is the question worth asking: whether the bytes are +/// here at all, not whether they are here under the name they are wanted +/// under. +fn held_content( + root: &Utf8Path, +) -> Result, ApplyError> { + if !root.is_dir() { + return Ok(BTreeMap::new()); + } + + let manifest = + scan::scan(root).map_err(|e| ApplyError::Scan(root.to_owned(), e))?; + + Ok(manifest + .entries + .into_iter() + .map(|entry| (entry.digest, root.join(entry.path))) + .collect()) +} + +fn content_for( + entry: &FileEntry, + store: &Store, + held: &BTreeMap, +) -> Result, ApplyError> { + // Delivered content first: it is what a sender just took the trouble to + // upload, so preferring it keeps a freshly delivered file from being + // shadowed by a stale copy that happens to collide. + if store.has(&entry.digest) { + return Ok(store.get(&entry.digest)?); + } + + // Otherwise it may already be in the tree under another name. + if let Some(source) = held.get(&entry.digest) { + return std::fs::read(source) + .map_err(|e| ApplyError::Write(source.clone(), e)); + } + + Err(ApplyError::MissingContent { + path: entry.path.clone(), + digest: entry.digest.clone(), + }) +} + +fn write_file( + path: &Utf8Path, + contents: &[u8], + executable: bool, +) -> Result<(), ApplyError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| ApplyError::CreateDir(parent.to_owned(), e))?; + } + + // Removed rather than truncated: a source file checked out read-only + // cannot be opened for writing even by its owner, and replacing it is the + // whole point. + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(ApplyError::Remove(path.to_owned(), e)), + } + + std::fs::write(path, contents) + .map_err(|e| ApplyError::Write(path.to_owned(), e))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = if executable { 0o755 } else { 0o644 }; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)) + .map_err(|e| ApplyError::Write(path.to_owned(), e))?; + } + #[cfg(not(unix))] + let _ = executable; + + Ok(()) +} + +/// Reject anything that would put a file outside the tree. +/// +/// A manifest arrives over the wire and its paths become filesystem paths, so +/// `../../.ssh/authorized_keys` has to stop here. Absolute paths are refused +/// for the same reason, and a component of `.` is refused because it is only +/// ever noise in a path a scan produced. +fn check_path(path: &str) -> Result<(), ApplyError> { + let unsafe_path = || ApplyError::UnsafePath(path.to_owned()); + + if path.is_empty() || path.starts_with('/') || path.contains('\\') { + return Err(unsafe_path()); + } + // Windows drive letters would be absolute over there and are meaningless + // here either way. + if path.chars().nth(1) == Some(':') { + return Err(unsafe_path()); + } + for component in path.split('/') { + if component.is_empty() || component == ".." || component == "." { + return Err(unsafe_path()); + } + } + + Ok(()) +} + +/// Remove directories left behind with nothing in them. +/// +/// Deleting the last file in a directory otherwise leaves the directory, and a +/// build tool that globs would keep finding a package that no longer has any +/// sources in it. +fn prune_empty_dirs(root: &Utf8Path) -> Result<(), ApplyError> { + // Depth first, so a directory whose only content was other now-empty + // directories is caught in the same pass. + let mut directories = Vec::new(); + collect_dirs(root, &mut directories)?; + directories + .sort_by_key(|path| std::cmp::Reverse(path.components().count())); + + for directory in directories { + if directory == root { + continue; + } + let empty = std::fs::read_dir(&directory) + .map(|mut entries| entries.next().is_none()) + .unwrap_or(false); + if empty { + std::fs::remove_dir(&directory) + .map_err(|e| ApplyError::Remove(directory, e))?; + } + } + + Ok(()) +} + +fn collect_dirs( + root: &Utf8Path, + into: &mut Vec, +) -> Result<(), ApplyError> { + let entries = match std::fs::read_dir(root) { + Ok(entries) => entries, + Err(_) => return Ok(()), + }; + + for entry in entries.flatten() { + let Ok(path) = Utf8PathBuf::from_path_buf(entry.path()) else { + continue; + }; + if !path.is_dir() { + continue; + } + // A build's output directory is not ours to tidy. + if crate::ALWAYS_IGNORED + .iter() + .any(|name| path.file_name() == Some(*name)) + { + continue; + } + collect_dirs(&path, into)?; + into.push(path); + } + + Ok(()) +} + +/// What removing a tree's build output came to. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cleaned { + /// Whether there was anything there. + pub existed: bool, + /// How much space it was taking. + /// + /// Measured before removal rather than inferred from free space, which on + /// a shared instance is being moved by other things at the same time. + pub bytes: u64, +} + +/// Remove everything a build wrote under `root`. +/// +/// The counterpart to what synchronization refuses to touch: the same +/// directory it will never send and never delete is the one this exists to +/// delete, and both read the name from the same place. +/// +/// Source is left alone. A cleaned tree is one a build starts over in, not one +/// that has to be pushed again. +pub fn clean(root: &Utf8Path) -> Result { + let output = root.join(crate::BUILD_OUTPUT); + if !output.is_dir() { + return Ok(Cleaned::default()); + } + + let bytes = size_of(&output); + std::fs::remove_dir_all(&output) + .map_err(|e| ApplyError::Remove(output, e))?; + + Ok(Cleaned { + existed: true, + bytes, + }) +} + +/// How much a directory holds, following nothing. +/// +/// Symlinks are counted as themselves rather than followed: a build that +/// linked to something outside its own output should not have that thing's +/// size attributed to it, and following one out of the tree to measure it +/// would be a strange thing to do on the way to a delete. +fn size_of(path: &Utf8Path) -> u64 { + let Ok(entries) = std::fs::read_dir(path) else { + return 0; + }; + + entries + .flatten() + .map(|entry| { + let Ok(metadata) = entry.metadata() else { + return 0; + }; + if metadata.is_dir() { + Utf8PathBuf::from_path_buf(entry.path()) + .map(|child| size_of(&child)) + .unwrap_or(0) + } else { + metadata.len() + } + }) + .sum() +} diff --git a/vw-sync/tests/sync.rs b/vw-sync/tests/sync.rs new file mode 100644 index 0000000..e7c8720 --- /dev/null +++ b/vw-sync/tests/sync.rs @@ -0,0 +1,607 @@ +// Behaviour of a full synchronization round trip, exercised the way the two +// ends will use it: scan a sender's tree, ask a receiver what it needs, hand +// over exactly that, and make the receiver match. +// +// Both ends are real directories here. The engine is where the correctness +// lives — the HTTP layer above it only moves bytes — so it is worth testing +// against a filesystem rather than against a mock of one. + +use camino::{Utf8Path, Utf8PathBuf}; +use tempfile::TempDir; +use vw_api_types_versions::latest::{CommitResult, Digest, TreeManifest}; +use vw_sync::{apply, clean, clear, missing, scan, Store}; + +/// A sender and a receiver, with somewhere to stage delivered content. +struct Pair { + _dir: TempDir, + sender: Utf8PathBuf, + receiver: Utf8PathBuf, + store: Store, +} + +impl Pair { + fn new() -> Pair { + let dir = TempDir::new().expect("scratch directory"); + let root = Utf8Path::from_path(dir.path()).expect("utf8 temp dir"); + let (sender, receiver, store) = ( + root.join("sender"), + root.join("receiver"), + Store::new(root.join("store")), + ); + std::fs::create_dir_all(&sender).expect("sender root"); + std::fs::create_dir_all(&receiver).expect("receiver root"); + + Pair { + _dir: dir, + sender, + receiver, + store, + } + } + + fn write(&self, path: &str, contents: &str) { + let full = self.sender.join(path); + std::fs::create_dir_all(full.parent().unwrap()).expect("parent"); + std::fs::write(&full, contents).expect("write"); + } + + fn write_receiver(&self, path: &str, contents: &str) { + let full = self.receiver.join(path); + std::fs::create_dir_all(full.parent().unwrap()).expect("parent"); + std::fs::write(&full, contents).expect("write"); + } + + /// One complete synchronization: plan, deliver what is missing, commit. + /// + /// Returns what the commit did and how many blobs went over the wire, + /// which is the number the whole design exists to keep small. + fn sync(&self) -> (CommitResult, usize) { + let manifest = scan(&self.sender).expect("scan sender"); + let plan = + missing(&self.receiver, &self.store, &manifest).expect("plan"); + + for digest in &plan.missing { + self.store + .put(digest, &self.content_for(&manifest, digest)) + .expect("deliver"); + } + + let result = + apply(&self.receiver, &self.store, &manifest).expect("apply"); + (result, plan.missing.len()) + } + + /// The sender's copy of some content, found by digest. + fn content_for(&self, manifest: &TreeManifest, digest: &Digest) -> Vec { + let entry = manifest + .entries + .iter() + .find(|entry| entry.digest == *digest) + .expect("the plan asked for something the manifest names"); + std::fs::read(self.sender.join(&entry.path)).expect("read sender file") + } + + /// Every path on the receiver, so a test can say exactly what is there. + fn receiver_paths(&self) -> Vec { + let mut paths: Vec = scan(&self.receiver) + .expect("scan receiver") + .entries + .into_iter() + .map(|entry| entry.path) + .collect(); + paths.sort(); + paths + } + + fn receiver_contents(&self, path: &str) -> String { + std::fs::read_to_string(self.receiver.join(path)) + .unwrap_or_else(|e| panic!("reading {path}: {e}")) + } +} + +#[test] +fn a_tree_arrives_intact() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("ip/core.xci", ""); + pair.write("vw.toml", "[workspace]"); + + let (result, uploaded) = pair.sync(); + + assert_eq!(uploaded, 3); + assert_eq!( + result, + CommitResult { + created: 3, + updated: 0, + deleted: 0, + unchanged: 0 + } + ); + assert_eq!( + pair.receiver_paths(), + ["hdl/top.vhd", "ip/core.xci", "vw.toml"] + ); + assert_eq!(pair.receiver_contents("hdl/top.vhd"), "entity top is end;"); +} + +#[test] +fn syncing_an_unchanged_tree_sends_nothing() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.sync(); + + // The point of content addressing: a second sync of the same tree is + // entirely talk and no bytes. + let (result, uploaded) = pair.sync(); + + assert_eq!(uploaded, 0); + assert_eq!(result.unchanged, 1); + assert_eq!(result.created + result.updated + result.deleted, 0); +} + +#[test] +fn only_the_edited_file_goes_over_the_wire() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("hdl/other.vhd", "entity other is end;"); + pair.write("vw.toml", "[workspace]"); + pair.sync(); + + pair.write("hdl/top.vhd", "entity top is end; -- edited"); + let (result, uploaded) = pair.sync(); + + assert_eq!(uploaded, 1, "only the edited file should be delivered"); + assert_eq!(result.updated, 1); + assert_eq!(result.unchanged, 2); + assert_eq!( + pair.receiver_contents("hdl/top.vhd"), + "entity top is end; -- edited" + ); +} + +#[test] +fn a_deleted_file_is_deleted_on_the_receiver() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("hdl/gone.vhd", "entity gone is end;"); + pair.sync(); + + std::fs::remove_file(pair.sender.join("hdl/gone.vhd")).expect("remove"); + let (result, _) = pair.sync(); + + // A stale source file is not harmless: it still compiles, and it is a + // baffling way to spend an afternoon. + assert_eq!(result.deleted, 1); + assert_eq!(pair.receiver_paths(), ["hdl/top.vhd"]); +} + +#[test] +fn a_rename_costs_nothing() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("hdl/big.vhd", &"x".repeat(100_000)); + pair.sync(); + + std::fs::rename( + pair.sender.join("hdl/big.vhd"), + pair.sender.join("hdl/renamed.vhd"), + ) + .expect("rename"); + let (result, uploaded) = pair.sync(); + + // The bytes are already on the receiver under the old name, so nothing + // needs to cross the wire — the receiver copies them into place itself. + assert_eq!(uploaded, 0, "a rename should not re-send the content"); + assert_eq!(result.created, 1); + assert_eq!(result.deleted, 1); + assert_eq!(pair.receiver_paths(), ["hdl/renamed.vhd", "hdl/top.vhd"]); + assert_eq!(pair.receiver_contents("hdl/renamed.vhd").len(), 100_000); +} + +#[test] +fn a_whole_directory_can_move_without_resending_it() { + let pair = Pair::new(); + for i in 0..5 { + pair.write(&format!("hdl/old/f{i}.vhd"), &format!("entity f{i};")); + } + pair.sync(); + + std::fs::rename(pair.sender.join("hdl/old"), pair.sender.join("hdl/new")) + .expect("rename directory"); + let (_, uploaded) = pair.sync(); + + assert_eq!(uploaded, 0); + assert_eq!( + pair.receiver_paths(), + [ + "hdl/new/f0.vhd", + "hdl/new/f1.vhd", + "hdl/new/f2.vhd", + "hdl/new/f3.vhd", + "hdl/new/f4.vhd", + ] + ); +} + +#[test] +fn build_output_is_never_sent_or_deleted() { + let pair = Pair::new(); + pair.write("vw.toml", "[workspace]"); + pair.write("hdl/top.vhd", "entity top is end;"); + // What a synthesis run leaves behind on the sender. + pair.write("target/synth/top.dcp", "checkpoint"); + pair.write("driver/target/debug/thing", "binary"); + + let (_, uploaded) = pair.sync(); + assert_eq!(uploaded, 2, "only the two source files"); + + // And what a build on the receiver produces afterwards. A second sync must + // leave it alone: deleting a synthesis run on every keystroke would be a + // remarkable way to lose an afternoon. + pair.write_receiver("target/synth/top.dcp", "receiver checkpoint"); + pair.write_receiver("driver/target/debug/thing", "receiver binary"); + + pair.write("hdl/top.vhd", "entity top is end; -- edited"); + pair.sync(); + + assert_eq!( + pair.receiver_contents("target/synth/top.dcp"), + "receiver checkpoint", + ); + assert_eq!( + pair.receiver_contents("driver/target/debug/thing"), + "receiver binary", + ); +} + +#[test] +fn gitignored_files_are_invisible_at_both_ends() { + let pair = Pair::new(); + pair.write(".gitignore", "*.log\n*.fst\n"); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("vivado.log", "noise"); + pair.write("bench/wave.fst", "waveform"); + + let (_, uploaded) = pair.sync(); + assert_eq!(uploaded, 2, "the ignore file and the source file"); + assert_eq!(pair.receiver_paths(), [".gitignore", "hdl/top.vhd"]); + + // The receiver reads the same rules out of the tree it was handed, so its + // own logs survive a sync rather than being treated as strays. + pair.write_receiver("vivado.log", "receiver noise"); + pair.sync(); + assert_eq!(pair.receiver_contents("vivado.log"), "receiver noise"); +} + +#[test] +fn a_nested_ignore_file_applies_to_its_own_subtree() { + let pair = Pair::new(); + pair.write("driver/.gitignore", "generated\n"); + pair.write("driver/Cargo.toml", "[package]"); + pair.write("driver/generated/bindings.rs", "// generated"); + pair.write("hdl/generated/keep.vhd", "not covered by driver's rules"); + + pair.sync(); + + // git reads nested ignore files as scoped to their directory, and so must + // this — otherwise `driver`'s rules would quietly eat `hdl`'s files. + assert_eq!( + pair.receiver_paths(), + [ + "driver/.gitignore", + "driver/Cargo.toml", + "hdl/generated/keep.vhd", + ] + ); +} + +#[test] +fn the_executable_bit_survives() { + let pair = Pair::new(); + pair.write("tools/build.sh", "#!/bin/sh\necho hi\n"); + pair.write("hdl/top.vhd", "entity top is end;"); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + pair.sender.join("tools/build.sh"), + std::fs::Permissions::from_mode(0o755), + ) + .expect("chmod"); + } + + pair.sync(); + + let manifest = scan(&pair.receiver).expect("scan receiver"); + let script = manifest + .entries + .iter() + .find(|entry| entry.path == "tools/build.sh") + .expect("the script arrived"); + assert!( + script.executable, + "a build script that cannot run is no use" + ); + + let source = manifest + .entries + .iter() + .find(|entry| entry.path == "hdl/top.vhd") + .expect("the source arrived"); + assert!(!source.executable); +} + +#[test] +fn flipping_the_executable_bit_is_a_change() { + let pair = Pair::new(); + pair.write("tools/build.sh", "#!/bin/sh\n"); + pair.sync(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + pair.sender.join("tools/build.sh"), + std::fs::Permissions::from_mode(0o755), + ) + .expect("chmod"); + + let (result, uploaded) = pair.sync(); + + // The content did not change, so nothing needs sending — but the mode + // did, so the file is still rewritten. + assert_eq!(uploaded, 0); + assert_eq!(result.updated, 1); + assert!(scan(&pair.receiver) + .expect("scan") + .entries + .iter() + .any(|entry| entry.path == "tools/build.sh" && entry.executable)); + } +} + +#[test] +fn emptied_directories_do_not_linger() { + let pair = Pair::new(); + pair.write("hdl/keep.vhd", "entity keep is end;"); + pair.write("hdl/doomed/a.vhd", "entity a is end;"); + pair.write("hdl/doomed/b.vhd", "entity b is end;"); + pair.sync(); + + std::fs::remove_dir_all(pair.sender.join("hdl/doomed")).expect("remove"); + pair.sync(); + + assert!( + !pair.receiver.join("hdl/doomed").exists(), + "a directory with no sources left in it should go too", + ); + assert!(pair.receiver.join("hdl").is_dir()); +} + +#[test] +fn a_receiver_with_a_different_tree_is_brought_into_line() { + let pair = Pair::new(); + + // What another machine left behind: some of it right, some stale, some + // never on this sender at all. + pair.write_receiver("hdl/top.vhd", "entity top is end;"); + pair.write_receiver("hdl/stale.vhd", "entity stale is end;"); + pair.write_receiver("old/thing.vhd", "entity thing is end;"); + + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("hdl/new.vhd", "entity new is end;"); + + let (result, uploaded) = pair.sync(); + + // Working from a second machine is not a conflict to resolve — the + // manifest says what should exist, and the receiver ends up saying it. + assert_eq!(uploaded, 1, "only the file the receiver has never seen"); + assert_eq!(result.unchanged, 1); + assert_eq!(result.created, 1); + assert_eq!(result.deleted, 2); + assert_eq!(pair.receiver_paths(), ["hdl/new.vhd", "hdl/top.vhd"]); +} + +#[test] +fn a_manifest_cannot_write_outside_the_tree() { + let pair = Pair::new(); + let store = &pair.store; + + // A manifest arrives over the wire and its paths become filesystem paths. + for path in [ + "../escaped.vhd", + "hdl/../../escaped.vhd", + "/etc/passwd", + "hdl/./top.vhd", + "", + "..", + "C:\\windows\\system32", + ] { + let manifest = TreeManifest { + entries: vec![vw_api_types_versions::latest::FileEntry { + path: path.to_owned(), + digest: vw_sync::digest_bytes(b"payload"), + executable: false, + }], + }; + assert!( + apply(&pair.receiver, store, &manifest).is_err(), + "'{path}' should be refused", + ); + } + + assert!(!pair.receiver.join("../escaped.vhd").exists()); +} + +#[test] +fn content_that_does_not_match_its_digest_is_refused() { + let dir = TempDir::new().expect("scratch directory"); + let store = Store::new( + Utf8Path::from_path(dir.path()).expect("utf8").join("store"), + ); + + let honest = vw_sync::digest_bytes(b"the real thing"); + assert!(store.put(&honest, b"the real thing").is_ok()); + + // Every later lookup goes by digest, so storing content under a digest it + // does not have would poison the store with something wrong under a name + // that looks right. + assert!(store.put(&honest, b"something else entirely").is_err()); + assert_eq!(store.get(&honest).expect("still intact"), b"the real thing"); +} + +#[test] +fn a_digest_cannot_escape_the_store() { + let dir = TempDir::new().expect("scratch directory"); + let root = Utf8Path::from_path(dir.path()).expect("utf8"); + let store = Store::new(root.join("store")); + + // A digest names a file in the store, so it reaches the filesystem. + for hostile in [ + "../../../../etc/passwd", + "..", + "", + "not-hex-at-all", + &"f".repeat(63), + &"F".repeat(64), + ] { + let digest = Digest(hostile.to_owned()); + assert!( + store.put(&digest, b"payload").is_err(), + "'{hostile}' should be refused", + ); + assert!(!store.has(&digest)); + } +} + +#[test] +fn a_commit_with_content_still_undelivered_is_refused() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + + // Committing without having uploaded what the plan asked for should fail + // rather than write a truncated tree. + let manifest = scan(&pair.sender).expect("scan"); + let result = apply(&pair.receiver, &pair.store, &manifest); + + assert!(result.is_err()); + assert!(pair.receiver_paths().is_empty()); +} + +#[test] +fn clearing_makes_the_next_sync_send_everything() { + let pair = Pair::new(); + pair.write("vw.toml", "[workspace]"); + pair.write("hdl/top.vhd", "entity top is end;"); + + let (_, uploaded) = pair.sync(); + assert_eq!(uploaded, 2); + + let cleared = clear(&pair.receiver, &pair.store).expect("clear"); + assert_eq!(cleared.deleted, 2); + assert!(pair.receiver_paths().is_empty()); + + // The point of the whole exercise: with nothing in the tree and nothing + // held, the plan can only ask for all of it. A clear that emptied the tree + // but left the store would have the receiver quietly rebuild from content + // whose trustworthiness was the reason for clearing. + let (result, uploaded) = pair.sync(); + assert_eq!(uploaded, 2, "everything should cross the wire again"); + assert_eq!(result.created, 2); + assert_eq!(result.unchanged, 0); + assert_eq!(pair.receiver_contents("hdl/top.vhd"), "entity top is end;"); +} + +#[test] +fn clearing_leaves_build_output_alone() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.sync(); + + // Hours of synthesis. Forcing a sync is a statement about source, and + // taking the build with it would make `--force` something nobody would + // dare run. + pair.write_receiver("target/synth/top.dcp", "checkpoint"); + pair.write_receiver(".gitignore", "notes.txt"); + pair.write_receiver("notes.txt", "scratch"); + + clear(&pair.receiver, &pair.store).expect("clear"); + + assert_eq!(pair.receiver_contents("target/synth/top.dcp"), "checkpoint"); + assert_eq!(pair.receiver_contents("notes.txt"), "scratch"); +} + +#[test] +fn clearing_a_receiver_that_has_nothing_is_not_an_error() { + // A first sync run with `--force`, and a second force straight after the + // first. Neither has anything to remove, and neither is a mistake. + let pair = Pair::new(); + + let first = + clear(&pair.receiver, &pair.store).expect("clear an empty tree"); + assert_eq!(first.deleted, 0); + + pair.write("hdl/top.vhd", "entity top is end;"); + pair.sync(); + clear(&pair.receiver, &pair.store).expect("clear"); + let again = clear(&pair.receiver, &pair.store).expect("clear again"); + + assert_eq!(again.deleted, 0); +} + +#[test] +fn cleaning_removes_the_build_output_and_nothing_else() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write(".gitignore", "notes.txt"); + pair.sync(); + + // Hours of synthesis, plus something the receiver's own `.gitignore` + // covers — written here rather than synced, because an ignored file is + // invisible to synchronization by design. + pair.write_receiver("target/synth/top.dcp", "checkpoint"); + pair.write_receiver("target/logs/vivado.log", "log"); + pair.write_receiver("notes.txt", "scratch"); + + let cleaned = clean(&pair.receiver).expect("clean"); + + assert!(cleaned.existed); + assert!(cleaned.bytes > 0, "it should have measured what it removed"); + assert!(!pair.receiver.join("target").exists()); + // Source survives: a cleaned tree is one a build starts over in, not one + // that has to be pushed again. + assert_eq!(pair.receiver_contents("hdl/top.vhd"), "entity top is end;"); + assert_eq!(pair.receiver_contents("notes.txt"), "scratch"); +} + +#[test] +fn cleaning_a_tree_with_no_build_output_is_not_an_error() { + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.sync(); + + let cleaned = clean(&pair.receiver).expect("clean"); + + assert!(!cleaned.existed); + assert_eq!(cleaned.bytes, 0); +} + +#[test] +fn a_sync_after_a_clean_sends_nothing() { + // The point of cleaning only `target/`: the source is still there and + // still correct, so the next sync has no work to do. If cleaning took + // source with it, every clean would cost a full re-upload. + let pair = Pair::new(); + pair.write("hdl/top.vhd", "entity top is end;"); + pair.write("vw.toml", "[workspace]"); + pair.sync(); + pair.write_receiver("target/synth/top.dcp", "checkpoint"); + + clean(&pair.receiver).expect("clean"); + let (result, uploaded) = pair.sync(); + + assert_eq!(uploaded, 0, "the source never left"); + assert_eq!(result.unchanged, 2); +} diff --git a/vw-vivado/Cargo.toml b/vw-vivado/Cargo.toml new file mode 100644 index 0000000..00e23eb --- /dev/null +++ b/vw-vivado/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "vw-vivado" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Vivado EDA backend: spawns a long-lived Vivado worker and drives it via the vw-eda wire protocol" + +[dependencies] +vw-eda = { path = "../vw-eda" } +vw-lib = { path = "../vw-lib" } +vw-htcl = { path = "../vw-htcl" } +camino.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +async-trait.workspace = true +tempfile.workspace = true +tracing.workspace = true +portable-pty.workspace = true +similar.workspace = true +colored = "2.0" + +# Unix-only. Used for `libc::kill(pid, SIGINT)` in +# `VivadoBackend::interrupt` — the eval-cancellation path documented on +# that method. Windows falls back to a no-op. +[target.'cfg(unix)'.dependencies] +libc = "0.2" diff --git a/vw-vivado/build.rs b/vw-vivado/build.rs new file mode 100644 index 0000000..f4c7f43 --- /dev/null +++ b/vw-vivado/build.rs @@ -0,0 +1,10 @@ +// Cargo doesn't track files included via `include_str!` for +// rebuild purposes — it only knows about `.rs` source files. +// The Vivado shim is `include_str!`'d into `worker.rs` and +// baked into the binary at compile time; without this build +// script edits to the shim go unnoticed until something else +// triggers a recompile of `vw-vivado`, leaving the deployed +// shim out of sync with the source. +fn main() { + println!("cargo:rerun-if-changed=shim/vivado-shim.tcl"); +} diff --git a/vw-vivado/shim/vivado-shim.tcl b/vw-vivado/shim/vivado-shim.tcl new file mode 100644 index 0000000..35addb5 --- /dev/null +++ b/vw-vivado/shim/vivado-shim.tcl @@ -0,0 +1,1500 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# vw <-> Vivado wire protocol shim. +# +# Sourced by `vivado -mode tcl -source vivado-shim.tcl`. The shim +# connects back to vw over a loopback TCP socket on the port given in +# `$env(VW_PROTOCOL_ADDR)` and then reads newline-delimited JSON +# requests on that socket, writing responses on the same socket. +# +# Why a socket and not stdin/stdout: user TCL is free to call `puts`, +# Vivado prints its own banners and source-echo, and mixing all of +# that with the wire protocol on one stream forces either text +# markers (which a hostile or unlucky `puts` could spoof) or fragile +# OS-specific channels like FIFOs and chardevs. A loopback TCP socket +# works identically on Linux, macOS, and Windows, can't be polluted +# by anything the user writes to stdout, and frees vw to treat +# Vivado's stdout however it wants (forward for `vw run`, capture for +# the REPL/LSP). + +package require json + +namespace eval ::vw { + variable protocol_sock {} + # The eval id currently being processed. `puts` writes that would + # go to stdout while `capturing` is set are forwarded immediately + # as `{"id":N,"stream":"stdout","data":...}` notifications tagged + # with this id, so vw can stream output as it's produced rather + # than waiting for the final response. Required for any + # long-running command (synth_design, route_design, ...). + variable current_eval_id 0 + variable capturing 0 + # Reentrancy guard for the send_msg_id override. Without this, a + # message emitted *during* our own stack-walking or JSON encoding + # would recurse back into the override and either deadlock or + # double-emit. We just fall back to the original handler when + # already inside our wrapper. + variable in_send_msg_id 0 + # Cap on stack frames captured per message. Vivado's internal + # call chains can be 50+ frames deep through tclapp loaders; + # rendering all of them would drown the actual message. The + # cap is per-message, not per-session, so a future deeper trace + # still gets its first N frames. + variable stack_frame_cap 20 + # Set at startup from `VW_TRACE_STACK_CAPTURE`. When true, + # `capture_stack` emits a per-frame `[vw-stack]` log line with + # the info-frame dict, level-args probe, and keep/drop reason + # for every frame it examines. Useful when a warning tag + # renders with fewer frames than expected: the log shows + # exactly which frames existed and why each was kept or + # filtered. Zero cost when unset. + variable trace_stack_capture 0 + catch { + set trace_stack_capture \ + [expr {$::env(VW_TRACE_STACK_CAPTURE) eq "1"}] + } +} + +# ---------- puts capture ---------- +# +# Rename the real `puts` so we can install a wrapper that, while +# capturing, forwards each stdout write to vw as a streaming +# notification. Anything that targets a specific channel (stderr, +# the protocol socket, a file) passes through unchanged. Outside of +# eval (`capturing == 0`), stdout writes also pass through — Vivado's +# own messages between commands stay on the process's stdout where vw +# handles them per its `--verbose` setting. + +rename puts ::vw::real_puts + +proc puts {args} { + set len [llength $args] + set start 0 + set nonewline 0 + if {$len > 0 && [lindex $args 0] eq "-nonewline"} { + set nonewline 1 + set start 1 + } + set remaining [expr {$len - $start}] + if {$::vw::capturing} { + if {$remaining == 1} { + # `puts ?-nonewline? string` — implicit stdout + set str [lindex $args $start] + if {!$nonewline} { append str "\n" } + # Catch-wrap because attach_stack_if_message is + # defined later in this script. During shim sourcing + # there's a tiny window where puts exists (this proc) + # but the helper doesn't yet; we'd rather pass the + # raw string through than crash the puts itself. + catch {set str [::vw::attach_stack_if_message $str 2]} + ::vw::stream_stdout $::vw::current_eval_id $str + return + } elseif {$remaining == 2 \ + && [lindex $args $start] eq "stdout"} { + set str [lindex $args [expr {$start + 1}]] + if {!$nonewline} { append str "\n" } + catch {set str [::vw::attach_stack_if_message $str 2]} + ::vw::stream_stdout $::vw::current_eval_id $str + return + } + } + # Fall through to the real puts for: any non-stdout channel, or + # any stdout write when not capturing. + ::vw::real_puts {*}$args +} + +# ---------- JSON helpers ---------- + +# Hand-encode a string per RFC 8259. Vivado's bundled Tcllib doesn't +# include `json::write`, so we provide the minimum we need. +# +# Implementation: `string map` does the bulk-substitution in one +# native Tcl C call, vs. a per-char Tcl loop (which is what we +# used to do). The difference is dramatic at scale — a 1MB puts +# output (the kind `puts [util::props -object $cpm5]` produces) +# went from minutes of per-char `string index`/`scan`/`switch` +# iteration to ~100ms via `string map`. Rare control chars +# (codepoints < 0x20 other than the named whitespace escapes) +# trigger a slow per-char fallback; in practice Vivado property +# values don't contain them, so the fast path covers everything. +proc ::vw::json_string {value} { + # Order matters: backslash must be substituted FIRST so the + # backslashes we introduce for the other escapes aren't + # themselves re-escaped. + set escaped [string map [list \ + "\\" "\\\\" \ + "\"" "\\\"" \ + "\b" "\\b" \ + "\f" "\\f" \ + "\n" "\\n" \ + "\r" "\\r" \ + "\t" "\\t"] $value] + # Fast path: no remaining control chars → just wrap in quotes. + if {![regexp {[\x00-\x08\x0B\x0E-\x1F]} $escaped]} { + return "\"$escaped\"" + } + # Slow path: per-char loop for the remaining control chars. + # Only hit when the string contains rare control codepoints + # — Vivado property values shouldn't, but a user `puts` of + # binary-ish data might. + set out "\"" + set len [string length $escaped] + for {set i 0} {$i < $len} {incr i} { + set ch [string index $escaped $i] + scan $ch %c codepoint + if {$codepoint < 0x20} { + append out [format "\\u%04x" $codepoint] + } else { + append out $ch + } + } + append out "\"" + return $out +} + +# ---------- response helpers ---------- + +# Send a streaming stdout chunk during eval. These notifications are +# distinguishable from the final response by the presence of a +# `stream` field (and absence of `ok`). +proc ::vw::stream_stdout {id data} { + variable protocol_sock + set j [::vw::json_string $data] + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"stream\":\"stdout\",\"data\":$j}" + flush $protocol_sock +} + +## `pending_response_id` / `pending_response_sent` — main-loop +## deadman for the "request in, no response out" case. The +## dispatch handler sets `pending_response_id` before invoking +## the per-op branch, and every `send_ok`/`send_err` clears it. +## If the main loop exits `::vw::dispatch` with an unclosed id +## (e.g. `interp cancel -unwind` from a Ctrl-C during `place_design` +## blew through the eval-branch's `catch` without letting the +## `send_err` branch run), the loop synthesises a fallback error +## response so the vw Rust worker's `read_response_for(id).await` +## unblocks instead of hanging the whole session. Without this +## guard, the REPL stays stuck at `vivado: running` forever. +variable ::vw::pending_response_id 0 +variable ::vw::pending_response_sent 0 + +proc ::vw::send_ok {id result} { + variable protocol_sock + set j_result [::vw::json_string $result] + # Use real_puts explicitly so the wrapper above can never + # accidentally divert protocol traffic into a stream notification. + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"ok\":true,\"result\":$j_result}" + flush $protocol_sock + if {$::vw::pending_response_id == $id} { + set ::vw::pending_response_sent 1 + } +} + +proc ::vw::send_err {id message {code ""} {info ""}} { + variable protocol_sock + set j_msg [::vw::json_string $message] + set fields "\"message\":$j_msg" + if {$code ne ""} { + append fields ",\"code\":[::vw::json_string $code]" + } + if {$info ne ""} { + append fields ",\"info\":[::vw::json_string $info]" + } + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"ok\":false,\"error\":{$fields}}" + flush $protocol_sock + if {$::vw::pending_response_id == $id} { + set ::vw::pending_response_sent 1 + } +} + +# ---------- shim-initiated RPC ---------- +# +# `vw::rpc_call METHOD [ARGS_JSON]` sends an RPC request to the +# `vw` Rust process and blocks until the response arrives. +# Returns the result value on success, throws with the vw-side +# error message on failure. +# +# Direction is the MIRROR of the eval loop: normally vw sends +# requests and this shim answers. Here the shim sends and vw +# answers. The response uses the same `{id, ok, result}` / +# `{id, ok:false, error}` shape as an eval response — no new +# response type — so it round-trips through the normal +# read/write machinery. +# +# ARGS_JSON is a JSON-encoded string (e.g. `[::json::dict2json +# {name value}]` or a bare literal like `null` / `{}`), NOT a +# Tcl dict. Callers that only need a bare call pass nothing; +# the arg defaults to `null`. +# +# Usage is meant to be called ONLY from inside an eval — during +# an eval we're the sole consumer of the protocol socket, so +# reading the response synchronously via `gets $sock` doesn't +# race the main dispatch loop. Calling from outside an eval +# would race that loop and swallow a real request; the guard +# below errors clearly rather than deadlocking. +variable ::vw::rpc_next_id 1 + +proc ::vw::rpc_call {method {args_json "null"}} { + variable protocol_sock + if {!$::vw::capturing} { + error "vw::rpc_call must be invoked from inside an eval" + } + set id $::vw::rpc_next_id + incr ::vw::rpc_next_id + set j_method [::vw::json_string $method] + ::vw::real_puts $protocol_sock \ + "{\"id\":$id,\"rpc\":true,\"method\":$j_method,\"args\":$args_json}" + flush $protocol_sock + # Read exactly one response line. Repeat if vw sends anything + # else in the meantime (e.g. an unexpected stream chunk); we + # only stop when we see a response tagged with our id. + while {1} { + if {[gets $protocol_sock line] < 0} { + if {[eof $protocol_sock]} { + error "vw::rpc_call: protocol socket closed before response" + } + continue + } + set line [string trim $line] + if {$line eq ""} { continue } + if {[catch {::json::json2dict $line} resp]} { + error "vw::rpc_call: unparseable response from vw: $resp" + } + if {![dict exists $resp id]} { continue } + if {[dict get $resp id] != $id} { continue } + if {[dict exists $resp ok] && [dict get $resp ok]} { + return [expr {[dict exists $resp result] \ + ? [dict get $resp result] : ""}] + } + set msg "" + if {[dict exists $resp error]} { + set err [dict get $resp error] + if {[dict exists $err message]} { + set msg [dict get $err message] + } + } + error "vw::rpc_call: $msg" + } +} + +proc ::vw::log {msg} { + puts stderr "\[vw-shim\] $msg" + flush stderr +} + +# Global-namespace call helper for wrappers. +# +# Each generated `vivado_cmd::` wrapper forwards to the +# underlying Vivado builtin via this proc so the forwarded call +# runs with the interp's current namespace set to `::`. That +# matters for builtins like `synth_ip` which source XDC files +# whose scripts use *unqualified* names (`create_clock`, +# `get_ports`, …). Without a global-namespace call, Tcl resolves +# those unqualified names against whatever the wrapper's own +# namespace happens to be (`::vivado_cmd::`), and picks the +# wrapper again — which then throws on the XDC's positional +# args because kwargs expects `-flag` form. +# +# We define the proc at `::` (via `namespace eval ::`) so its +# execution context is `::`. The body invokes its args via +# `{*}$cmd {*}$args` — a direct arg-expansion, NOT a string +# re-parse. That's the critical difference from +# `namespace eval :: [list …]` / `namespace inscope :: …` / +# `uplevel #0 [list …]`, all of which serialize their args to a +# script string and lose Tcl_Obj internal reps. bd_cell handles +# would round-trip to plain paths like `/cpm5`, which Vivado's +# `set_property -objects` then rejects as "Invalid option value". +namespace eval :: { + proc _vw_global_call {cmd args} { + {*}$cmd {*}$args + } +} + +# ---------- kwargs runtime ---------- +# +# Wrapper procs lowered from htcl declare themselves as +# `proc {args} { ::vw::kwargs $args {param default ...} ; }`. +# This helper parses `args` against `sig` (a dict of `param default +# param default ...`) and uses `upvar 1` to set each parameter as +# a local in the caller's frame. After this returns, the wrapper +# body sees `$dir`, `$cell`, `$name`, etc. just as if they were +# standard Tcl parameters with defaults. +# +# Why this exists: htcl is keyword-only at the call site, but Tcl +# proc dispatch is positional. Without this runtime parsing, the +# only way to make `wrap -name x` work would be to rewrite the +# call site to positional form at compile time — which our lowerer +# used to do, but only for top-level calls, not for calls inside +# proc bodies / namespace eval / [ ... ]. Moving the keyword parse +# to runtime makes every call site work uniformly. +# +# Arg shapes supported: +# `-flag value` — value-bearing flag, sets $flag = value +# `-flag` — bare boolean flag (end of args), sets $flag = 1 +# `-flag -other ...`— bare boolean flag (next token is another +# known flag), sets $flag = 1, continues +# +# The bare-flag heuristic matches Vivado's calling convention: +# their APIs and internal Tcl use `-quiet`/`-verbose`/etc. as bare +# booleans. The "is next token a known flag?" disambiguator avoids +# eating a legitimate value that happens to start with `-` (e.g. +# `-filter -name` where the user intended `-filter` to take `-name` +# as its value — but a leading `-` value is exotic enough that we +# accept the ambiguity). +proc ::vw::kwargs {argv sig} { + # Initialize each parameter to its declared default. Also + # initialize a `__vw_kw__set` flag to 0 — wrappers can + # check this to distinguish "user supplied this arg" from + # "we filled in the default", which matters for + # set_property -dict where setting unsupplied properties + # (with their defaults) re-validates the whole cell and + # rejects values Vivado considers out of range for the + # cell's current state. + foreach {name default} $sig { + upvar 1 $name var + set var $default + upvar 1 __vw_kw_${name}_set seen + set seen 0 + } + set n [llength $argv] + set i 0 + while {$i < $n} { + set flag [lindex $argv $i] + if {![string match -* $flag]} { + error "kwargs: expected -flag, got '$flag'" + } + set key [string range $flag 1 end] + if {![dict exists $sig $key]} { + set allowed [join [dict keys $sig] ", "] + error "kwargs: unknown flag '$flag'; allowed: $allowed" + } + # Decide whether the current flag is bare or takes a value. + # Bare iff: at end of args, OR next token is another known + # -flag. + set bare 1 + set next_i [expr {$i + 1}] + if {$next_i < $n} { + set peek [lindex $argv $next_i] + if {![string match -* $peek]} { + set bare 0 + } else { + set peek_key [string range $peek 1 end] + if {![dict exists $sig $peek_key]} { + # Peek looks like a flag but isn't ours — assume + # it's a value for the current flag (e.g. a CLI + # path or arg starting with `-`). + set bare 0 + } + } + } + upvar 1 $key var + upvar 1 __vw_kw_${key}_set seen + set seen 1 + if {$bare} { + set var 1 + incr i + } else { + set var [lindex $argv $next_i] + incr i 2 + } + } +} + +# ---------- bulk property fetch ---------- +# +# `::vw::props_dict ` returns a paired Tcl list (NAME VAL +# NAME VAL …) of every property on ``. The point is a +# single Vivado RPC instead of N: htcl wrappers that want the +# full property bag (e.g. `util::props`) would otherwise issue +# one `extern::get_property` per property × hundreds of +# properties on an IP cell. The PTY round-trip is the dominant +# cost; doing the iteration entirely Vivado-side cuts it to +# constant per call. +proc ::vw::props_dict {obj} { + set out [list] + foreach name [list_property $obj] { + lappend out $name [get_property $name $obj] + } + return $out +} + +# ---------- user-set property tracking ---------- +# +# Vivado offers no per-property "is this at the default?" API +# accessible from Tcl (`get_property`, `list_property`, +# `report_property -all`, `bd::get_properties` all return every +# property's current value with no user-vs-default distinction). +# The only system-of-record is `write_bd_tcl`'s output, which +# requires a full BD serialization round-trip per query. +# +# Instead we keep our own tally: every time +# `vivado_cmd::set_property` is invoked (the htcl-level chokepoint +# all wrappers and user code go through), it records the +# (object, name, value) triples here. `::vw::user_props_dict` / +# `::vw::user_props_nested` read back from this side-channel — +# returning ONLY the properties the user / wrapper explicitly +# pushed, never the ones Vivado cascaded as derived defaults. +# +# Cost: O(properties set) for record (dict insertion), O(props +# returned) for retrieval (dict walk). No file I/O, no full- +# design serialization. Persists across batches in the worker +# until `:restart`. +# +# Caveat: tracks only properties set via the +# `vivado_cmd::set_property` wrapper. Direct `extern::set_property` +# bypasses the recording. That's by design — the wrappers are +# the documented boundary, and bypassing them is an explicit +# opt-out of the tracking machinery. + +namespace eval ::vw { + variable user_set_props +} + +# Record one or more (name, value) pairs as user-set on `obj`. +# `args` is the paired list (name1 val1 name2 val2 …) — same +# shape the wrapper builds before calling `set_property -dict`. +# Last-set wins per property name within an object. +proc ::vw::record_user_props {obj args} { + variable user_set_props + if {![info exists user_set_props]} { + array set user_set_props {} + } + set key [::vw::_user_props_key $obj] + if {![info exists user_set_props($key)]} { + set user_set_props($key) [dict create] + } + set current $user_set_props($key) + foreach {n v} $args { + dict set current $n $v + } + set user_set_props($key) $current +} + +# Canonical key for the per-object side-channel storage. +# +# `PATH` uniquely identifies a BD cell (`/cips/cpm5` etc.), but +# it's not defined on every Vivado object type — a project-level +# IP handle from `create_ip` / `get_ips` doesn't have one, and +# querying it emits `[Vivado 12-1341] Failed to get property +# 'PATH' on IP 'foo'` via `send_msg_id`. Tcl's `catch` doesn't +# suppress that because Vivado routes the error through the +# message bus rather than returning a Tcl-level error. `-quiet` +# on the getter does suppress it — that's why we use it here. +# +# The lookup falls through PATH → NAME → the raw object string +# so any object type produces a stable key without noise. +proc ::vw::_user_props_key {obj} { + set p "" + catch { set p [get_property -quiet PATH $obj] } + if {$p ne ""} { return $p } + catch { set p [get_property -quiet NAME $obj] } + if {$p ne ""} { return $p } + return $obj +} + +# Parse a `::set_property` argv into (dict-pairs, objects) and +# funnel each object's pairs into [`record_user_props`]. Called by +# the [`install_set_property_recorder`] wrapper before the real +# set_property runs — so the tally reflects what the caller *tried* +# to set even if Vivado later rejects the write. +# +# Recognizes the two forms our IP wrappers actually emit: +# +# set_property -dict {NAME1 VAL1 NAME2 VAL2 …} -objects $cell +# set_property -name NAME -value VAL -objects $cell +# +# Positional-form `set_property NAME VAL $obj` also works. Unknown +# flags (`-quiet`, `-verbose`) are consumed silently. If we can't +# recover both a name/value dict and an object list, we return +# without recording — the write still happens; the tally just +# stays where it was. +proc ::vw::_record_set_property_call {args} { + set pairs [list] + set objects [list] + set positional [list] + set explicit_name "" + set explicit_value "" + set i 0 + while {$i < [llength $args]} { + set tok [lindex $args $i] + switch -- $tok { + -dict { + foreach {n v} [lindex $args [expr {$i + 1}]] { + lappend pairs $n $v + } + incr i 2 + } + -objects { + set objects [lindex $args [expr {$i + 1}]] + incr i 2 + } + -name { + set explicit_name [lindex $args [expr {$i + 1}]] + incr i 2 + } + -value { + set explicit_value [lindex $args [expr {$i + 1}]] + incr i 2 + } + -quiet - + -verbose { + incr i + } + default { + lappend positional $tok + incr i + } + } + } + # Positional shape: `set_property NAME VALUE OBJECTS`. + if {[llength $positional] == 3 && [llength $pairs] == 0 + && $explicit_name eq ""} { + lappend pairs [lindex $positional 0] [lindex $positional 1] + if {[llength $objects] == 0} { + set objects [lindex $positional 2] + } + } + # Trailing positional target: `set_property -dict {…} $cell`. + if {[llength $objects] == 0 && [llength $positional] == 1} { + set objects [lindex $positional 0] + } + if {$explicit_name ne "" && [llength $pairs] == 0} { + lappend pairs $explicit_name $explicit_value + } + if {[llength $pairs] == 0 || [llength $objects] == 0} { + return + } + foreach obj $objects { + catch { ::vw::record_user_props $obj {*}$pairs } + } +} + +# Install a `::set_property` wrapper that funnels every write +# through [`_record_set_property_call`] before delegating to +# whatever `::set_property` currently is (Vivado's C++ builtin, or +# the marker wrapper installed by +# [`install_all_context_wrappers`]). Regeneration-proof — the +# `vivado_cmd::set_property` htcl wrapper is generated from the +# Vivado command reference and re-emitted by `regenerate.sh`, so +# manual edits there don't survive. Keeping the recording here in +# the shim means the side-channel tally stays populated regardless +# of what the generated wrapper looks like. +# +# Called at startup AFTER [`install_all_context_wrappers`] so the +# recorder ends up OUTSIDE the marker wrapper: recording happens +# first, then BEGIN, then the real write, then END. +proc ::vw::install_set_property_recorder {} { + if {[info commands ::vw::orig_set_property_for_record] ne ""} { + return + } + if {[info commands ::set_property] eq ""} { return } + rename ::set_property ::vw::orig_set_property_for_record + proc ::set_property {args} { + catch { ::vw::_record_set_property_call {*}$args } + uplevel 1 [list ::vw::orig_set_property_for_record {*}$args] + } + ::vw::log "installed set_property recorder" +} + +# Return the recorded (name, value) paired list for `obj`. Empty +# list when nothing has been recorded. +proc ::vw::user_props_dict {obj} { + variable user_set_props + if {![info exists user_set_props]} { return [list] } + set key [::vw::_user_props_key $obj] + if {![info exists user_set_props($key)]} { return [list] } + set out [list] + dict for {k v} $user_set_props($key) { + lappend out $k $v + } + return $out +} + +# Same shape as `::vw::props_nested` but seeded from the recorded +# user-set property tally instead of from `list_property` + +# `get_property`. Each value is classified via the structural +# `_lift_value` helper and inserted by dot-split path so the +# result is a nested `Properties` with only the explicitly-set +# sub-keys present. +proc ::vw::user_props_nested {obj} { + set plain [dict create] + foreach {name raw} [::vw::user_props_dict $obj] { + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# `::vw::props_nested ` returns the FULL output `util::props` +# wants — a nested `Properties` dict where dotted property names +# (CONFIG.X.Y) expand into hierarchy, and each leaf value is +# already a `[list Scalar ]` or `[list Nested ]` tuple. +# +# Lives in the shim (plain Tcl) rather than in user-side htcl +# because: +# - One Vivado RPC for the entire fetch + classification + +# nesting pipeline, vs. one RPC for the fetch + thousands +# of htcl-proc kwargs-envelope invocations per recursive +# sub-key for the post-processing. +# - CPM5 has ~200 top-level properties, each whose value is +# itself a paired-dict with dozens of sub-keys. The htcl- +# side post-processing was hitting tens of thousands of +# kwargs invocations × envelope overhead → minutes. Native +# Tcl inside Vivado does the same work in well under a +# second. +# +# The structural classifier (`::vw::_lift_value`) mirrors what +# `lift::lift_recursive` did in user-htcl: pure shape inference, +# no Vivado lookups. The wrap step (`::vw::_wrap_nested`) walks +# the plain nested dict once and tags intermediate levels as +# `Property::Nested(...)`. Leaves already carry their tag from +# `_lift_value`. +proc ::vw::props_nested {obj} { + set plain [dict create] + foreach name [list_property $obj] { + set raw [get_property $name $obj] + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# Structural inference on a raw property value. Returns a +# `[list Scalar v]` or `[list Nested inner]` tuple. Mirror of +# lift::looks_like_paired_dict + lift::lift_recursive in plain +# Tcl with no kwargs envelope. +proc ::vw::_lift_value {raw} { + if {[catch {llength $raw} n]} { return [list Scalar $raw] } + if {$n == 0 || $n % 2 != 0} { return [list Scalar $raw] } + foreach {k _v} $raw { + if {![regexp {^[A-Za-z_][A-Za-z0-9_.]*$} $k]} { + return [list Scalar $raw] + } + } + set inner [dict create] + foreach {k v} $raw { + dict set inner $k [::vw::_lift_value $v] + } + return [list Nested $inner] +} + +# Walk a plain nested Tcl dict and wrap each intermediate +# level as `[list Nested ]`. A value is a leaf when +# it's a 2-element list whose head is "Scalar" or "Nested" +# (the existing Property tuple shape). Anything else is a +# sub-dict to descend into. +proc ::vw::_wrap_nested {plain} { + set out [dict create] + dict for {k v} $plain { + if {[llength $v] == 2 \ + && ([lindex $v 0] eq "Scalar" \ + || [lindex $v 0] eq "Nested")} { + dict set out $k $v + } else { + dict set out $k [list Nested [::vw::_wrap_nested $v]] + } + } + return $out +} + +# `::vw::config_from_dotted_pairs {pairs}` — lift a flat paired- +# list of `dotted.key raw-value` entries into a nested tagged +# `Properties` value. Same transform `::vw::props_nested` +# performs on `list_property` output — split each key on `.`, +# insert at the resulting path in a plain nested dict, `_wrap_nested` +# to tag intermediate levels, `_lift_value` to tag each leaf. +# +# The generated `::configure` procs call this in their bodies +# to convert the assembled `_vw_d` (built by `lappend _vw_d +# CONFIG. ` loops) into the proper `::Config` +# shape: a Properties value where CONFIG at the top wraps a +# `Property::Nested` containing every `` sub-key. Consumers +# then use `dict get [::Config::to -v $cfg] CONFIG` + +# `Property::as_nested -v ...` to extract the sub-tree, matching +# the pattern `props::get` documents. +proc ::vw::config_from_dotted_pairs {pairs} { + set plain [dict create] + foreach {name raw} $pairs { + set leaf [::vw::_lift_value $raw] + dict set plain {*}[split $name "."] $leaf + } + return [::vw::_wrap_nested $plain] +} + +# `::vw::config_to_dotted_flat {nested}` — inverse of the lift. +# Walks a nested tagged Properties tree and emits a flat paired +# list `TOP.LEAF value TOP.LEAF value ...` matching the shape +# Vivado's `set_property -dict` expects for an IP cell. +# +# **Depth invariant.** The generated `::configure` procs +# always assemble `_vw_d` with keys of the form `CONFIG.` +# (one dot at the top, two path segments). `Properties::from_ +# dotted_pairs` splits on `.` and inserts, producing a two-level +# tagged structure: root → Nested-wrapped CONFIG → tagged entries +# (one per Vivado property). +# +# So the flatten pairs off exactly two levels: iterate the root +# dict for TOP keys (`CONFIG`), unwrap that Nested to get the +# entries, then emit `TOP.LEAF = untag(value)` per entry. A Scalar +# entry unwraps to its bare string. A Nested entry — e.g. +# `CONFIG.CPM_CONFIG` where the caller passed a Properties value — +# unwraps to its raw paired-list dict (recursively stripping any +# further tags inside), which is what Vivado stores as the value +# of a nested-dict property. +# +# Naively recursing past the two-level structure would emit +# `CONFIG.CPM_CONFIG.CPM_PCIE0_MODES` which Vivado then rejects +# with `[BD 41-1276] Cannot set the parameter … Parameter does +# not exist`, since CPM_CONFIG is a single property (accepting a +# nested dict value), not a namespace. +proc ::vw::config_to_dotted_flat {nested} { + set out [list] + dict for {top_key top_val} $nested { + set top_tag [lindex $top_val 0] + set top_payload [lindex $top_val 1] + if {$top_tag ne "Nested"} { + # Unexpected shape at the root — configure-built Configs + # always wrap the top namespace as Nested via + # _wrap_nested. Emit under the raw key rather than + # silently drop. + lappend out $top_key [::vw::_untag_recursive $top_val] + continue + } + dict for {leaf_key leaf_val} $top_payload { + lappend out "$top_key.$leaf_key" \ + [::vw::_untag_recursive $leaf_val] + } + } + return $out +} + +# Recursively strip `Property::Scalar` / `Property::Nested` tags +# from a tagged Properties value. Scalar returns its bare string. +# Nested returns its inner dict with each value recursively +# untagged. Used inside `config_to_dotted_flat` to convert a +# Nested-typed property's tagged payload into the raw paired +# dict Vivado expects as that property's value. +proc ::vw::_untag_recursive {tagged} { + if {[llength $tagged] != 2} { return $tagged } + set tag [lindex $tagged 0] + set payload [lindex $tagged 1] + if {$tag eq "Scalar"} { + return $payload + } elseif {$tag eq "Nested"} { + set out [list] + dict for {k v} $payload { + lappend out $k [::vw::_untag_recursive $v] + } + return $out + } + return $tagged +} + +# ---------- send_msg_id override ---------- +# +# Why we override: when Vivado emits a WARNING/ERROR/INFO/CRITICAL +# WARNING via ::common::send_msg_id, the raw line goes to stdout +# with no call-context — the user sees `WARNING: [Common 17-1496] +# ...` and has no way to tell which Tcl proc triggered it. Hooking +# the Tcl entry point lets us capture the call stack at emit time +# and render it as `at file:line in proc` continuation lines. +# +# Tradeoffs to be aware of: +# - The original ::common::send_msg_id is NOT called. That means +# `set_msg_config -id X -suppress` won't suppress Tcl-emitted +# messages (it still works for messages Vivado's C code emits, +# which our PTY-level filter handles). Acceptable for v1; we +# can replicate suppression here if it becomes a real need. +# - Messages emitted from Vivado's C code (synth, route, etc.) +# bypass this override and are caught by the PTY-line filter +# in the worker, with no stack — that's a fundamental limit. + +# True when `str`'s first line looks like a Vivado-standard +# message: starts (after optional leading whitespace) with +# ERROR:/WARNING:/CRITICAL WARNING:/INFO:. Used by the puts +# wrapper to decide whether to attach a stack — we only want +# traces on message-formatted output, not on every `puts hi`. +proc ::vw::is_vivado_message {str} { + set first $str + set nl [string first "\n" $str] + if {$nl >= 0} { + set first [string range $str 0 [expr {$nl - 1}]] + } + set trimmed [string trimleft $first] + if {[string match "ERROR:*" $trimmed]} { return 1 } + if {[string match "CRITICAL WARNING:*" $trimmed]} { return 1 } + if {[string match "WARNING:*" $trimmed]} { return 1 } + if {[string match "INFO:*" $trimmed]} { return 1 } + return 0 +} + +# Severity of a Vivado-style message: one of `ERROR`, `CRITICAL`, +# `WARNING`, `INFO`. Returns empty for non-messages. Used by +# `attach_stack_if_message` to decide whether the stack is worth +# attaching (INFO is suppressed by default — see VW_INFO_WITH_STACK). +proc ::vw::message_severity {str} { + set first $str + set nl [string first "\n" $str] + if {$nl >= 0} { + set first [string range $str 0 [expr {$nl - 1}]] + } + set trimmed [string trimleft $first] + if {[string match "ERROR:*" $trimmed]} { return "ERROR" } + if {[string match "CRITICAL WARNING:*" $trimmed]} { return "CRITICAL" } + if {[string match "WARNING:*" $trimmed]} { return "WARNING" } + if {[string match "INFO:*" $trimmed]} { return "INFO" } + return "" +} + +# If `str` looks like a Vivado-style message, append the current +# Tcl call stack as `\n at ` continuation lines and +# return the augmented string. Otherwise return `str` unchanged. +# +# `skip_caller_frames` tells the stack walk how many wrapper +# layers to step past so the deepest reported frame is the user's +# code, not our shim's plumbing. For the puts wrapper that's 2 +# (this helper + the puts wrapper itself). +proc ::vw::attach_stack_if_message {str skip_caller_frames} { + if {![::vw::is_vivado_message $str]} { + return $str + } + # INFO messages are noisy under heavy Vivado activity (CIPS + # customization emits dozens per call). By default we suppress + # their stack so the scrollback stays scannable. The user opts + # in with `vw repl --info-with-stack` (or the `vw run` flag), + # which sets VW_INFO_WITH_STACK=1 on the spawned process. + # WARNING / ERROR / CRITICAL always keep their stacks. + if {[::vw::message_severity $str] eq "INFO"} { + set env_default 0 + catch { set env_default $::env(VW_INFO_WITH_STACK) } + if {$env_default ne "1"} { + return $str + } + } + set stack [::vw::capture_stack $skip_caller_frames] + if {[llength $stack] == 0} { + return $str + } + set has_trailing_nl 0 + set body $str + if {[string index $str end] eq "\n"} { + set has_trailing_nl 1 + set body [string range $str 0 end-1] + } + foreach frame $stack { + append body "\n at $frame" + } + if {$has_trailing_nl} { append body "\n" } + return $body +} + +# Walk the Tcl call stack starting at the caller of our override +# (`info frame 1` — skipping our wrapper itself) and build a list +# of "at file:line in proc" strings, deepest-first. Uses both +# `info frame` (gives script file/line) and `info level` (gives +# proc name + args) for each depth; merges whatever's available. +# Capped at `$::vw::stack_frame_cap` frames so a 50-deep tclapp +# loader chain doesn't drown the actual message. +# +# Returns at least one entry even when nothing is locatable — +# `(stack: depth=N, no locatable frames)` so the user can +# distinguish "override didn't fire" from "override fired but +# Tcl gave us nothing to render." +proc ::vw::capture_stack {skip_caller_frames} { + variable stack_frame_cap + variable trace_stack_capture + set out [list] + set depth [info frame] + set level_depth [info level] + # Skip our own frame plus whatever the caller asked us to skip. + set start [expr {1 + $skip_caller_frames}] + if {$trace_stack_capture} { + ::vw::log "\[vw-stack\] BEGIN capture skip=$skip_caller_frames\ + depth=$depth level_depth=$level_depth start=$start" + } + for {set i $start} {$i <= $depth} {incr i} { + if {[llength $out] >= $stack_frame_cap} { break } + set frame "" + catch {set frame [info frame -$i]} + # `info level -k` is indexed independently of `info frame` + # — k=0 is the current proc, k=-1 the caller, etc. We map + # frame index i to level index k by clamping; mismatches + # are common (frames can include non-proc evals) but worth + # trying as a fallback. + set level_args "" + set k [expr {$i - $skip_caller_frames - 1}] + if {$k > 0 && $k < $level_depth} { + catch {set level_args [info level -$k]} + } + set entry [::vw::format_frame $frame $level_args] + if {$trace_stack_capture} { + set kept "kept" + if {$entry eq ""} { set kept "dropped" } + ::vw::log "\[vw-stack\] i=$i k=$k $kept frame=\{$frame\}\ + level_args=\{$level_args\}\ + entry=\"$entry\"" + } + if {$entry ne ""} { lappend out $entry } + } + if {$trace_stack_capture} { + ::vw::log "\[vw-stack\] END capture out=\{[join $out { | }]\}" + } + if {[llength $out] == 0} { + lappend out "(stack: info-frame-depth=$depth\ + info-level-depth=$level_depth\ + — no locatable frames; message likely\ + emitted from byte-compiled or C-bridged Tcl)" + } + return $out +} + +# Turn one `info frame` dict (and an optional `info level` args +# list as a fallback proc-name source) into the human-readable +# string we render. Drops frames that have nothing locatable at +# all — they're just noise. +proc ::vw::format_frame {frame level_args} { + set proc "" + catch {set proc [dict get $frame proc]} + set file "" + catch {set file [dict get $frame file]} + set line "" + catch {set line [dict get $frame line]} + set cmd "" + catch {set cmd [dict get $frame cmd]} + # `info level -k` returns the proc invocation as `procname + # arg1 arg2 ...`; the first element is the proc name. + if {$proc eq "" && $level_args ne ""} { + set proc [lindex $level_args 0] + } + + # Drop frames that are part of our own plumbing — they're + # always noise to the user. The signal in a stack trace is + # "which line of MY code led to this message"; frames in + # the shim file, the ::vw:: namespace, our send_msg_id + # override, or the ::log:: helpers are all infrastructure. + if {[string match "*vivado-shim.tcl" $file]} { + return "" + } + if {[string match "::vw::*" $proc]} { + return "" + } + if {$proc eq "::common::send_msg_id"} { + return "" + } + if {[string match "::log::*" $proc]} { + return "" + } + + set location "" + if {$file ne "" && $line ne ""} { + set location "${file}:${line}" + } elseif {$line ne ""} { + # `eval` frames without a source file — common for our + # `uplevel #0 $tcl` shim entry — still tell the user + # "line N of the script you submitted." + set location ":${line}" + } + if {$location ne "" && $proc ne ""} { + return "${location} in ${proc}" + } elseif {$location ne ""} { + return $location + } elseif {$proc ne ""} { + return $proc + } elseif {$cmd ne ""} { + # Last-ditch: no proc and no location, but we know what + # command this frame was running. Truncate so a very long + # command doesn't blow out the trace. + set short [string range $cmd 0 80] + if {[string length $cmd] > 80} { append short "..." } + return "(cmd: $short)" + } + return "" +} + +# Severity normalizer. Vivado is inconsistent about case and +# uses underscores in CRITICAL_WARNING; we normalize to the same +# uppercase, space-separated form the PTY-line classifier expects +# so the worker can route warnings/errors to the right StreamKind. +proc ::vw::normalize_severity {sev} { + set s [string toupper [string trim $sev]] + switch -- $s { + "CRITICAL_WARNING" - + "CRITICAL WARNING" { return "CRITICAL WARNING" } + "ERROR" - + "FATAL" - + "FATAL_ERROR" { return "ERROR" } + "WARNING" { return "WARNING" } + "INFO" - + "STATUS" { return "INFO" } + default { return $s } + } +} + +# Install our wrapper *after* Vivado has had a chance to define +# ::common::send_msg_id. If the proc doesn't exist yet (very early +# init, headless mode without the common namespace), we silently +# skip — Vivado's PTY emission still works, just without our +# stack capture. +# +# Logs status once per successful install and once per skipped +# attempt (with the reason), so the user can see in the REPL +# whether the override is live without enabling --verbose. +proc ::vw::install_send_msg_override {} { + if {[info commands ::vw::orig_send_msg_id] ne ""} { + # Already installed — silent on the retry path so we don't + # spam the log on every eval. + return + } + set candidates [info commands ::common::send_msg*] + if {[info commands ::common::send_msg_id] eq ""} { + ::vw::log "::common::send_msg_id not present;\ + ::common::send_msg* = {$candidates};\ + stack-capture override NOT installed" + return + } + rename ::common::send_msg_id ::vw::orig_send_msg_id + + # The Vivado-Tcl signature is `send_msg_id id severity msg + # [optional args]`. We accept the same. + proc ::common::send_msg_id {id severity msg args} { + # Reentrancy guard — if our stack walk somehow triggers + # another send_msg_id, fall back to the original. + if {$::vw::in_send_msg_id} { + return [uplevel 1 [list ::vw::orig_send_msg_id \ + $id $severity $msg {*}$args]] + } + set ::vw::in_send_msg_id 1 + set ok [catch { + set sev_norm [::vw::normalize_severity $severity] + # INFO is noisy — suppress the stack by default, matching + # the puts-wrapper path. The user opts in with `vw repl + # --info-with-stack` (worker exports VW_INFO_WITH_STACK=1). + # WARNING / ERROR / CRITICAL always keep their stacks. + set attach_stack 1 + if {$sev_norm eq "INFO"} { + set env_default 0 + catch { set env_default $::env(VW_INFO_WITH_STACK) } + if {$env_default ne "1"} { set attach_stack 0 } + } + set out "${sev_norm}: \[${id}\] ${msg}" + if {$attach_stack} { + # Skip 1 caller frame so the deepest frame in the + # rendered stack is the one that called send_msg_id, + # not the user proc that called our wrapper. + set stack [::vw::capture_stack 1] + foreach frame $stack { + append out "\n at ${frame}" + } + } + if {$::vw::capturing} { + ::vw::stream_stdout $::vw::current_eval_id "$out\n" + } else { + # Outside an eval — fall back to the original so the + # message still appears wherever Vivado normally + # would have put it. + ::vw::orig_send_msg_id $id $severity $msg {*}$args + } + } err] + set ::vw::in_send_msg_id 0 + if {$ok != 0} { + # Our override threw — never let that prevent Vivado from + # at least seeing the message. Fall through to original. + ::vw::log "send_msg_id override failed: $err" + return [uplevel 1 [list ::vw::orig_send_msg_id \ + $id $severity $msg {*}$args]] + } + } + ::vw::log "installed send_msg_id override" +} + +# Commands wrapped with [`install_command_context`] so any +# traceless WARNINGs / ERRORs Vivado's C++ side emits during the +# call get the Tcl call stack attached. Each name MUST be a global +# Tcl command (no leading `::` — the wrapper installs as `::$name`). +# The list is open-ended: add a command here whenever a user hits a +# new noisy builtin and the worker filter shows the warning landing +# without a stack. There's no observable cost — the wrapper is a +# thin around-trace, only the message-tagging window is widened. +set ::vw::context_wrapped_commands { + set_property + generate_netlist_ip +} + +# Wrap a single Vivado command so we can attach the Tcl call stack +# to warnings/errors its C++ implementation emits. The C++ paths +# (notably `[IP_Flow 19-7090] Invalid parameter '…' provided, +# Ignoring` for `set_property`, `[Coretcl 2-176] No IPs found` for +# `generate_netlist_ip`) bypass `::common::send_msg_id` and write +# directly through Vivado's internal message bus to the PTY — +# there's no Tcl frame to grab by the time the bytes arrive at the +# Rust worker. So we capture the stack here, while the Tcl +# interpreter is *about* to enter the C++ command, emit it as a +# marker the worker recognizes and strips, then the worker tags any +# warnings that arrive while the marker is active. Markers go via +# `::vw::real_puts stdout` so they bypass our own `puts` override +# and land on the PTY directly. Idempotent — re-running this on +# every eval is harmless once the wrapper is in place. +proc ::vw::install_command_context {name} { + set orig "::vw::orig_${name}_for_ctx" + if {[info commands $orig] ne ""} { + return + } + if {[info commands ::$name] eq ""} { return } + rename ::$name $orig + # Build the wrapper body with `$orig` interpolated, NOT + # `$name` — the wrapper has to forward to the renamed original + # without a name-lookup detour. `set rc` is computed and + # forwarded so the wrapped command's return value, error code, + # and -errorinfo all flow back unchanged to the caller. + proc ::$name {args} [string map [list @ORIG@ $orig] { + # Skip 1 = this wrapper's own frame, so the deepest reported + # frame is the user proc that called the wrapped command. + set frames [::vw::capture_stack 1] + ::vw::emit_pty_ctx_begin $frames + set rc [catch { + uplevel 1 [list @ORIG@ {*}$args] + } result options] + ::vw::emit_pty_ctx_end + return -options $options $result + }] + ::vw::log "installed context wrap for ::$name" +} + +# Install context wrappers for every command in +# `::vw::context_wrapped_commands`. Called once after the protocol +# socket opens and again at the top of every eval — each +# `install_command_context` is idempotent, so re-attempts are cheap +# once installed and recover gracefully when a command first +# appears after a later library was sourced. +proc ::vw::install_all_context_wrappers {} { + foreach name $::vw::context_wrapped_commands { + catch {::vw::install_command_context $name} + } +} + +# Push a context marker onto the PTY. Format: a sentinel-prefixed +# line per frame plus begin/end bookends, so the Rust PTY filter +# can match line-by-line without needing a base64 decoder. +proc ::vw::emit_pty_ctx_begin {frames} { + ::vw::real_puts stdout "__VW_CTX_BEGIN__" + foreach f $frames { + ::vw::real_puts stdout "__VW_CTX_FRAME__:$f" + } + ::vw::real_puts stdout "__VW_CTX_READY__" + flush stdout +} + +proc ::vw::emit_pty_ctx_end {} { + ::vw::real_puts stdout "__VW_CTX_END__" + flush stdout +} + +# ---------- user-proc body wrap ---------- +# +# Traceless warnings from Vivado's C++ (e.g. `[Coretcl 2-176] No +# IPs found`) bypass `::common::send_msg_id`, so the per-command +# wrappers above only tag warnings emitted while THAT specific +# command is in flight. Real-world Vivado calls are deeper — a +# warning inside `generate_netlist_ip` may actually come from a +# nested `get_ips` call inside the C++ path — and enumerating +# every possible culprit isn't tractable. +# +# So we also instrument every USER-defined proc: rewrite each new +# proc's body to emit a marker BEGIN/READY at entry (with +# `capture_stack` from *inside* the body — the frame stack +# includes the proc itself) and an END on exit. The Rust +# marker-stack tracks nesting, so nested user procs each get their +# own frames and the innermost wins for tagging. Result: a warning +# fired anywhere under `configure_clock`'s call chain lands with +# `at ip/clock.htcl:N in ::configure_clock` attached, even when +# Vivado's C++ emits it silently. +# +# Only fires while `::vw::capturing == 1` — i.e. during a user +# eval — so Vivado's own Tcl-lib initialization defines procs +# unchanged. `::vw::*` procs are also skipped so our own plumbing +# doesn't recurse into itself. + +# Compute the fully-qualified name a `proc NAME ...` invocation +# would produce, given the caller's namespace. Used by the +# ::proc override's filter — we only wrap user procs that end up +# in the top-level `::` namespace, which is where htcl-lowered +# procs live. +proc ::vw::qualify_proc_name {name caller_ns} { + if {[string match ::* $name]} { return $name } + set caller_ns [string trimright $caller_ns ::] + if {$caller_ns eq ""} { return ::$name } + return ${caller_ns}::$name +} + +# Install the ::proc override. Idempotent — re-running is cheap +# once the wrapper is in place. The original `proc` is renamed to +# `::vw::orig_proc_for_body_wrap` and delegated to. +## Install a proc WITHOUT the body-wrap. Called by codegen-emitted +## infrastructure (enum variant constructors, `::to_raw` / +## `::from_raw` lifts, list/dict monomorphized repr walkers, +## primitive-type identity procs) where the wrap's per-call +## `capture_stack` + `emit_pty_ctx_begin/end` cost dwarfs the actual +## work — a proc that only does `return [list Scalar $v]` shouldn't +## take 100μs of instrumentation per call, and calling it 200K +## times during collection construction pushes total overhead into +## the multi-second range for no debug benefit. +## +## Named `codegen_proc` (not `proc_no_wrap` or similar) so the +## intent is clear at the emit site: this is for COMPILER-EMITTED +## infrastructure, not for user-authored procs the user might want +## to opt out of instrumentation on. Per-user opt-out — if we ever +## need it — is a separate mechanism. +## +## Behavior when the body-wrap isn't installed yet (very early +## shim startup, before `install_proc_body_wrap` runs): falls +## through to the raw `::proc` builtin. Codegen shipped before the +## wrap install still produces working procs; they don't gain the +## wrap retroactively once it installs, which is exactly the point +## — they opted out. +proc ::vw::codegen_proc {name spec body} { + if {[info commands ::vw::orig_proc_for_body_wrap] ne ""} { + uplevel 1 [list ::vw::orig_proc_for_body_wrap $name $spec $body] + } else { + uplevel 1 [list ::proc $name $spec $body] + } +} + +proc ::vw::install_proc_body_wrap {} { + if {[info commands ::vw::orig_proc_for_body_wrap] ne ""} { + return + } + rename ::proc ::vw::orig_proc_for_body_wrap + # Marker template: `@BODY@` gets literal-substituted with the + # user's body via `string map`, avoiding format/subst + # interpolation risks. `catch` preserves rc/result/errorcode/ + # errorinfo across the wrap so the wrapped proc behaves + # identically to the unwrapped original. + variable proc_body_template { + ::vw::emit_pty_ctx_begin [::vw::capture_stack 0] + set _vw_ctx_rc [catch {@BODY@} _vw_ctx_result _vw_ctx_opts] + ::vw::emit_pty_ctx_end + return -options $_vw_ctx_opts $_vw_ctx_result + } + ::vw::orig_proc_for_body_wrap ::proc {name spec body} { + # Delegate straight through when we're not inside a user + # eval — Vivado's own lib procs go untouched. + if {!$::vw::capturing} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + set caller_ns [uplevel 1 { namespace current }] + set qualified [::vw::qualify_proc_name $name $caller_ns] + # Skip our own helpers and any Vivado internal proc that a + # user eval might reach into. `::vw::*` covers our shim, + # `::tcl::*` guards Tcl's core, everything else in the + # top-level or user namespaces gets the wrap. + # + # Also skip codegen-emitted infrastructure procs — the + # compiler-generated per-type `repr` / `to` / `from` / + # `to_raw` / `from_raw` lifts, `tag` / `payload` / + # `empty` helpers, and the top-level `::putr`. Each of + # these is a tiny passthrough proc where the per-call + # `capture_stack` + `emit_pty_ctx_begin/end` cost + # dominates: a `list` return with a million + # elements calls `bd_pin::repr` a million times, and a + # `Properties`-typed proc constructing 200K entries calls + # `Property::Scalar` 200K times. The raw log for those + # cases shows the wait period is almost entirely + # `__VW_CTX_BEGIN__` / `__VW_CTX_END__` marker traffic, + # not the actual work. Stack context on a proc that just + # does `return [list Scalar $v]` adds nothing. + # + # Enum variant constructors (`Property::Scalar` etc.) + # have no distinctive name pattern, so we detect them + # structurally via the body-shape regex below. + if {[string match ::vw::* $qualified] + || [string match ::tcl::* $qualified] + || [string match *::repr $qualified] + || [string match *::to $qualified] + || [string match *::from $qualified] + || [string match *::to_raw $qualified] + || [string match *::from_raw $qualified] + || [string match *::tag $qualified] + || [string match *::payload $qualified] + || [string match *::empty $qualified] + || $qualified eq "::putr"} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + # Enum-variant-constructor detection: body is exactly + # `return [list ]` (empty payload) or + # `return [list $v]` (single-payload). This is the + # canonical shape `emit_constructor` in vw-htcl emits, and + # a user writing that body verbatim would get their proc + # silently un-instrumented — an acceptable false-positive + # since (a) no stack trace context is genuinely useful for + # a one-liner list-wrapper, and (b) the pattern is narrow + # enough to be near-impossible to hit by accident. + if {[regexp \ + {^\s*return\s+\[list\s+[A-Za-z_][A-Za-z0-9_]*(?:\s+\$v)?\]\s*$} \ + $body]} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + # Detect a re-wrap: if the body already contains our + # marker call, don't stack another layer around it. Redefs + # of a user proc during eval (e.g. re-`src`ing a library) + # would otherwise nest wrappers on every reload. + if {[string first "::vw::emit_pty_ctx_begin" $body] >= 0} { + return [uplevel 1 [list ::vw::orig_proc_for_body_wrap \ + $name $spec $body]] + } + set new_body [string map \ + [list @BODY@ $body] $::vw::proc_body_template] + uplevel 1 [list ::vw::orig_proc_for_body_wrap $name $spec $new_body] + } + ::vw::log "installed ::proc body wrap" +} + +# ---------- dispatch ---------- + +proc ::vw::dispatch {line} { + if {[catch {::json::json2dict $line} req]} { + ::vw::send_err 0 "protocol parse error: $req" + return + } + if {![dict exists $req id] || ![dict exists $req op]} { + ::vw::send_err 0 "missing id or op" + return + } + set id [dict get $req id] + set op [dict get $req op] + # Arm the main-loop deadman as soon as we've committed to a + # specific id — from here on, if this dispatch doesn't reach + # a `send_ok` / `send_err`, the main loop synthesises an + # error response for `id` so the client never blocks. + set ::vw::pending_response_id $id + set ::vw::pending_response_sent 0 + switch -- $op { + eval { + if {![dict exists $req tcl]} { + ::vw::send_err $id "eval request missing tcl field" + return + } + set tcl [dict get $req tcl] + set ::vw::current_eval_id $id + set ::vw::capturing 1 + set rc [catch {uplevel #0 $tcl} result opts] + set ::vw::capturing 0 + if {$rc != 0} { + set ecode "" + set einfo "" + catch {set ecode [dict get $opts -errorcode]} + catch {set einfo [dict get $opts -errorinfo]} + ::vw::send_err $id $result $ecode $einfo + } else { + ::vw::send_ok $id $result + } + } + shutdown { + ::vw::send_ok $id "" + ::vw::log "shim shutting down" + exit 0 + } + default { + ::vw::send_err $id "unknown op: $op" + } + } +} +#---- diagnostic levels ---- + +# Failure to assign a block design address space WARN -> CRIT +set_msg_config -id {[BD 5-700]} -new_severity {CRITICAL WARNING} + +# ---------- main ---------- + +if {![info exists ::env(VW_PROTOCOL_ADDR)]} { + ::vw::log "VW_PROTOCOL_ADDR not set; exiting" + exit 1 +} + +if {![regexp {^(.*):(\d+)$} $::env(VW_PROTOCOL_ADDR) -> ::vw::host ::vw::port]} { + ::vw::log "invalid VW_PROTOCOL_ADDR: $::env(VW_PROTOCOL_ADDR)" + exit 1 +} + +if {[catch {socket $::vw::host $::vw::port} sock]} { + ::vw::log "failed to connect to $::vw::host:$::vw::port: $sock" + exit 1 +} + +set ::vw::protocol_sock $sock +fconfigure $sock -buffering line -translation lf + +::vw::log "connected to $::vw::host:$::vw::port" + +# Try installing the send_msg_id override now. If Vivado hasn't +# defined ::common::send_msg_id yet (unusual but possible in +# headless / minimal-mode configurations), the override will be +# re-attempted on the first eval — it's idempotent. +catch {::vw::install_send_msg_override} +catch {::vw::install_all_context_wrappers} +catch {::vw::install_set_property_recorder} +catch {::vw::install_proc_body_wrap} + +# Silence Vivado's per-command performance report — the +# `: Time (s): cpu = … Memory (MB): peak = …` chatter that +# appears after any command whose elapsed time exceeds +# `tcl.statsThreshold` seconds (default: quite low, so most +# `create_bd_cell` calls trip it). Users who WANT the stats can +# re-lower the threshold in their own script; setting it high +# by default keeps the REPL / `vw run` output focused on the +# user's actual results. +catch {set_param tcl.statsThreshold 9999999} + +while {1} { + if {[gets $sock line] < 0} { + if {[eof $sock]} { + ::vw::log "protocol socket closed; exiting" + break + } + continue + } + set line [string trim $line] + if {$line eq ""} { continue } + # Retry installs on each eval until they succeed — both procs + # bail out cheaply once installed. + catch {::vw::install_send_msg_override} + catch {::vw::install_all_context_wrappers} +catch {::vw::install_set_property_recorder} +catch {::vw::install_proc_body_wrap} + # Arm the response deadman. `::vw::dispatch` sets + # `pending_response_id` as soon as it parses the request id; + # `send_ok` / `send_err` clear `pending_response_sent`. Wrap + # the dispatch in `catch` so an `interp cancel -unwind` + # ripping through the eval-branch's own `catch` doesn't kill + # the main loop — we synthesise a fallback error response + # and keep serving requests. See the `pending_response_*` + # variable docs. + set ::vw::pending_response_id 0 + set ::vw::pending_response_sent 0 + catch {::vw::dispatch $line} dispatch_result dispatch_opts + set dispatch_rc [dict get $dispatch_opts -code] + # Restore capturing to a sane state — the eval-branch always + # clears it, but if `catch` unwound before the clear line ran + # it might still be 1 and would misroute the next eval's puts. + set ::vw::capturing 0 + if {$::vw::pending_response_id != 0 && !$::vw::pending_response_sent} { + set fallback_msg "shim dispatch aborted without sending a response" + if {$dispatch_rc != 0} { + append fallback_msg " (rc=$dispatch_rc): $dispatch_result" + } else { + append fallback_msg " (dispatch returned but did not send)" + } + ::vw::log "deadman firing for id=$::vw::pending_response_id: $fallback_msg" + ::vw::send_err $::vw::pending_response_id $fallback_msg + } +} + +close $sock +exit 0 diff --git a/vw-vivado/src/handlers.rs b/vw-vivado/src/handlers.rs new file mode 100644 index 0000000..9c01072 --- /dev/null +++ b/vw-vivado/src/handlers.rs @@ -0,0 +1,1375 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Shared RPC handlers wired into every Vivado spawn (`vw run`, +//! `vw repl`, `vw test`). Each method here answers a `vw::` htcl +//! proc whose answer lives on the tool side rather than in Vivado. +//! +//! Methods: +//! - `workspace_root` — the discovered `vw.toml` parent dir. +//! - `top` — resolved top-entity name for the current session. +//! Precedence: active variant's `top` field wins, then the +//! workspace-level `top`. Returns the empty string when +//! neither is configured so htcl callers can branch on +//! `[vw::top]` without try/catch. Consumed by `vw::synth` +//! (as the fallback when the caller omits `-top`) and by +//! `vw::_resolve_top` (used by `vw::place` / `vw::route` / +//! `vw::report`). +//! - `critical_warning_count` — session-scoped monotonic count of +//! CRITICAL WARNING chunks the stream sink has classified so far. +//! `vw::synth` / `vw::place` snapshot this before + after each +//! phase to decide whether to persist a checkpoint (a CW delta +//! means the artifact would poison the next warm run). Vivado's +//! own `get_msg_config -count -severity` doesn't see CWs from +//! opt/place/phys_opt sub-processes, so we track them ourselves. +//! - `diff_files` — read two files and return a unified diff of +//! their contents. Used by `test::assert_file_eq` to render a +//! readable failure message instead of a two-line "files differ" +//! note. +//! - `vhdl_dependency_sources` — every VHDL file shipped by any +//! transitive dep (regular `[dependencies]` only), grouped by +//! target library. Consumed by `design.htcl` to feed +//! `read_vhdl -library …`. +//! - `vhdl_dependency_sources_with_test` — same, but the entry +//! workspace's `[test-dependencies]` are also included. +//! Consumed by `test/*.htcl` where test-deps are in scope. +//! - `vhdl_design_sources` — every VHDL file under +//! `/hdl/`. Consumed alongside dependency sources +//! to compile the workspace's own design. +//! - `vhdl_ip_sources` — every generated IP wrapper under +//! `/target/ip/**/*.vhd`. Populated by +//! `vw::make_wrapper`. Kept separate from design sources +//! because wrappers have their own regen lifecycle and +//! typically compile into a distinct library (`ip`). +//! - `design_constraints` — every Vivado constraint file under +//! `/constraints/**/*.{xdc,sdc}`. Fed to `read_xdc` +//! during synth prep. +//! - `design_synth_constraints` / `design_place_constraints` / +//! `design_route_constraints` — phase-scoped variants that +//! walk `constraints/synth/`, `constraints/place/`, +//! `constraints/route/` respectively. Used to attach USED_IN +//! flags to `read_xdc` so route-only constraints don't apply +//! during synthesis (and vice versa). +//! - `synth_needs_update` — content-hash comparison between the +//! checkpoint's sidecar manifest and the current tracked source +//! set (design VHDL, IP wrappers, synth XDC, workspace `.htcl`, +//! vw.toml, vw.lock). Returns `true` when the checkpoint OR +//! manifest is missing OR the fingerprints disagree. Backs +//! the `vw::synth` cache path. +//! - `synth_mark_checkpoint` — writes the sidecar manifest for a +//! freshly-produced checkpoint. Called after +//! `vivado_cmd::write_checkpoint` so the next invocation can +//! compare fingerprints and skip resynth on unchanged sources. +//! - `mark_project_configured` — records that the on-disk Vivado +//! project at `/target/vw-project//` has been +//! successfully configured against the current +//! `/ip/**/*.htcl` + `vw.toml` fingerprint. Called by +//! `vw::configure_ip` after `ip::configure` + `save_project` +//! complete. The freshness check itself runs Rust-side before +//! spawning Vivado (see `vw_lib::project_needs_wipe`), so there +//! is no matching `project_needs_update` RPC — invalidation +//! happens once per session, not per htcl call. +//! - `place_needs_update` / `place_mark_checkpoint` — same +//! shape again but scoped to the place stage. Fingerprint +//! covers `/constraints/place/**` plus the upstream synth +//! DCP file (which proxies for "everything synth depended +//! on" — if any synth-scope source changed, synth re-ran +//! and the DCP is fresh, invalidating place). Backs +//! `vw::place`. +//! - `route_needs_update` / `route_mark_checkpoint` — one stage +//! down from place. Fingerprint covers +//! `/constraints/route/**` plus the upstream place DCP +//! file (proxy chain: any synth-scope change → synth re-ran → +//! place re-ran → place DCP fresh → route invalidates). Backs +//! `vw::route`. +//! - `compile_htcl_module` — parses + lowers an htcl module (any +//! `src`-shaped path resolved against the workspace root) and +//! returns the concatenated Tcl. `vw::configure_ip` uses this +//! to auto-load `/ip/module.htcl` when `::ip::configure` +//! isn't already defined, so a `src ip` in `design.htcl` isn't +//! a hidden requirement. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; +use std::time::SystemTime; + +use serde_json::Value; + +use crate::rpc::{FnHandler, RpcHandler}; + +/// Shared "already loaded in this Vivado session" map. Populated +/// by the CLI / REPL after each successful htcl load with the +/// paths + mtimes of every file that was shipped to Vivado. +/// Consulted by [`compile_htcl_module`] so on-demand +/// module compilation skips re-shipping files whose procs are +/// already installed — which is both a big perf win and, more +/// importantly, KEEPS the RPC path from re-loading dep files that +/// design.htcl never touched. Concretely: `design.htcl`'s +/// `src @vw` pulls @vw + @vivado-cmd; a workspace `ip/cips.htcl` +/// also does `src @cpm5` / `src @cips` / `src @clk-wizard`; those +/// three are NOT in the initial session because design.htcl +/// never reached them, and the auto-load path must actually +/// compile + ship them the first time. See the safety note on +/// [`SharedPreload`] for how the map's invariants are enforced. +pub type PreloadedPaths = HashMap; + +/// Shared handle callers hold to update the "already-loaded" +/// map after each session commit. The RPC handler holds the same +/// Arc so reads see the most recent update without any explicit +/// message passing. +/// +/// **Invariant**: only entries added to this map after Vivado +/// has finished evaling the corresponding Tcl are safe to skip. +/// Adding a path prematurely would cause `compile_htcl_module` +/// to omit content Vivado hasn't seen yet, leading to +/// `invalid command name` errors at runtime. The REPL updates +/// after every batch's `EvalDone`; the CLI updates once at the +/// end of the initial load. +pub type SharedPreload = Arc>; + +/// Session-scoped monotonic counter of CRITICAL WARNING (and +/// higher) chunks the stream classifier has seen. Vivado's own +/// `get_msg_config -count -severity "CRITICAL WARNING"` doesn't +/// see CWs emitted from sub-processes launched inside `opt_design` +/// / `place_design` / `phys_opt_design` (documented gotcha), so +/// htcl procs like `vw::synth` and `vw::place` need OUR counter +/// to decide whether to persist a checkpoint. Owned by whichever +/// call site sets up the stream sink (vw-cli / vw-repl); the RPC +/// handler holds the same Arc so `vw::critical_warning_count` +/// reads see the latest value with no message-passing. +pub type SharedCriticalWarningCount = Arc; + +/// Build the FnHandler used by every Vivado spawn. `workspace_root` +/// is the workspace-root path to serve for `vw::workspace_root`; +/// pass `None` when the caller couldn't discover one (the RPC +/// method then returns an error instead of a bogus path). +pub fn make_handler(workspace_root: Option) -> Arc { + make_handler_with_variant(workspace_root, None) +} + +/// Like [`make_handler`] but also carries a session-scoped active +/// variant name — the value the CLI's `--variant ` flag +/// picked. When present, RPC methods that filter by variant +/// (currently `vhdl_design_sources`) fall back to this instead +/// of the workspace default. Explicit per-call `variant` kwargs +/// still take precedence. +pub fn make_handler_with_variant( + workspace_root: Option, + active_variant: Option, +) -> Arc { + let preloaded: SharedPreload = Arc::new(RwLock::new(HashMap::new())); + make_handler_with_preloaded(workspace_root, active_variant, preloaded) +} + +/// Convenience over [`make_handler_full`] that allocates a fresh +/// (unshared) CW counter. Suitable for callers that don't need to +/// read the counter themselves (e.g. `vw test` — where each test +/// spawns its own worker and only cares about eval results, not +/// CW-gated checkpoint decisions). +pub fn make_handler_with_preloaded( + workspace_root: Option, + active_variant: Option, + preloaded: SharedPreload, +) -> Arc { + let cw = Arc::new(std::sync::atomic::AtomicU64::new(0)); + make_handler_full(workspace_root, active_variant, preloaded, cw) +} + +/// Full-detail constructor: carries every shared piece of state +/// the RPC surface needs. `preloaded` is the "already loaded in +/// this Vivado session" map [`compile_htcl_module`] consults — +/// callers who want on-demand htcl loading to skip files already +/// shipped to this Vivado session should clone the returned map's +/// Arc, hold onto it, and update it after every successful load. +/// `cw_count` is the session's monotonic CRITICAL WARNING counter +/// — callers that own the stream sink should clone this Arc and +/// bump it on every `Severity::CriticalWarning` chunk, and htcl +/// procs read it via the `critical_warning_count` RPC. +pub fn make_handler_full( + workspace_root: Option, + active_variant: Option, + preloaded: SharedPreload, + cw_count: SharedCriticalWarningCount, +) -> Arc { + let workspace_root = workspace_root.map(Arc::new); + let active_variant = active_variant.map(Arc::new); + FnHandler::new(move |method: String, args: Value| { + let ws = workspace_root.clone(); + let av = active_variant.clone(); + let pl = preloaded.clone(); + let cw = cw_count.clone(); + async move { + dispatch( + &method, + args, + ws.as_deref().map(|p| p.as_ref()), + av.as_deref().map(|s| s.as_str()), + &pl, + &cw, + ) + .await + } + }) +} + +async fn dispatch( + method: &str, + args: Value, + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + preloaded: &SharedPreload, + cw_count: &SharedCriticalWarningCount, +) -> Result { + match method { + "workspace_root" => workspace_root + .map(|p| Value::String(p.to_string_lossy().into_owned())) + .ok_or_else(|| { + "no workspace root: entry file has no `vw.toml` in its \ + parent chain" + .to_string() + }), + "critical_warning_count" => Ok(Value::from( + cw_count.load(std::sync::atomic::Ordering::Relaxed), + )), + "active_variant" => { + Ok(active_variant_value(workspace_root, active_variant)) + } + "top" => Ok(top_value(workspace_root, active_variant)), + "project_name" => project_name_value(workspace_root), + "diff_files" => diff_files(args), + "vhdl_dependency_sources" => { + vhdl_dependency_sources( + workspace_root, + /*include_test=*/ false, + extract_exclude_sim_only(&args), + ) + .await + } + "vhdl_dependency_sources_with_test" => { + vhdl_dependency_sources( + workspace_root, + /*include_test=*/ true, + extract_exclude_sim_only(&args), + ) + .await + } + "vhdl_design_sources" => vhdl_design_sources( + workspace_root, + extract_variant(&args).or_else(|| active_variant.map(String::from)), + ), + "vhdl_ip_sources" => vhdl_ip_sources(workspace_root), + "design_constraints" => design_constraints(workspace_root), + "design_synth_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Synth) + } + "design_place_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Place) + } + "design_route_constraints" => { + design_phase_constraints(workspace_root, ConstraintPhase::Route) + } + "synth_needs_update" => { + synth_needs_update(workspace_root, active_variant, args) + } + "synth_mark_checkpoint" => { + synth_mark_checkpoint(workspace_root, active_variant, args) + } + "mark_project_configured" => { + mark_project_configured(workspace_root, args) + } + "place_needs_update" => place_needs_update(workspace_root, args), + "place_mark_checkpoint" => place_mark_checkpoint(workspace_root, args), + "route_needs_update" => route_needs_update(workspace_root, args), + "route_mark_checkpoint" => route_mark_checkpoint(workspace_root, args), + "compile_htcl_module" => { + compile_htcl_module(workspace_root, args, preloaded).await + } + other => Err(format!("unknown RPC method: {other}")), + } +} + +/// `compile_htcl_module` — inputs `{path: ""}` where +/// `` follows the same resolution rules as an htcl `src` +/// statement (relative path, absolute path, `@dep/…`, or +/// directory-as-module). Loads the entry file + every transitive +/// import, lowers each command to Tcl, and returns the +/// concatenated Tcl string. +/// +/// Callers `eval` the returned string to install everything the +/// module exports into the current interpreter. Used by +/// `vw::configure_ip` to auto-load `/ip/module.htcl` when +/// `::ip::configure` isn't already defined — so the user doesn't +/// have to remember to `src ip` in their `design.htcl` before +/// calling the wrapper. +/// +/// The pipeline mirrors what `vw run` / `vw repl` do internally +/// (parse → sig-table → per-command lowering → extern rewrite) +/// but skips overload dispatchers, putr rewrites, and origin +/// markers. Those matter for user-facing tracebacks and +/// interactive `putr` — neither is needed for on-demand loading +/// of a well-formed module. +async fn compile_htcl_module( + workspace_root: Option<&std::path::Path>, + args: Value, + preloaded: &SharedPreload, +) -> Result { + // Extract owned inputs so the closure passed to + // `spawn_blocking` doesn't borrow anything from this async + // frame. `path` becomes a String, the preload map is + // snapshotted here (short critical section on the RwLock — + // NOT held across the compile). + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "compile_htcl_module: args must be an object with a `path` \ + string field" + .to_string() + })?; + let path: String = obj + .get("path") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| { + "compile_htcl_module: missing string `path`".to_string() + })?; + let preloaded_snapshot: std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + > = preloaded.read().map(|g| g.clone()).unwrap_or_default(); + let ws_owned = ws.clone(); + + // Offload the heavy work — file I/O, htcl parse, per-command + // lowering — to the blocking thread pool. Without this, the + // async frame runs to completion on the current runtime + // thread and starves every other task (in particular the + // REPL's 250ms redraw tick, which is why the input timer + // appeared to freeze during long compiles). Only mouse + // movement was un-freezing it because a mouse event would + // hit the `crossterm_events` branch of the select! and + // trigger a draw as a side effect of handling the event. + tokio::task::spawn_blocking(move || { + compile_htcl_module_blocking(ws_owned, path, preloaded_snapshot) + }) + .await + .map_err(|e| format!("compile_htcl_module: join error: {e}"))? +} + +/// Sync core of `compile_htcl_module` — runs on the tokio +/// blocking thread pool so the async runtime stays responsive. +/// All heavy work (loader recursion, parse, lower, extern +/// rewrite, disk writes) lives here. +/// +/// Disk cache: after a successful compile, writes a manifest at +/// `/target/.vw-compile-.tcl.manifest` +/// listing every loaded file's path + mtime. On subsequent +/// invocations, if the manifest exists AND every file's current +/// mtime matches, returns the cached `.tcl` verbatim in +/// milliseconds instead of re-running the loader. +/// +/// The cache is invalidated by any file's mtime changing OR the +/// preload set changing (folded into the manifest fingerprint). +/// Not invalidated by ADDING new files (a new `.htcl` under `ip/` +/// wouldn't be in the manifest); users adding sources should +/// `rm target/.vw-compile-*` to force recompile. +fn compile_htcl_module_blocking( + ws: camino::Utf8PathBuf, + path: String, + preloaded_snapshot: std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> Result { + // Compute cache paths early so we can short-circuit on hit. + let dbg_name = format!( + ".vw-compile-{}.tcl", + path.replace(['/', '\\'], "_").replace('@', "at_"), + ); + let target_dir = ws.join("target"); + let cache_tcl = target_dir.join(&dbg_name); + let cache_manifest_name = format!("{dbg_name}.manifest"); + let cache_manifest = target_dir.join(&cache_manifest_name); + + // Cache hit path: manifest exists AND every listed file's + // current mtime matches. The manifest also embeds a hash of + // the preload snapshot's paths (sorted) — if the caller now + // has a different set of files preloaded, the compile output + // would differ, so we invalidate on that too. + if let Some(cached) = try_load_cached_compile( + cache_manifest.as_std_path(), + cache_tcl.as_std_path(), + &preloaded_snapshot, + ) { + return Ok(Value::String(cached)); + } + // Build the resolver with the workspace's transitive deps plus + // the Cargo-parity self-injection. Mirrors vw-cli's + // `load_htcl_program_with_mode` at a minimum — we don't need + // test deps here (the auto-load path never fires from inside + // a test). + let mut resolver = vw_htcl::Resolver::new(); + if let Ok(paths) = vw_lib::transitive_dep_cache_paths(&ws) { + for (name, cache_path) in paths { + resolver = resolver.with_dep(name, cache_path); + } + } + if let Ok(cfg) = vw_lib::load_workspace_config(&ws) { + resolver = resolver.with_dep_if_absent( + cfg.workspace.name.clone(), + ws.as_std_path().to_path_buf(), + ); + } + + // Resolve the import path against the workspace root (that's + // how `design.htcl` would resolve `src ip`). Directory-as- + // module + `.htcl` extension logic is inside `resolve()`. + let entry = resolver + .resolve(ws.as_std_path(), &path) + .map_err(|e| format!("compile_htcl_module: {e}"))?; + + // Preload: skip files the caller has already shipped to this + // Vivado session. + // + // Correctness rule: only files that are actually installed + // in the Vivado interp belong in this map. An earlier version + // of this code preloaded every `.htcl` under every dep root + // the resolver knew about — that was wrong because a + // workspace can declare deps (e.g. `@cpm5` / `@cips` / + // `@clk-wizard` in metroid) that `design.htcl`'s entry + // graph never actually pulls in. Those procs weren't + // installed at startup, and preloading their files caused + // `invalid command name "cpm5::cpm_pcie0"` at runtime. + // Trusting the caller-populated map avoids that class of + // bug entirely. + let mut noop = NoopLoadObserver; + let program = vw_htcl::loader::load_with_preloaded( + &entry, + &resolver, + &mut noop, + &preloaded_snapshot, + ) + .map_err(|e| format!("compile_htcl_module: loading {path}: {e}"))?; + + // Parse the flattened source once, then build the same set + // of auxiliary tables vw-cli's runner uses. Enum-prelude and + // overload-dispatcher emission both consult these — skipping + // them means user procs referencing an enum namespace (e.g. + // `proc Property::as_nested`) or a monomorphized generic + // repr (e.g. `dict_string_Property::repr`) reach Tcl before + // the namespace exists and error with `unknown namespace`. + let parsed = vw_htcl::parser::parse(&program.source); + let mut _ignored: Vec = Vec::new(); + let enum_decl_table = + vw_htcl::build_enum_decl_table(&parsed.document, &mut _ignored); + let type_decl_table = + vw_htcl::build_type_decl_table(&parsed.document, &mut _ignored); + let type_decl_names: std::collections::HashSet = + type_decl_table.keys().cloned().collect(); + let (_full_sigs, overload_table) = + vw_htcl::build_signature_table_with_overloads( + &parsed.document, + &type_decl_names, + &mut _ignored, + ); + let table = vw_htcl::signature_table(&parsed.document); + + let mut out = String::new(); + // Preludes first — namespace/proc definitions the lowered + // commands below depend on. Order matches vw-cli's runner: + // primitives create root type namespaces, enum preludes + // create per-enum namespaces + variant constructors, and + // overload dispatchers create the switch-arm procs. + // + // All three emissions are idempotent (Tcl `namespace eval X + // {}` on an existing X is a no-op, `proc` redefinition + // replaces). Safe to re-ship even when design.htcl already + // installed the same preludes at startup. + for p in vw_htcl::emit_primitive_prelude() { + out.push_str(&p); + out.push('\n'); + } + for ed in enum_decl_table.values() { + let prelude = vw_htcl::emit_enum_prelude(ed); + if !prelude.trim().is_empty() { + out.push_str(&prelude); + out.push('\n'); + } + } + for info in overload_table.values() { + let dispatcher = vw_htcl::emit_dispatcher(info); + if !dispatcher.trim().is_empty() { + out.push_str(&dispatcher); + out.push('\n'); + } + } + + // Now the lowered user commands. Same per-statement lowering + // vw-cli does (minus putr rewrites and origin markers — this + // path never fires from an interactive `putr` and the caller + // already has an origin frame from `vw::configure_ip`). + // + // Critically: proc declarations for overload specializations + // get lowered under a MANGLED internal name (e.g. + // `Property::as_nested::v_Property::Scalar`) so the + // dispatchers emitted above can route to the right variant. + // Without this, both overloads of `Property::as_nested` + // land at the same unmangled name and the second definition + // silently shadows the first via Tcl proc redefinition — + // which produces `called on Scalar value` errors when the + // dispatcher expects the mangled variants to exist. + // + // Perf: use the `_with_putr_and_index` / `_with_name_and_index` + // variants and pass a PRE-BUILT `LineIndex`. The non-`_index` + // variants rebuild a LineIndex per call — an O(source_size) + // newline scan. For a 16 MB / 13k-statement compile that was + // O(stmts × source_size) quadratic and dominated wall-clock + // at ~100s. With a shared index the loop is O(stmts × avg + // command body) which finishes in a couple of seconds. + let line_index = vw_htcl::LineIndex::new(&program.source); + let empty_putr: std::collections::HashMap = + std::collections::HashMap::new(); + for stmt in &parsed.document.stmts { + let vw_htcl::ast::Stmt::Command(cmd) = stmt else { + continue; + }; + let lowered = match overload_specialization_mangle(cmd, &overload_table) + { + Some(mangled) => { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + unreachable!( + "overload_specialization_mangle already \ + validated this is a Proc" + ) + }; + vw_htcl::lower_proc_decl_with_name_and_index( + proc, + &program.source, + &table, + Some(&mangled), + &empty_putr, + &line_index, + ) + } + None => vw_htcl::lower_command_with_putr_and_index( + cmd, + &program.source, + &table, + &empty_putr, + &line_index, + ), + }; + // `extern::name` → `::name` so wrapper bodies that forward + // via `extern::` reach Vivado as bare native names — same + // rewrite vw-cli applies before shipping to the backend. + let tcl = vw_htcl::rewrite_externs(&lowered).text; + if !tcl.trim().is_empty() { + out.push_str(&tcl); + out.push('\n'); + } + } + + // Persist the cache: compiled Tcl + a manifest listing every + // loaded file's path + mtime + a hash of the preload set. + // Next invocation short-circuits if all mtimes match AND the + // preload hash matches (same set of files already-loaded). + // Silent on write errors — cache misses on next run are + // annoying but not incorrect. + let _ = std::fs::create_dir_all(target_dir.as_std_path()); + let _ = std::fs::write(cache_tcl.as_std_path(), &out); + let _ = write_compile_manifest( + cache_manifest.as_std_path(), + &program, + &preloaded_snapshot, + ); + + Ok(Value::String(out)) +} + +/// Manifest file format (plain text): +/// ```text +/// preload-hash +/// +/// +/// ... +/// ``` +/// One line per loaded file. `preload-hash` is FNV-1a over the +/// sorted list of preload paths so a caller-side change to +/// what's already-loaded invalidates the cache too. +fn write_compile_manifest( + manifest_path: &std::path::Path, + program: &vw_htcl::LoadedProgram, + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> std::io::Result<()> { + use std::fmt::Write; + let mut body = String::new(); + let ph = preload_fingerprint(preloaded); + writeln!(body, "preload-hash {ph}").ok(); + for f in &program.files { + let Some(mtime) = f.mtime else { continue }; + let Ok(dur) = mtime.duration_since(std::time::UNIX_EPOCH) else { + continue; + }; + writeln!(body, "{} {}", dur.as_nanos(), f.path.display()).ok(); + } + std::fs::write(manifest_path, body) +} + +/// Try to serve a compile from the cache. Returns `Some(tcl)` iff +/// - the manifest exists, +/// - every listed file's current mtime matches the recorded one, +/// - AND the recorded preload-hash matches the current one. +/// +/// Any mismatch (or missing file, or unreadable cache) → `None`, +/// and the caller falls through to a fresh compile. +fn try_load_cached_compile( + manifest_path: &std::path::Path, + cache_path: &std::path::Path, + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> Option { + let manifest = std::fs::read_to_string(manifest_path).ok()?; + let mut lines = manifest.lines(); + let ph_line = lines.next()?; + let ph_str = ph_line.strip_prefix("preload-hash ")?; + let stored_ph: u64 = ph_str.parse().ok()?; + if stored_ph != preload_fingerprint(preloaded) { + return None; + } + for line in lines { + let mut parts = line.splitn(2, ' '); + let stored_ns: u128 = parts.next()?.parse().ok()?; + let path = std::path::Path::new(parts.next()?); + let meta = std::fs::metadata(path).ok()?; + let mtime = meta.modified().ok()?; + let current_ns = + mtime.duration_since(std::time::UNIX_EPOCH).ok()?.as_nanos(); + if current_ns != stored_ns { + return None; + } + } + // All checks pass — serve the cached Tcl. + std::fs::read_to_string(cache_path).ok() +} + +/// FNV-1a over the sorted preload path set. Deliberately ignores +/// mtimes — the file-mtime check above already covers content +/// changes for preloaded files; this fingerprint only detects +/// changes to WHICH files are preloaded (a different caller state). +fn preload_fingerprint( + preloaded: &std::collections::HashMap< + std::path::PathBuf, + std::time::SystemTime, + >, +) -> u64 { + let mut paths: Vec<&std::path::Path> = + preloaded.keys().map(|p| p.as_path()).collect(); + paths.sort(); + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for p in paths { + for b in p.to_string_lossy().as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + // Separator so `/a/b` + `/c` doesn't collide with `/a` + `/b/c`. + h ^= 0xff; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// `mark_project_configured` — inputs `{name: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the current `/ip/**/*.htcl` + `vw.toml` fingerprint next +/// to the on-disk Vivado project's `.xpr` at +/// `/target/vw-project//.xpr.manifest`. Called +/// by `vw::configure_ip` after `ip::configure` and `save_project` +/// both succeed. +/// +/// The `name` arg is what Vivado's `[current_project]` returns — +/// canonically the workspace name that seeded [`AutoProject`]. +/// We take it as an arg rather than deriving it here so htcl +/// stays authoritative about the project name (and there's no +/// silent divergence if the workspace ever ships multiple named +/// projects). The freshness *check* runs Rust-side once per +/// session before Vivado spawns (see `vw_lib::project_needs_wipe` +/// consumed from `vw-cli` / `vw-repl`), so there is intentionally +/// no matching `project_needs_update` RPC. +fn mark_project_configured( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "mark_project_configured: args must be an object with a `name` \ + string field" + .to_string() + })?; + let name = obj.get("name").and_then(Value::as_str).ok_or_else(|| { + "mark_project_configured: missing string `name` field".to_string() + })?; + let project_dir = vw_lib::vw_project_dir(&ws); + vw_lib::write_project_manifest(&ws, project_dir.as_std_path(), name) + .map_err(|e| format!("writing project manifest: {e}"))?; + Ok(Value::Null) +} + +/// `place_needs_update` — inputs +/// `{checkpoint: , synth_checkpoint: }`; +/// output is a JSON bool. Compares the place checkpoint's +/// sidecar manifest against the current fingerprint of every +/// place-scoped XDC under `/constraints/place/**` PLUS the +/// synth DCP file (which proxies for "everything synth +/// depended on"). `true` when the checkpoint / manifest is +/// missing or the fingerprints disagree. +fn place_needs_update( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "place_needs_update")?; + let synth_checkpoint = + extract_synth_checkpoint_arg(&args, "place_needs_update")?; + let needs = vw_lib::place_needs_update( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&synth_checkpoint), + ) + .map_err(|e| format!("checking place checkpoint freshness: {e}"))?; + Ok(Value::Bool(needs)) +} + +/// `place_mark_checkpoint` — inputs +/// `{checkpoint: , synth_checkpoint: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the current place-scope fingerprint. Called by `vw::place` +/// after `vivado_cmd::write_checkpoint`. +fn place_mark_checkpoint( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "place_mark_checkpoint")?; + let synth_checkpoint = + extract_synth_checkpoint_arg(&args, "place_mark_checkpoint")?; + vw_lib::write_place_checkpoint_manifest( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&synth_checkpoint), + ) + .map_err(|e| format!("writing place checkpoint manifest: {e}"))?; + Ok(Value::Null) +} + +/// Sibling of [`extract_checkpoint_arg`] for the `synth_checkpoint` +/// slot on place-stage RPC methods. Kept as a distinct helper so +/// the error message names the exact missing key. +fn extract_synth_checkpoint_arg( + args: &Value, + method: &str, +) -> Result { + let obj = args.as_object().ok_or_else(|| { + format!( + "{method}: args must be an object with a `synth_checkpoint` \ + string field" + ) + })?; + obj.get("synth_checkpoint") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("{method}: missing string `synth_checkpoint`")) +} + +/// `route_needs_update` — inputs +/// `{checkpoint: , place_checkpoint: }`; +/// output is a JSON bool. Compares the route checkpoint's +/// sidecar manifest against the current fingerprint of every +/// route-scoped XDC under `/constraints/route/**` PLUS the +/// place DCP file (which proxies for "everything place depended +/// on"). `true` when the checkpoint / manifest is missing or the +/// fingerprints disagree. +fn route_needs_update( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "route_needs_update")?; + let place_checkpoint = + extract_place_checkpoint_arg(&args, "route_needs_update")?; + let needs = vw_lib::route_needs_update( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&place_checkpoint), + ) + .map_err(|e| format!("checking route checkpoint freshness: {e}"))?; + Ok(Value::Bool(needs)) +} + +/// `route_mark_checkpoint` — inputs +/// `{checkpoint: , place_checkpoint: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the current route-scope fingerprint. Called by `vw::route` +/// after `vivado_cmd::write_checkpoint`. +fn route_mark_checkpoint( + workspace_root: Option<&std::path::Path>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let checkpoint = extract_checkpoint_arg(&args, "route_mark_checkpoint")?; + let place_checkpoint = + extract_place_checkpoint_arg(&args, "route_mark_checkpoint")?; + vw_lib::write_route_checkpoint_manifest( + &ws, + std::path::Path::new(&checkpoint), + std::path::Path::new(&place_checkpoint), + ) + .map_err(|e| format!("writing route checkpoint manifest: {e}"))?; + Ok(Value::Null) +} + +/// Sibling of [`extract_synth_checkpoint_arg`] for the +/// `place_checkpoint` slot on route-stage RPC methods. +fn extract_place_checkpoint_arg( + args: &Value, + method: &str, +) -> Result { + let obj = args.as_object().ok_or_else(|| { + format!( + "{method}: args must be an object with a `place_checkpoint` \ + string field" + ) + })?; + obj.get("place_checkpoint") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("{method}: missing string `place_checkpoint`")) +} + +/// Silent [`vw_htcl::loader::LoadObserver`] — the RPC path has no +/// progress bar or CLI channel to talk to. Every callback stays +/// at the default no-op impl. +struct NoopLoadObserver; +impl vw_htcl::loader::LoadObserver for NoopLoadObserver {} + +/// Mirror of `vw-cli`'s `overload_specialization_mangle` and +/// `vw-repl/src/lower.rs`'s equivalent. If `cmd` is a top-level +/// `proc` whose name is an overload public name AND whose first +/// arg annotation is a qualified enum variant, return the +/// mangled internal name to emit it under. The dispatcher +/// (produced by `emit_dispatcher`) routes calls to these mangled +/// names by argument type. Skipping this in `compile_htcl_module` +/// caused both overloads of a proc to collapse onto the same +/// unmangled name — the second one silently shadowed the first +/// via Tcl proc redefinition, and the dispatcher's runtime +/// switch never found either specialization. +fn overload_specialization_mangle( + cmd: &vw_htcl::Command, + overloads: &vw_htcl::OverloadTable, +) -> Option { + let vw_htcl::CommandKind::Proc(proc) = &cmd.kind else { + return None; + }; + let name = proc.name.as_deref()?; + if !overloads.contains_key(name) { + return None; + } + let sig = proc.signature.as_ref()?; + let first = sig.args.first()?; + let vw_htcl::TypeExpr::Qualified { variant, .. } = + first.type_annotation.as_ref()? + else { + return None; + }; + Some(vw_htcl::mangle_specialization(name, variant)) +} + +/// Shared arg extractor for the checkpoint-scoped RPC methods. +/// Every one takes `{checkpoint: }` and errors +/// identically on missing/wrong-typed input — factoring keeps +/// the message shape consistent across +/// `synth_*` / `ip_*` handlers. +fn extract_checkpoint_arg( + args: &Value, + method: &str, +) -> Result { + let obj = args.as_object().ok_or_else(|| { + format!( + "{method}: args must be an object with a `checkpoint` \ + string field" + ) + })?; + obj.get("checkpoint") + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("{method}: missing string `checkpoint`")) +} + +/// `synth_mark_checkpoint` — inputs `{checkpoint: }`; +/// output is a JSON null. Writes the sidecar manifest recording +/// the tracked source set's current fingerprint next to the +/// checkpoint. Called by `vw::synth` immediately after +/// `vivado_cmd::write_checkpoint` succeeds so the next invocation +/// can compare fingerprints and skip resynthesis when the sources +/// are unchanged. +fn synth_mark_checkpoint( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "synth_mark_checkpoint: args must be an object with a \ + `checkpoint` string field" + .to_string() + })?; + let checkpoint = + obj.get("checkpoint") + .and_then(Value::as_str) + .ok_or_else(|| { + "synth_mark_checkpoint: missing string `checkpoint`".to_string() + })?; + vw_lib::write_synth_checkpoint_manifest( + &ws, + std::path::Path::new(checkpoint), + active_variant, + ) + .map_err(|e| format!("writing checkpoint manifest: {e}"))?; + Ok(Value::Null) +} + +/// `synth_needs_update` — inputs `{checkpoint: }`; output +/// is a JSON bool. Delegates to [`vw_lib::synth_needs_update`], +/// which stats the checkpoint against the tracked source set +/// (design VHDL, IP wrappers, synth XDC, workspace htcl, +/// vw.toml, vw.lock). Missing checkpoint → `true`; any source +/// strictly newer than the checkpoint → `true`; otherwise `false`. +/// +/// The active variant is threaded through so a variant-specific +/// design surface (which vw::synth already respects via +/// `vhdl_design_sources`) is used for the mtime scan too — +/// otherwise a variant-owned file change might invalidate a +/// checkpoint that doesn't actually include it. +fn synth_needs_update( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, + args: Value, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let obj = args.as_object().ok_or_else(|| { + "synth_needs_update: args must be an object with a `checkpoint` \ + string field" + .to_string() + })?; + let checkpoint = + obj.get("checkpoint") + .and_then(Value::as_str) + .ok_or_else(|| { + "synth_needs_update: missing string `checkpoint`".to_string() + })?; + let needs = vw_lib::synth_needs_update( + &ws, + std::path::Path::new(checkpoint), + active_variant, + ) + .map_err(|e| format!("checking checkpoint freshness: {e}"))?; + Ok(Value::Bool(needs)) +} + +/// `vhdl_dependency_sources` — return every transitive-dep VHDL +/// file, grouped by target library, as a JSON object of the shape +/// `{"library_name": ["/abs/path/a.vhd", "/abs/path/b.vhd"], …}`. +/// Grouped-by-library because the primary consumer is a Vivado +/// `read_vhdl -library $files` loop — the tuple-per-file +/// shape would force the caller to bucket, which we can do here. +async fn vhdl_dependency_sources( + workspace_root: Option<&std::path::Path>, + include_test: bool, + exclude_sim_only: bool, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + // Auto-fetch missing git deps. `workspace_has_unlocked_git_deps` + // is a cheap disk-only check — no network — so the common + // "already up-to-date" path stays cheap. When it returns + // true we invoke the same fetch machinery `vw update` uses; + // downstream enumeration then sees a fully-populated + // `vw.lock` and `~/.vw/deps` layout. + let unlocked = vw_lib::workspace_has_unlocked_git_deps(&ws, include_test) + .map_err(|e| format!("checking lockfile: {e}"))?; + if unlocked { + tracing::info!( + workspace = %ws, + "auto-updating workspace: git deps missing from vw.lock", + ); + // Look up netrc credentials the same way `vw update` does + // so a workspace with private git deps (e.g. an + // organization's internal GitHub repo, gitea, …) doesn't + // 401 when auto-update kicks in. `None` is fine when + // every git URL is public — libgit2 falls back to + // unauthenticated clone. + let creds = + vw_lib::get_access_credentials_for_workspace(&ws, include_test); + vw_lib::update_workspace_with_token(&ws, creds) + .await + .map_err(|e| format!("auto-updating workspace: {e}"))?; + } + let sources = vw_lib::vhdl_dependency_sources_ext( + &ws, + include_test, + exclude_sim_only, + ) + .map_err(|e| format!("enumerating VHDL dep sources: {e}"))?; + let mut by_library: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for src in sources { + by_library + .entry(src.library) + .or_default() + .push(Value::String(src.path.to_string_lossy().into_owned())); + } + let obj: serde_json::Map = by_library + .into_iter() + .map(|(lib, files)| (lib, Value::Array(files))) + .collect(); + Ok(Value::Object(obj)) +} + +/// `vhdl_design_sources` — return every VHDL file under +/// `/hdl/` as a JSON array of absolute-path strings. +/// Empty array when the workspace has no `hdl/` dir yet. +fn vhdl_design_sources( + workspace_root: Option<&std::path::Path>, + variant: Option, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + // When no variant was passed but the workspace declares + // variants, fall back to the workspace default so the flow + // "just works" from `design.htcl` without an explicit + // selector. The design-sources filter uses the resolved + // name to keep only shared + active-variant files. + let resolved = match variant { + Some(name) => Some(name), + None => workspace_default_variant_name(&ws), + }; + let paths = + vw_lib::vhdl_design_sources_for_variant(&ws, resolved.as_deref()) + .map_err(|e| format!("enumerating VHDL design sources: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Look up the workspace's default variant name. Returns `None` +/// when the workspace has no variants OR the variants block is +/// malformed (no default flag on a multi-entry list). Errors +/// are swallowed here — the caller has already produced a +/// diagnostic through the check machinery; we don't want the +/// RPC path to surface the same problem twice. +fn workspace_default_variant_name(ws: &camino::Utf8Path) -> Option { + let cfg = vw_lib::load_workspace_config(ws).ok()?; + cfg.workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.name.clone()) +} + +/// `project_name` — return the `[workspace] name` field of the +/// entry workspace's `vw.toml`, as a JSON string. Errors when no +/// workspace can be discovered; the shim propagates that as a +/// normal Tcl error so htcl callers can branch on it. +/// +/// Used by `vw::synth` and other library procs that want to tag +/// log messages with the project name (`log::info -id [vw::project_name] +/// -msg "…"`) without hard-coding it in every entry file. +fn project_name_value( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let cfg = vw_lib::load_workspace_config(&ws) + .map_err(|e| format!("failed to load workspace config at {ws}: {e}"))?; + Ok(Value::String(cfg.workspace.name.clone())) +} + +/// `active_variant` — return the name of the variant driving this +/// Vivado session, as a JSON string. Session-scoped precedence: +/// the CLI's `--variant ` selector wins; when unset, the +/// workspace default is used; when the workspace declares no +/// variants at all, the empty string is returned so htcl callers +/// can branch on `[vw::active_variant]` without try/catch. +fn active_variant_value( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, +) -> Value { + if let Some(name) = active_variant { + return Value::String(name.to_string()); + } + let ws = workspace_root + .and_then(|p| camino::Utf8PathBuf::from_path_buf(p.to_path_buf()).ok()); + let name = ws + .as_deref() + .and_then(workspace_default_variant_name) + .unwrap_or_default(); + Value::String(name) +} + +/// `top` — return the resolved top-entity name for the current +/// session, as a JSON string. Empty string when nothing is +/// configured (neither the active variant nor the workspace-level +/// `[workspace] top` field is set) — callers branch on that +/// without needing try/catch. +/// +/// Variant resolution mirrors `active_variant_value`: the CLI's +/// `--variant ` selector wins, then the workspace default +/// variant (via `default = true` on `[[workspace.variants]]`). +/// The resolved variant name feeds +/// [`vw_lib::WorkspaceInfo::resolve_top`], which prefers the +/// per-variant `top` when set and falls back to the +/// workspace-level `top` otherwise. +/// +/// Errors are swallowed here (returned as empty string) for the +/// same reason as `active_variant_value` — a broken vw.toml has +/// already been diagnosed by the check machinery; the RPC path +/// shouldn't double-report. +fn top_value( + workspace_root: Option<&std::path::Path>, + active_variant: Option<&str>, +) -> Value { + let Some(ws) = workspace_root + .and_then(|p| camino::Utf8PathBuf::from_path_buf(p.to_path_buf()).ok()) + else { + return Value::String(String::new()); + }; + let Ok(cfg) = vw_lib::load_workspace_config(&ws) else { + return Value::String(String::new()); + }; + // Prefer the CLI-selected variant; fall back to the workspace + // default variant's name (matching how `active_variant` picks). + let variant_name: Option = match active_variant { + Some(n) => Some(n.to_string()), + None => cfg + .workspace + .default_variant() + .ok() + .flatten() + .map(|v| v.name.clone()), + }; + Value::String( + cfg.workspace + .resolve_top(variant_name.as_deref()) + .unwrap_or_default(), + ) +} + +/// `vhdl_ip_sources` — return every generated IP wrapper under +/// `/target/ip/**/*.vhd` as a JSON array of +/// absolute-path strings. Empty array when nothing has been +/// wrapped yet. +fn vhdl_ip_sources( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = vw_lib::vhdl_ip_sources(&ws) + .map_err(|e| format!("enumerating VHDL IP sources: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// `design_constraints` — return every constraint file under +/// `/constraints/**/*.{xdc,sdc}` as a JSON array of +/// absolute-path strings. Empty array when the workspace has no +/// `constraints/` dir. +fn design_constraints( + workspace_root: Option<&std::path::Path>, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = vw_lib::design_constraints(&ws) + .map_err(|e| format!("enumerating constraint files: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Which phase-scoped constraints dir to enumerate. Mirrors the +/// phase-specific accessors in `vw_lib`; kept as a small internal +/// enum so the dispatch match up top can name each variant without +/// duplicating the workspace-root plumbing three times. +enum ConstraintPhase { + Synth, + Place, + Route, +} + +fn design_phase_constraints( + workspace_root: Option<&std::path::Path>, + phase: ConstraintPhase, +) -> Result { + let ws = workspace_root_or_error(workspace_root)?; + let paths = match phase { + ConstraintPhase::Synth => vw_lib::design_synth_constraints(&ws), + ConstraintPhase::Place => vw_lib::design_place_constraints(&ws), + ConstraintPhase::Route => vw_lib::design_route_constraints(&ws), + } + .map_err(|e| format!("enumerating phase-scoped constraint files: {e}"))?; + Ok(paths_to_json_array(paths)) +} + +/// Pull the `variant` string out of an RPC args object. Missing / +/// null / non-string values return `None` — the handler then +/// falls back to the workspace's default variant. +fn extract_variant(args: &Value) -> Option { + args.as_object() + .and_then(|o| o.get("variant")) + .and_then(Value::as_str) + .map(|s| s.to_string()) +} + +/// Pull the `exclude_sim_only` boolean out of an RPC args object. +/// Missing / null / non-boolean values default to `false` so the +/// legacy call shape (`vw::vhdl_dependency_sources` with no args) +/// stays byte-for-byte compatible. +fn extract_exclude_sim_only(args: &Value) -> bool { + args.as_object() + .and_then(|o| o.get("exclude_sim_only")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn paths_to_json_array(paths: Vec) -> Value { + Value::Array( + paths + .into_iter() + .map(|p| Value::String(p.to_string_lossy().into_owned())) + .collect(), + ) +} + +/// Small helper: workspace-root paths served through the RPC are +/// std `Path`, but `vw_lib` takes `Utf8Path`. Do the conversion in +/// one place and normalize the error to the RPC message shape. +fn workspace_root_or_error( + workspace_root: Option<&std::path::Path>, +) -> Result { + let raw = workspace_root.ok_or_else(|| { + "no workspace root: entry file has no `vw.toml` in its parent chain" + .to_string() + })?; + camino::Utf8PathBuf::from_path_buf(raw.to_path_buf()).map_err(|p| { + format!("workspace root is not valid UTF-8: {}", p.display()) + }) +} + +/// `diff_files` — inputs `{actual: , expected: }`; +/// output is a JSON string carrying the unified diff, or an empty +/// string when the files are byte-equal. Paths are used verbatim +/// (the htcl side already resolves them relative to workspace +/// root before dispatching). +fn diff_files(args: Value) -> Result { + let obj = args.as_object().ok_or_else(|| { + "diff_files: args must be an object with `actual` and \ + `expected` string fields" + .to_string() + })?; + let actual = obj + .get("actual") + .and_then(Value::as_str) + .ok_or_else(|| "diff_files: missing string `actual`".to_string())?; + let expected = obj + .get("expected") + .and_then(Value::as_str) + .ok_or_else(|| "diff_files: missing string `expected`".to_string())?; + let a_bytes = std::fs::read(actual) + .map_err(|e| format!("reading actual `{actual}`: {e}"))?; + let b_bytes = std::fs::read(expected) + .map_err(|e| format!("reading expected `{expected}`: {e}"))?; + if a_bytes == b_bytes { + return Ok(Value::String(String::new())); + } + // Prefer text mode when both files are valid UTF-8 — the + // typical case for VHDL/htcl/SDC/XDC. When either side is + // binary, fall through to a short "binary files differ" + // marker so we don't spew hex. + let (Ok(a_text), Ok(b_text)) = + (std::str::from_utf8(&a_bytes), std::str::from_utf8(&b_bytes)) + else { + return Ok(Value::String(format!( + "binary files differ ({} bytes actual, {} bytes expected)", + a_bytes.len(), + b_bytes.len(), + ))); + }; + let diff = render_unified_diff(a_text, b_text, expected, actual); + Ok(Value::String(diff)) +} + +/// Render a colored unified diff between `expected` and `actual`. +/// The header lines put `expected` on `-` (removed) and `actual` +/// on `+` (added), which matches the "you wanted X but got Y" +/// narrative that assertion messages want. +fn render_unified_diff( + actual: &str, + expected: &str, + expected_path: &str, + actual_path: &str, +) -> String { + use colored::Colorize; + use similar::{ChangeTag, TextDiff}; + let diff = TextDiff::from_lines(expected, actual); + let mut out = String::new(); + out.push_str(&format!("--- {}\n", expected_path).red().to_string()); + out.push_str(&format!("+++ {}\n", actual_path).green().to_string()); + for group in diff.grouped_ops(3) { + // Each group is a run of consecutive ops sharing context — + // similar's own recommended unit for rendering per-hunk + // headers. + for op in &group { + for change in diff.iter_inline_changes(op) { + let sign = match change.tag() { + ChangeTag::Delete => "-", + ChangeTag::Insert => "+", + ChangeTag::Equal => " ", + }; + let mut line = String::new(); + line.push_str(sign); + for (emphasized, value) in change.iter_strings_lossy() { + if emphasized { + // Inline-changed subrun — bold within the + // colored line to draw the eye to the exact + // byte-run that differs. + match change.tag() { + ChangeTag::Delete => { + line.push_str( + &value.on_red().white().bold().to_string(), + ); + } + ChangeTag::Insert => { + line.push_str( + &value + .on_green() + .white() + .bold() + .to_string(), + ); + } + ChangeTag::Equal => line.push_str(&value), + } + } else { + line.push_str(&value); + } + } + // `iter_inline_changes` already emits the trailing + // newline for line-based diffs; only add one if + // it's missing so the last line of a hunk renders + // cleanly. + if !line.ends_with('\n') { + line.push('\n'); + } + let colored_line = match change.tag() { + ChangeTag::Delete => line.red().to_string(), + ChangeTag::Insert => line.green().to_string(), + ChangeTag::Equal => line.dimmed().to_string(), + }; + out.push_str(&colored_line); + } + } + } + out +} diff --git a/vw-vivado/src/lib.rs b/vw-vivado/src/lib.rs new file mode 100644 index 0000000..8543716 --- /dev/null +++ b/vw-vivado/src/lib.rs @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado [`EdaBackend`](vw_eda::EdaBackend) implementation. +//! +//! Spawns `vivado -mode tcl` as a long-lived worker, sources the +//! embedded shim TCL file at startup, and exchanges newline-delimited +//! JSON with it over stdio. Resolution order for the `vivado` +//! executable is: `VW_VIVADO` env var, then `PATH` lookup. v0 supports +//! the `eval` op only; structured ops land in phase 4. + +mod handlers; +mod raw_log; +mod rpc; +mod selection; +pub mod stream; +mod worker; + +pub use handlers::{ + make_handler, make_handler_full, make_handler_with_preloaded, + make_handler_with_variant, PreloadedPaths, SharedCriticalWarningCount, + SharedPreload, +}; +pub use raw_log::raw_log_path_for_workspace; +pub use rpc::{FnHandler, RpcHandler}; +pub use selection::{resolve_workspace_selection, Selection}; +pub use stream::{ + severity_of, stream_kind_for, Block, BlockAccumulator, LogLevel, Severity, +}; +pub use worker::{ + interrupt_process_group, AutoProject, VivadoBackend, VivadoConfig, +}; + +// Re-exported so the many call sites that reach for `vw_vivado::StreamKind` +// keep working now that streaming belongs to the backend abstraction rather +// than to this one backend. +pub use vw_eda::{StdoutSink, StreamKind}; diff --git a/vw-vivado/src/raw_log.rs b/vw-vivado/src/raw_log.rs new file mode 100644 index 0000000..5199b4d --- /dev/null +++ b/vw-vivado/src/raw_log.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Helpers for the raw Vivado byte-log — the ground-truth file every +//! `vw run` / `vw repl` session writes so users have an unfiltered +//! record of what Vivado emitted, independent of the block classifier +//! and log-level filters. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Compute the raw-log path for a workspace: creates +/// `/target/logs/` if it doesn't exist and returns the +/// timestamped file path inside it. The filename encodes the +/// wall-clock time at the call site as `vivado--.log` +/// so two runs in the same session sort in start order and never +/// collide. +/// +/// Returns `Err` only when the parent directory couldn't be created — +/// the caller can decide whether to abort the session or continue +/// without a raw log (typically the latter, since the log is a +/// diagnostic aid, not a build artifact). +pub fn raw_log_path_for_workspace( + workspace: &Path, +) -> std::io::Result { + let dir = workspace.join("target").join("logs"); + std::fs::create_dir_all(&dir)?; + let name = format!("vivado-{}.log", timestamp_slug()); + Ok(dir.join(name)) +} + +/// Format the current wall-clock time as `YYYYMMDD-HHMMSS`. Manually +/// computed from `UNIX_EPOCH` rather than pulling in `chrono` — the +/// output is one filename component, and vw-vivado has no other +/// reason to depend on a date/time crate. +/// +/// Uses UTC because local-time conversion requires reading `/etc/ +/// localtime`, which fails on some minimal containers, and the log +/// filename doesn't need human-friendly local-time semantics — it +/// only has to be monotonic within a session. +fn timestamp_slug() -> String { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let (year, month, day, hour, minute, second) = split_unix_epoch(now); + format!("{year:04}{month:02}{day:02}-{hour:02}{minute:02}{second:02}") +} + +/// Break a Unix timestamp (seconds since 1970-01-01 UTC) into +/// `(year, month, day, hour, minute, second)`. Deliberately +/// self-contained — the raw-log module has one caller and doesn't +/// warrant a chrono dependency for what amounts to a filename slug. +/// +/// The algorithm is Zeller's congruence-style month/day extraction +/// via the "civil from days" formula popularized by Howard Hinnant +/// (public domain, `date` C++ library). Correct across the Gregorian +/// range vw ever runs in. +fn split_unix_epoch(secs: u64) -> (u64, u32, u32, u32, u32, u32) { + let second = (secs % 60) as u32; + let mins = secs / 60; + let minute = (mins % 60) as u32; + let hours = mins / 60; + let hour = (hours % 24) as u32; + let mut days = (hours / 24) as i64; + // Shift epoch so day 0 is 0000-03-01 (Hinnant's convention: months + // run March→February to make leap-day handling uniform). + days += 719468; + let era = if days >= 0 { days } else { days - 146096 } / 146097; + let doe = (days - era * 146097) as u64; // day of era + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // year of era + let year_shifted = (yoe as i64) + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // day of year + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let year = (year_shifted + if month <= 2 { 1 } else { 0 }) as u64; + (year, month, day, hour, minute, second) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timestamp_slug_shape() { + let s = timestamp_slug(); + // Format is YYYYMMDD-HHMMSS = 15 chars, digit + hyphen split. + assert_eq!(s.len(), 15, "unexpected slug shape: {s}"); + assert_eq!(s.as_bytes()[8], b'-'); + assert!(s[..8].bytes().all(|b| b.is_ascii_digit())); + assert!(s[9..].bytes().all(|b| b.is_ascii_digit())); + } + + #[test] + fn split_unix_epoch_at_zero_is_1970_01_01() { + assert_eq!(split_unix_epoch(0), (1970, 1, 1, 0, 0, 0)); + } + + #[test] + fn split_unix_epoch_known_timestamps() { + // 2026-06-25 12:34:56 UTC = 1782390896 (verified via + // `date -u -d '2026-06-25 12:34:56' +%s`). + assert_eq!(split_unix_epoch(1782390896), (2026, 6, 25, 12, 34, 56)); + // Leap-day boundary: 2024-02-29 00:00:00 UTC = 1709164800 + assert_eq!(split_unix_epoch(1709164800), (2024, 2, 29, 0, 0, 0)); + // Y2038 boundary + 1 second: 2038-01-19 03:14:08 UTC + assert_eq!(split_unix_epoch(2147483648), (2038, 1, 19, 3, 14, 8)); + } + + #[test] + fn raw_log_path_creates_target_logs_dir() { + let tmp = tempfile::tempdir().unwrap(); + let path = raw_log_path_for_workspace(tmp.path()).unwrap(); + assert!(path.starts_with(tmp.path().join("target").join("logs"))); + // Directory was created as a side effect. + assert!(tmp.path().join("target").join("logs").is_dir()); + // File name matches vivado-.log. + let name = path.file_name().unwrap().to_string_lossy().to_string(); + assert!(name.starts_with("vivado-"), "unexpected name: {name}"); + assert!(name.ends_with(".log"), "unexpected name: {name}"); + } +} diff --git a/vw-vivado/src/rpc.rs b/vw-vivado/src/rpc.rs new file mode 100644 index 0000000..7e010ff --- /dev/null +++ b/vw-vivado/src/rpc.rs @@ -0,0 +1,74 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Shim-initiated RPC — how htcl library procs reach into `vw` +//! (Rust) for values Vivado can't provide on its own. +//! +//! Direction is the mirror of the eval loop: `vw` normally sends +//! requests and the shim answers. Here, the shim sends an +//! [`RpcCall`](vw_eda::protocol::RpcCall) and `vw` answers. The +//! answer is written back as a plain [`Response`](vw_eda::protocol::Response) +//! keyed on the RPC call's id, so no new response type is needed. +//! +//! ## Shape of a handler +//! +//! An [`RpcHandler`] is a `Send + Sync` object with a single async +//! method that looks up a `method` string and computes a JSON +//! payload. Callers construct a concrete handler with whatever +//! Rust-side state they need to serve (workspace root, design +//! source list, dep graph, …) and hand it to +//! [`VivadoConfig::rpc_handler`](crate::VivadoConfig::rpc_handler). +//! +//! Handler methods are named as flat strings (`"workspace_root"`). +//! Namespaces on the htcl side (`vw::workspace_root`) are a +//! call-site convention, not a wire concern. + +use async_trait::async_trait; +use serde_json::Value; +use std::sync::Arc; + +/// Trait implemented by anything that can service RPC calls from +/// the shim. Registered via +/// [`VivadoConfig::rpc_handler`](crate::VivadoConfig::rpc_handler). +/// +/// Unknown methods should return `Err("unknown method: …".into())` — +/// the shim surfaces that verbatim to the caller. +#[async_trait] +pub trait RpcHandler: Send + Sync { + async fn call(&self, method: &str, args: Value) -> Result; +} + +/// Convenience impl so callers can wrap a closure without hand- +/// rolling a struct. `Arc` because the trait bound is +/// `Send + Sync + 'static` and we hold it in an `Arc`. +#[async_trait] +impl RpcHandler for FnHandler +where + F: Fn(String, Value) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send, +{ + async fn call(&self, method: &str, args: Value) -> Result { + (self.f)(method.to_string(), args).await + } +} + +/// Type-erased wrapper for closure-based [`RpcHandler`] +/// construction. Use [`FnHandler::new`] rather than constructing +/// directly. +pub struct FnHandler { + f: F, +} + +impl FnHandler { + #[allow(clippy::new_ret_no_self)] // returns a trait-object Arc, not Self + pub fn new(f: F) -> Arc + where + F: Fn(String, Value) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + + Send + + 'static, + { + Arc::new(FnHandler { f }) + } +} diff --git a/vw-vivado/src/selection.rs b/vw-vivado/src/selection.rs new file mode 100644 index 0000000..1073aa6 --- /dev/null +++ b/vw-vivado/src/selection.rs @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Resolving `--part` / `--variant` against a workspace. +//! +//! Lives here rather than in the CLI because whoever spawns Vivado has to do +//! this, and that is no longer only the machine the developer is sitting at. +//! An agent running a build on an instance resolves the same flags against the +//! same `vw.toml` — the copy in its own synced tree — and gets the same answer +//! for the same reasons. +//! +//! Nothing here prints. The CLI shows what it learns; an agent puts it in a +//! log and streams it back. Returning the notes instead of writing them lets +//! both be true of the same code. + +use camino::Utf8Path; + +use crate::AutoProject; + +/// What a run's part and variant flags come to for one workspace. +pub struct Selection { + /// The project to open, if the workspace names a part at all. + pub auto_project: Option, + /// The variant in force, which `vw::vhdl_design_sources` filters on so a + /// `design.htcl` does not have to name it. + pub active_variant: Option, + /// Things worth telling the user that happened on the way — a legacy IP + /// cache cleared, a stale project wiped, a fallback taken. + pub notes: Vec, +} + +/// Resolve the part and variant flags against the workspace's declared parts +/// and variants. +/// +/// Applies the mutual-exclusion rules: `--variant` against a part-mode +/// workspace is an error, `--part` against a variant-mode one is an error +/// (variants own their parts), and neither means the workspace's default. +pub fn resolve_workspace_selection( + ws: &Utf8Path, + part: Option<&str>, + variant: Option<&str>, +) -> Result { + let Ok(cfg) = vw_lib::load_workspace_config(ws) else { + return Ok(Selection { + auto_project: None, + active_variant: None, + notes: Vec::new(), + }); + }; + let ws_info = &cfg.workspace; + + if variant.is_some() && ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} has no `[[workspace.variants]]` block; \ + remove `--variant` or add variants to vw.toml", + )); + } + if part.is_some() && !ws_info.variants.is_empty() { + return Err(format!( + "workspace at {ws} is variant-mode (has \ + `[[workspace.variants]]`); use `--variant ` instead of \ + `--part` — variants own their parts inline", + )); + } + + let mut notes = Vec::new(); + + if !ws_info.variants.is_empty() { + let selected = + ws_info.select_variant(variant).map_err(|e| e.to_string())?; + let Some(v) = selected else { + return Ok(Selection { + auto_project: None, + active_variant: None, + notes, + }); + }; + let persist_dir = persist_dir(ws, &ws_info.name, &mut notes); + Ok(Selection { + auto_project: Some(AutoProject { + name: ws_info.name.clone(), + part: v.part.clone(), + persist_dir, + }), + active_variant: Some(v.name.clone()), + notes, + }) + } else { + let selected = ws_info + .select_target_part(part) + .map_err(|e| e.to_string())?; + let persist_dir = persist_dir(ws, &ws_info.name, &mut notes); + Ok(Selection { + auto_project: selected.map(|p| AutoProject { + name: ws_info.name.clone(), + part: p.to_string(), + persist_dir: persist_dir.clone(), + }), + active_variant: None, + notes, + }) + } +} + +/// The on-disk Vivado project directory, after the one-shot legacy IP-cache +/// cleanup and staleness wipe. +/// +/// A failure here is not fatal: it becomes a note and an in-memory project, so +/// the session still works. That covers a read-only workspace, or a `target/` +/// held by something else. +fn persist_dir( + ws: &Utf8Path, + name: &str, + notes: &mut Vec, +) -> Option { + match vw_lib::prepare_vw_project_dir(ws, name) { + Ok(prep) => { + if prep.legacy_cache_removed > 0 { + notes.push(format!( + "removed {} legacy IP cache entr{y} under {ws}/target/ip \ + — replaced by on-disk Vivado project", + prep.legacy_cache_removed, + y = if prep.legacy_cache_removed == 1 { + "y" + } else { + "ies" + }, + )); + } + if let Some(wiped) = &prep.wiped_project { + notes.push(format!( + "wiped stale Vivado project at {wiped} (source \ + fingerprint changed or manifest missing)", + )); + } + Some(prep.project_dir.into_std_path_buf()) + } + Err(e) => { + notes.push(format!( + "failed to prepare on-disk Vivado project dir under \ + {ws}/target/vw-project ({e}); falling back to in-memory \ + project (state won't persist across sessions)", + )); + None + } + } +} diff --git a/vw-vivado/src/stream.rs b/vw-vivado/src/stream.rs new file mode 100644 index 0000000..7c78f19 --- /dev/null +++ b/vw-vivado/src/stream.rs @@ -0,0 +1,487 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Block-level segmentation over vw-vivado's classified stream. +//! +//! The [`crate::worker::PtyClassifier`] already tags every chunk it +//! emits with a [`crate::StreamKind`] — `Info`/`Warning`/`Error` for +//! Vivado's severity-prefixed messages (with attached `at : +//! in ::` continuation lines already merged into the same chunk), +//! and `Stdout` for everything else. This module groups consecutive +//! `Stdout` chunks into a single [`Block::None`] so downstream renderers +//! can collapse or dim them as one unit instead of drowning the console +//! in Vivado's tables, section headers, banners, and license chatter. +//! +//! ### The design contract +//! +//! - **Diagnostic block = exactly one classified chunk.** Multi-line +//! diagnostics (severity line + stack frames) are already merged +//! upstream, so we don't have to reconstruct the "obvious +//! continuation" boundary here. +//! - **NONE block = consecutive Stdout chunks.** A run of Stdout chunks +//! feeds into one open NONE block; a Diagnostic chunk flushes it. The +//! accumulator is stateful for exactly this reason. +//! - **Ordering is preserved.** [`BlockAccumulator::push`] returns +//! `Vec` (0-2 elements) rather than a single option, because a +//! Diagnostic arrival both flushes the pending NONE AND emits the +//! diagnostic — the caller sees them in the correct order without +//! having to track state itself. +//! +//! ### Log-level filtering +//! +//! [`LogLevel`] gates which blocks the renderer actually shows. Debug is +//! the escape hatch: it renders raw, no collapse. Info+ suppresses NONE +//! content but keeps it as a rendered "placeholder" (dim in `vw run`, +//! collapsible in the REPL) so users know noise was elided; users pick a +//! level and get exactly the diagnostics at that severity or higher. + +use vw_eda::StreamKind; + +/// Severity ladder used for log-level filtering. Higher = more severe. +/// Vivado does not emit a `DEBUG:`-tagged message of its own; the +/// [`Severity::Debug`] slot exists so [`LogLevel::Debug`] has a proper +/// floor for filtering — at that level every block renders raw. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Severity { + /// Non-diagnostic noise: Vivado tables, section headers, banners, + /// license chatter, `VHDL Output written to …` lines. Rendered + /// dimmed (or collapsed in the REPL) at Info+ so a user can still + /// see something happened; rendered raw at Debug. + None, + /// `INFO: [tag-id]` messages — low-importance advisories. + Info, + /// `WARNING: [tag-id]` messages. + Warning, + /// `CRITICAL WARNING: [tag-id]` messages. Vivado ranks these + /// between WARNING and ERROR; semantically they mean "your run + /// may fail because of this" so downstream rendering treats them + /// like errors. + CriticalWarning, + /// `ERROR: [tag-id]` messages. + Error, +} + +/// The user-facing log level knob. Debug is the escape hatch — at +/// that level even non-diagnostic Stdout renders in full. Info is the +/// default: NONE blocks are elided (dim/collapsed), Info+ diagnostics +/// pass through. Higher levels suppress lower-severity diagnostics. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum LogLevel { + /// Show everything, including raw non-diagnostic output. The + /// classifier's block boundaries still apply for rendering + /// purposes, but nothing is elided. + Debug, + /// Show INFO+ diagnostics normally. Elide/collapse NONE blocks. + #[default] + Info, + /// Show WARNING+ diagnostics. Elide/collapse NONE and INFO. + Warning, + /// Show CRITICAL WARNING+ diagnostics. Elide/collapse NONE, INFO, + /// WARNING. + Critical, + /// Show only ERROR diagnostics. Elide/collapse everything else. + Error, +} + +impl LogLevel { + /// Should a diagnostic at the given severity render at this level? + /// Non-diagnostic ([`Severity::None`]) blocks always return `true` + /// at Debug (raw stream) and `false` otherwise — the renderer is + /// still expected to show a collapsed placeholder for them at + /// Info+, but that's a rendering choice, not a filter decision. + pub fn allows(self, sev: Severity) -> bool { + match self { + LogLevel::Debug => true, + LogLevel::Info => sev >= Severity::Info, + LogLevel::Warning => sev >= Severity::Warning, + LogLevel::Critical => sev >= Severity::CriticalWarning, + LogLevel::Error => sev >= Severity::Error, + } + } + + /// Should a NONE block render as a collapsed placeholder (rather + /// than being elided entirely OR rendered raw)? At Debug it renders + /// raw; at all higher levels it renders as a placeholder. + pub fn collapse_none(self) -> bool { + !matches!(self, LogLevel::Debug) + } + + /// Parse the CLI's `--log-level` argument. Accepts lowercase (the + /// canonical form) plus a few common aliases users type without + /// checking `--help`. + pub fn parse(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "debug" => Ok(LogLevel::Debug), + "info" => Ok(LogLevel::Info), + "warn" | "warning" => Ok(LogLevel::Warning), + "critical" | "crit" | "critical-warning" => Ok(LogLevel::Critical), + "error" | "err" => Ok(LogLevel::Error), + other => Err(format!( + "unknown log level `{other}`; expected one of \ + debug|info|warning|critical|error" + )), + } + } +} + +impl std::fmt::Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + LogLevel::Debug => "debug", + LogLevel::Info => "info", + LogLevel::Warning => "warning", + LogLevel::Critical => "critical", + LogLevel::Error => "error", + }; + f.write_str(s) + } +} + +/// One classified block. Ownership: `lines` is the block's content +/// verbatim (trailing `\n` on each line stripped). The renderer +/// re-adds newlines when it writes out. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Block { + /// A run of consecutive non-diagnostic Stdout chunks. Render + /// dimmed (vw run) or collapsed (repl) at Info+; raw at Debug. + None { lines: Vec }, + /// A single classified diagnostic — one severity-tagged line + /// plus any attached `at : in ::` continuation + /// lines (already merged upstream by [`crate::worker::PtyClassifier`]). + Diagnostic { + severity: Severity, + lines: Vec, + }, +} + +impl Block { + pub fn severity(&self) -> Severity { + match self { + Block::None { .. } => Severity::None, + Block::Diagnostic { severity, .. } => *severity, + } + } + + pub fn lines(&self) -> &[String] { + match self { + Block::None { lines } | Block::Diagnostic { lines, .. } => lines, + } + } + + /// Number of source lines in the block, including any continuation + /// lines on a diagnostic. Renderers use this to size the collapsed + /// placeholder ("▶ preview (N lines)"). + pub fn line_count(&self) -> usize { + self.lines().len() + } +} + +/// Map a [`StreamKind`] to a [`Severity`]. `Stdout` → `None` — the +/// classifier's "we don't know what this is" bucket becomes our +/// non-diagnostic tier. +pub fn severity_of(kind: StreamKind) -> Severity { + match kind { + StreamKind::Stdout => Severity::None, + StreamKind::Info => Severity::Info, + StreamKind::Warning => Severity::Warning, + StreamKind::CriticalWarning => Severity::CriticalWarning, + StreamKind::Error => Severity::Error, + } +} + +/// Inverse of [`severity_of`]: pick the [`StreamKind`] a renderer +/// should use to color a block of the given severity. `Severity::None` +/// maps to `StreamKind::Stdout` — the default, unclassified stream. +pub fn stream_kind_for(severity: Severity) -> StreamKind { + match severity { + Severity::None => StreamKind::Stdout, + Severity::Info => StreamKind::Info, + Severity::Warning => StreamKind::Warning, + Severity::CriticalWarning => StreamKind::CriticalWarning, + Severity::Error => StreamKind::Error, + } +} + +/// Streaming block segmenter. Feed classified chunks in with +/// [`push`](Self::push); collect trailing content with +/// [`flush`](Self::flush) when the stream ends. +/// +/// State: the accumulator holds at most one open NONE block. A +/// classified (non-Stdout) chunk flushes it and immediately emits +/// itself as a Diagnostic block, in that order. +#[derive(Debug, Default)] +pub struct BlockAccumulator { + pending_none: Vec, +} + +impl BlockAccumulator { + pub fn new() -> Self { + Self::default() + } + + /// Consume one classified chunk. Returns 0-2 blocks in emission + /// order: a pending-NONE flush (if any) followed by the new + /// Diagnostic. Stdout chunks return `[]` — their content is + /// accumulated for the eventual NONE flush. + /// + /// The `chunk` is split on `\n`, trailing empty lines are dropped + /// so a diagnostic that ends `…in ::proc\n` doesn't leave a + /// spurious empty line in its block. An entirely-empty chunk is + /// a no-op regardless of kind. + pub fn push(&mut self, kind: StreamKind, chunk: &str) -> Vec { + let lines = split_chunk_lines(chunk); + if lines.is_empty() { + return Vec::new(); + } + let severity = severity_of(kind); + if severity == Severity::None { + self.pending_none.extend(lines); + return Vec::new(); + } + let mut out = Vec::with_capacity(2); + if !self.pending_none.is_empty() { + out.push(Block::None { + lines: std::mem::take(&mut self.pending_none), + }); + } + out.push(Block::Diagnostic { severity, lines }); + out + } + + /// Emit any pending NONE block. Call once at end-of-stream (or + /// when a rendering surface needs the trailing noise to settle). + /// Returns `[]` when nothing is pending. + pub fn flush(&mut self) -> Vec { + if self.pending_none.is_empty() { + return Vec::new(); + } + vec![Block::None { + lines: std::mem::take(&mut self.pending_none), + }] + } + + /// Test-only: peek at pending state without emitting. + #[cfg(test)] + fn pending_none(&self) -> &[String] { + &self.pending_none + } +} + +/// Split a chunk into lines the segmenter can accumulate. Drops the +/// single trailing empty element that `split('\n')` produces for a +/// `"a\nb\n"`-shaped chunk. Interior empty lines are preserved — some +/// diagnostic messages include a blank line between the tag and the +/// continuation frames, and renderers want that whitespace back. +fn split_chunk_lines(chunk: &str) -> Vec { + if chunk.is_empty() { + return Vec::new(); + } + let mut lines: Vec = + chunk.split('\n').map(str::to_string).collect(); + if lines.last().map(String::is_empty).unwrap_or(false) { + lines.pop(); + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stdout_chunks_accumulate_into_one_none_block() { + let mut acc = BlockAccumulator::new(); + assert_eq!(acc.push(StreamKind::Stdout, "banner line 1\n"), vec![]); + assert_eq!(acc.push(StreamKind::Stdout, "banner line 2\n"), vec![]); + assert_eq!( + acc.push(StreamKind::Stdout, "banner line 3\nline 4\n"), + vec![] + ); + // Nothing emitted yet — the None block is still open. + assert_eq!(acc.pending_none().len(), 4); + let flushed = acc.flush(); + assert_eq!( + flushed, + vec![Block::None { + lines: vec![ + "banner line 1".into(), + "banner line 2".into(), + "banner line 3".into(), + "line 4".into(), + ] + }] + ); + assert!(acc.pending_none().is_empty()); + } + + #[test] + fn diagnostic_chunk_flushes_pending_none_then_emits_itself() { + let mut acc = BlockAccumulator::new(); + acc.push(StreamKind::Stdout, "table row 1\n"); + acc.push(StreamKind::Stdout, "table row 2\n"); + let out = acc.push( + StreamKind::Warning, + "WARNING: [Synth 8-100] something\n at foo.tcl:12 in ::bar\n", + ); + assert_eq!( + out, + vec![ + Block::None { + lines: vec!["table row 1".into(), "table row 2".into()] + }, + Block::Diagnostic { + severity: Severity::Warning, + lines: vec![ + "WARNING: [Synth 8-100] something".into(), + " at foo.tcl:12 in ::bar".into(), + ] + } + ] + ); + // Pending is now clean. + assert!(acc.pending_none().is_empty()); + } + + #[test] + fn back_to_back_diagnostics_emit_no_none_between_them() { + let mut acc = BlockAccumulator::new(); + let out1 = acc.push(StreamKind::Info, "INFO: [X 1-1] first\n"); + assert_eq!( + out1, + vec![Block::Diagnostic { + severity: Severity::Info, + lines: vec!["INFO: [X 1-1] first".into()] + }] + ); + let out2 = acc.push(StreamKind::Info, "INFO: [X 1-2] second\n"); + assert_eq!( + out2, + vec![Block::Diagnostic { + severity: Severity::Info, + lines: vec!["INFO: [X 1-2] second".into()] + }] + ); + } + + #[test] + fn critical_warning_is_its_own_severity() { + let mut acc = BlockAccumulator::new(); + let out = acc.push( + StreamKind::CriticalWarning, + "CRITICAL WARNING: [Vivado 12-4739] set_clock_groups: …\n", + ); + assert_eq!( + out, + vec![Block::Diagnostic { + severity: Severity::CriticalWarning, + lines: vec![ + "CRITICAL WARNING: [Vivado 12-4739] set_clock_groups: …" + .into() + ] + }] + ); + } + + #[test] + fn empty_and_newline_only_chunks_are_ignored() { + let mut acc = BlockAccumulator::new(); + assert_eq!(acc.push(StreamKind::Stdout, ""), vec![]); + assert_eq!(acc.push(StreamKind::Stdout, "\n"), vec![]); + // The lone "\n" IS a blank line and should accumulate — Vivado + // sometimes emits blank separator lines that a renderer might + // want to preserve as vertical whitespace. Our + // `split_chunk_lines` correctly turns "\n" into one empty + // string line. + assert_eq!(acc.pending_none(), &["".to_string()]); + } + + #[test] + fn trailing_none_after_last_diagnostic_flushes_at_end() { + let mut acc = BlockAccumulator::new(); + acc.push(StreamKind::Error, "ERROR: [X 1-1] boom\n"); + acc.push(StreamKind::Stdout, "postmortem line 1\n"); + acc.push(StreamKind::Stdout, "postmortem line 2\n"); + let out = acc.flush(); + assert_eq!( + out, + vec![Block::None { + lines: vec![ + "postmortem line 1".into(), + "postmortem line 2".into() + ] + }] + ); + } + + #[test] + fn log_level_allows_diagnostic_severities() { + // Debug shows everything. + assert!(LogLevel::Debug.allows(Severity::None)); + assert!(LogLevel::Debug.allows(Severity::Error)); + + // Info hides only NONE. + assert!(!LogLevel::Info.allows(Severity::None)); + assert!(LogLevel::Info.allows(Severity::Info)); + assert!(LogLevel::Info.allows(Severity::Error)); + + // Warning hides NONE + INFO. + assert!(!LogLevel::Warning.allows(Severity::None)); + assert!(!LogLevel::Warning.allows(Severity::Info)); + assert!(LogLevel::Warning.allows(Severity::Warning)); + assert!(LogLevel::Warning.allows(Severity::CriticalWarning)); + + // Critical hides NONE + INFO + WARNING. + assert!(!LogLevel::Critical.allows(Severity::Warning)); + assert!(LogLevel::Critical.allows(Severity::CriticalWarning)); + assert!(LogLevel::Critical.allows(Severity::Error)); + + // Error hides everything but ERROR. + assert!(!LogLevel::Error.allows(Severity::CriticalWarning)); + assert!(LogLevel::Error.allows(Severity::Error)); + } + + #[test] + fn log_level_collapse_none_is_off_only_at_debug() { + assert!(!LogLevel::Debug.collapse_none()); + assert!(LogLevel::Info.collapse_none()); + assert!(LogLevel::Warning.collapse_none()); + assert!(LogLevel::Critical.collapse_none()); + assert!(LogLevel::Error.collapse_none()); + } + + #[test] + fn log_level_parse_accepts_common_forms() { + assert_eq!(LogLevel::parse("debug"), Ok(LogLevel::Debug)); + assert_eq!(LogLevel::parse("INFO"), Ok(LogLevel::Info)); + assert_eq!(LogLevel::parse("warn"), Ok(LogLevel::Warning)); + assert_eq!(LogLevel::parse("Warning"), Ok(LogLevel::Warning)); + assert_eq!(LogLevel::parse("critical"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("crit"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("critical-warning"), Ok(LogLevel::Critical)); + assert_eq!(LogLevel::parse("err"), Ok(LogLevel::Error)); + assert_eq!(LogLevel::parse("error"), Ok(LogLevel::Error)); + assert!(LogLevel::parse("").is_err()); + assert!(LogLevel::parse("verbose").is_err()); + } + + #[test] + fn diagnostic_carrying_multiline_message_stays_one_block() { + // The upstream PTY classifier merges the "at foo.tcl:X in ::proc" + // continuation lines into the same chunk it hands us. The + // segmenter should NOT split them across blocks. + let mut acc = BlockAccumulator::new(); + let chunk = "ERROR: [Synth 8-5826] no such design unit 'foo'\n \ + at foo.tcl:1 in ::bar\n at design.htcl:1\n"; + let out = acc.push(StreamKind::Error, chunk); + assert_eq!(out.len(), 1); + let Block::Diagnostic { severity, lines } = &out[0] else { + panic!("expected Diagnostic"); + }; + assert_eq!(*severity, Severity::Error); + assert_eq!(lines.len(), 3); + assert!(lines[0].contains("ERROR:")); + assert!(lines[1].contains("at foo.tcl")); + assert!(lines[2].contains("at design.htcl")); + } +} diff --git a/vw-vivado/src/worker.rs b/vw-vivado/src/worker.rs new file mode 100644 index 0000000..079072a --- /dev/null +++ b/vw-vivado/src/worker.rs @@ -0,0 +1,2362 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Vivado worker process: spawn under a PTY, accept the shim's +//! loopback connection, drive the request/response loop. +//! +//! Vivado is spawned with its stdin/stdout/stderr attached to a +//! pseudo-terminal slave (via [`portable_pty`]). The PTY matters: +//! when stdout is a pipe, glibc puts it in full-block-buffering mode +//! and Vivado's banner / source-echo / info messages don't appear +//! until ~4 KB accumulates, which kills the `--verbose` UX. With a +//! PTY Vivado sees a TTY on stdout and switches to line buffering, +//! so output streams as it's produced. `portable_pty` works on Linux +//! and macOS via Unix PTYs, and on Windows via ConPTY. + +use std::fs::File; +use std::io::{BufWriter, Read, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use portable_pty::{ + native_pty_system, Child as PtyChild, CommandBuilder, MasterPty, PtySize, +}; +use tempfile::TempDir; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::tcp::OwnedWriteHalf; +use tokio::net::TcpListener; +use tracing::{debug, warn}; +use vw_eda::{ + BackendError, EdaBackend, EvalOutput, Request, RequestOp, Response, + ResponseResult, StdoutSink, StreamKind, WireMessage, +}; + +/// Embedded shim TCL. Written to a temp file at worker startup and +/// passed to `vivado -source`. +const SHIM_TCL: &str = include_str!("../shim/vivado-shim.tcl"); + +/// How long to wait for the shim to connect back to our loopback +/// listener. Vivado's startup takes most of this on a cold cache; on +/// a warm cache it's a few seconds. +const SHIM_CONNECT_TIMEOUT: Duration = Duration::from_secs(180); + +// --- TEMPORARY DIAGNOSTIC --------------------------------------------- +// +// Timing log to disentangle three arrival questions: +// +// 1. When did Vivado *write* the bytes to the PTY? (pump_read / +// pump_send events — captured on the pump thread.) +// 2. When did our select loop *pull* them from the pty_rx +// channel? (pty_rx_recv events.) +// 3. When did the protocol Response arrive relative to those? +// (response_arrival events.) +// +// Answering "is a late error message A) already-written-but-not- +// pumped, or B) not-yet-written-by-Vivado?" needs the delta between +// (1) and (3). Writing to `/tmp/vw-timing.log` so it survives the +// TUI's alternate-screen mode. Remove this block (and its callers) +// once we've settled the timing question. +fn vw_timing_log(event: &str, len: usize, preview: &str) { + use std::io::Write; + use std::sync::OnceLock; + static START: OnceLock = OnceLock::new(); + let start = START.get_or_init(std::time::Instant::now); + let elapsed_us = start.elapsed().as_micros(); + // Only write if the env var opts in — no forced disk I/O in + // normal runs. + static ENABLED: OnceLock = OnceLock::new(); + let enabled = + *ENABLED.get_or_init(|| std::env::var("VW_TIMING_LOG").is_ok()); + if !enabled { + return; + } + static FILE: OnceLock>> = + OnceLock::new(); + let file_slot = FILE.get_or_init(|| { + std::sync::Mutex::new( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/tmp/vw-timing.log") + .ok(), + ) + }); + if let Ok(mut guard) = file_slot.lock() { + if let Some(f) = guard.as_mut() { + let clean: String = + preview.chars().filter(|c| *c != '\n').take(140).collect(); + let _ = writeln!(f, "{elapsed_us:>10} {event} len={len} {clean}"); + let _ = f.flush(); + } + } +} + +/// Spawn-time configuration for [`VivadoBackend`]. +#[derive(Clone, Default)] +pub struct VivadoConfig { + /// Override the `vivado` executable path. If `None`, resolution + /// order is `$VW_VIVADO`, then a `vivado` lookup on `$PATH`. + pub vivado: Option, + /// Working directory for the spawned process. If `None`, a + /// scratch tempdir is created so Vivado's incidental files don't + /// litter the user's cwd. + pub working_dir: Option, + /// When `true`, forward Vivado's PTY output (banner, source-echo, + /// info messages) as it's produced. When `false` (default), the + /// bytes are read and discarded so they don't pollute either of + /// vw's output streams. User TCL `puts` is always captured per- + /// eval via the shim and streamed in the protocol, independent + /// of this setting. + /// + /// Where the verbose output goes depends on [`verbose_log`](Self::verbose_log): + /// when set, lines stream into that file; when unset, they go + /// to vw's stderr. + pub verbose: bool, + /// When `true`, every Vivado-formatted message + /// (`INFO:`/`WARNING:`/`ERROR:`/`CRITICAL WARNING:`) gets the + /// Tcl call stack appended as `at : in ::proc` + /// continuation lines. When `false` (default), stacks are + /// attached only to WARNINGs and ERRORs — INFO messages render + /// as a single line. INFOs are noisy under heavy Vivado + /// activity (CIPS customization can emit dozens per call) and + /// the stack adds little signal compared to the message text; + /// power users diagnosing why a particular INFO fires can flip + /// this on with `--info-with-stack`. + pub info_with_stack: bool, + /// Optional path to a log file for verbose output. When set, + /// supersedes the default stderr destination — necessary for + /// the REPL, which owns the terminal in alternate-screen mode + /// and would corrupt the TUI rendering if anyone wrote raw + /// bytes to stderr mid-frame. The file is created (or + /// truncated) at spawn time and flushed per-line so it's safe + /// to `tail -f` from another terminal. + pub verbose_log: Option, + /// Optional path to a byte-perfect raw log of Vivado's PTY + /// output. When set, every byte the PTY pump reads is teed to + /// this file BEFORE any line-splitting, classification, or + /// filtering — the file is the ground truth of what Vivado + /// emitted this session. Callers are responsible for making + /// the parent directory exist; the convention is + /// `/target/logs/vivado-.log`, produced + /// by [`raw_log_path_for_workspace`]. Distinct from + /// [`verbose_log`], which only captures unclassified firehose + /// output routed through the classifier. + pub raw_log: Option, + /// Optional RPC handler for shim-initiated calls (`vw::…` + /// procs that reach back into Rust for values Vivado can't + /// provide). Dispatched from the proto-read task per inbound + /// [`vw_eda::protocol::RpcCall`]. When `None`, any RPC call + /// from the shim is answered with "no RPC handler + /// configured". + pub rpc_handler: Option>, + /// When set, spawn creates an in-memory Vivado project with + /// the given (project_name, target_part) as soon as Vivado is + /// ready to accept commands. Eliminates the "no open project" + /// bootstrap problem for `ip::check` / `get_ipdefs` — those + /// commands need a project (the IP catalog is per-part). + /// Composes with `@test(dedicated-eda part=…)` where the test + /// runner overrides the part for isolated tests. + pub auto_project: Option, +} + +/// Auto-created in-memory Vivado project. Materialized inside +/// [`VivadoBackend::spawn`] via +/// `create_project -in_memory -name -part ` as soon +/// as the shim connects. Removes the need for every htcl entry to +/// bootstrap a project by hand and unblocks IP-catalog queries at +/// module load time. +#[derive(Clone, Debug)] +pub struct AutoProject { + /// Project name shown to Vivado. Typically the workspace name. + pub name: String, + /// Full Vivado part specifier + /// (e.g. `xcvm3358-vsvh1747-2M-e-S`). + pub part: String, + /// `None` → `create_project -in_memory` (the legacy path, + /// kept for `vw test` isolation — persisted state would + /// cross-contaminate parallel `@test(dedicated-eda)` runs + /// and dirty the workspace with test scratch). + /// + /// `Some(dir)` → on-disk Vivado project at + /// `//.xpr`. If that `.xpr` already exists + /// the spawn bootstrap runs `open_project` (warm path); if + /// it doesn't exist we `create_project -name -dir + /// ` + immediately `set_property source_mgmt_mode + /// None [current_project]` so Vivado's auto-scan doesn't + /// swallow out-of-fileset files like `vw::make_wrapper`'s + /// `target/ip//wrapper.vhd` into `xil_defaultlib`. + /// Callers are responsible for staleness invalidation (see + /// `vw_lib::project_needs_wipe`) — the worker takes whatever + /// state is on disk as-is. + pub persist_dir: Option, +} + +/// Vivado [`EdaBackend`] implementation. +pub struct VivadoBackend { + child: Option>, + /// Master end of the PTY. Kept alive so the slave (Vivado) doesn't + /// receive EOF on its stdin. + _master: Box, + /// Protocol-socket line reader. Backed by a background task + /// (spawned in [`Self::new`]) that owns the `BufReader` and + /// forwards each newline-terminated frame here. + /// + /// Why the task exists: `BufReader::read_line` is + /// **cancellation-unsafe** per tokio's docs — if it's the + /// event in a `tokio::select!` and another branch fires + /// first, the future is dropped, any bytes already accumulated + /// into the internal buffer are lost, and the BufReader is left + /// in an unspecified state where subsequent reads return + /// incorrect data. Our eval loop races protocol reads against + /// `pty_rx.recv()` in a biased `select!`, so a big-enough + /// response (`props::get` returns ~25 KB Properties reprs) + /// crossing a PTY-line arrival used to corrupt the buffer and + /// produce "malformed message from shim" parse errors + /// truncated mid-JSON. Reading through a channel makes the + /// select's read side cancellation-safe. + proto_read: + tokio::sync::mpsc::UnboundedReceiver>, + _proto_read_task: Option>, + proto_write: OwnedWriteHalf, + /// Shim-initiated RPC handler (see [`crate::RpcHandler`]). + /// Consulted by [`Self::dispatch_rpc_call`] for every inbound + /// [`WireMessage::Rpc`]. `None` (the default) → RPCs return + /// "no RPC handler configured" error. + rpc_handler: Option>, + next_id: AtomicU64, + stdout_pump: Option>, + stdout_sink: Option, + /// Lines the PTY pump has read from Vivado's process stdout, in + /// arrival order. Drained during eval so Vivado's own message + /// system (ERROR/WARNING/CRITICAL WARNING/INFO) reaches the + /// stdout sink alongside user `puts` output — otherwise the + /// "earlier errors" Vivado refers to when failing a command are + /// invisible to the caller. + pty_rx: tokio::sync::mpsc::UnboundedReceiver, + /// Mirrors [`VivadoConfig::verbose`]. When true, PTY lines that + /// don't classify (banner, source-echo, idle chatter) are + /// surfaced — to [`verbose_log`](Self::verbose_log) if set, + /// otherwise to vw's stderr. Classified lines always route + /// through the message filter regardless of verbose. + verbose: bool, + /// Optional log file the verbose firehose streams into. The + /// REPL uses this so verbose output doesn't blow through its + /// TUI alternate screen by hitting stderr. + verbose_log: Option>, + /// Off by default. When true (set via the + /// `VW_TRACE_MESSAGE_SOURCES` env var at spawn time), emit a + /// gray `[vw-pty]` Info line before every classified PTY + /// chunk so the caller can see which path produced it. + /// Useful for diagnosing "where is this warning coming from?" + /// questions; noisy enough that it shouldn't be on by + /// default. + trace_message_sources: bool, + /// Brief-buffer classifier for multi-line PTY warnings. See + /// [`PtyClassifier`] for the merging semantics. + pty_classifier: PtyClassifier, + /// Stack of ready-to-use frame sets sent by the shim via + /// `__VW_CTX_*` PTY markers. Each entry is a set of frames + /// captured at a nesting level: outer entries are older, the + /// top is the innermost currently-executing wrap (a user proc's + /// body, or a wrapped `set_property` / `generate_netlist_ip` + /// call). When a Warning/Error chunk lands without its own + /// trace, the top entry's frames get appended as `\n at + /// ` lines — that's what lets the REPL show "this + /// IP_Flow warning came from configure_cips → + /// create_versal_cips → set_property" even though Vivado's C++ + /// never went through our Tcl stack capture. Nesting matters + /// because user procs call each other and each level's marker + /// wraps the next; a single active slot would clobber outer + /// context when the inner call returned mid-warning-emission. + pty_context_stack: Vec>, + /// Frames currently being assembled between + /// `__VW_CTX_BEGIN__` and `__VW_CTX_READY__`. Pushed onto + /// `pty_context_stack` atomically on READY so a partial + /// marker stream can't leak half-formed traces into emitted + /// warnings. Scalar (not per-nesting-level) because BEGIN and + /// READY are emitted synchronously in a single Tcl step — the + /// shim never emits a nested BEGIN before its outer READY. + building_pty_context: Vec, + _shim_dir: TempDir, + /// Latched when we detect the shim has been torn down (via + /// the `Vivado%` prompt landing in the PTY during an eval — + /// see the shim-died detector in `read_response_for`). + /// Once set, every subsequent `eval` returns a fast-fail + /// error instead of writing the request into an orphaned + /// socket that nobody will ever read. Cleared when the + /// shim is successfully respawned via + /// [`Self::respawn_shim`]. + shim_dead: bool, + /// Absolute path of the shim `.tcl` file inside `_shim_dir`. + /// Needed by [`Self::respawn_shim`] to write + /// `source \n` back to Vivado's interactive PTY prompt + /// after Ctrl-C's `interp cancel -unwind` blew the shim's + /// dispatch loop out from under us. Vivado keeps the same + /// `VW_PROTOCOL_ADDR` env var across shim `source`s, so the + /// respawn re-connects to the listener below. + shim_path: PathBuf, + /// Held-open listener so [`Self::respawn_shim`] can accept a + /// second connection on the same port. Original spawn takes + /// the first accept; if that shim dies to a cancel, the + /// re-sourced shim connects again here. + listener: Option, + _scratch_dir: Option, +} + +impl VivadoBackend { + /// Spawn a Vivado worker under a PTY, wait for the shim to + /// connect back on our loopback listener, and return once we're + /// ready to accept [`EdaBackend::eval`] calls. + pub async fn spawn(config: VivadoConfig) -> Result { + let vivado_path = resolve_vivado(&config)?; + + let shim_dir = tempfile::Builder::new() + .prefix("vw-vivado-shim-") + .tempdir() + .map_err(BackendError::Io)?; + let shim_path = shim_dir.path().join("vivado-shim.tcl"); + tokio::fs::write(&shim_path, SHIM_TCL) + .await + .map_err(BackendError::Io)?; + + // cwd resolution order: + // 1. Explicit `config.working_dir` — caller override, + // used verbatim. + // 2. Persisted on-disk project's own dir + // (`/`) — when `auto_project` sets + // `persist_dir`. This anchors Vivado's runtime path + // resolution to the project's own location, which + // eliminates `[Vivado 12-13650] IP file has been moved + // from its original location` warnings the OOC + // child-synth of `synth_ip` was firing when cwd sat + // in a sibling tempdir. Any incidental files Vivado + // writes to cwd (vivado.jou, vivado.log, .Xil/, + // webtalk.log) land inside the project dir under + // `/target/vw-project//` — which is + // `.gitignore`d by design, so the cluttering is + // isolated to a machine-local dir. + // 3. Fresh tempdir — the `vw test` and legacy + // `-in_memory` paths (`persist_dir = None`) keep + // full isolation because they intentionally have no + // on-disk project home. + let (cwd, scratch_dir) = + match (&config.working_dir, &config.auto_project) { + (Some(dir), _) => (dir.clone(), None), + (None, Some(ap)) if ap.persist_dir.is_some() => { + let persist = ap.persist_dir.as_ref().unwrap(); + let project_cwd = persist.join(&ap.name); + // `prepare_vw_project_dir` (caller-side) already + // ensures `` exists and is wiped-on-stale, + // but the per-name subdir may not exist yet on a + // cold create path — Vivado spawns before + // `create_project -dir /` runs. + std::fs::create_dir_all(&project_cwd) + .map_err(BackendError::Io)?; + (project_cwd, None) + } + (None, _) => { + let tmp = tempfile::Builder::new() + .prefix("vw-vivado-cwd-") + .tempdir() + .map_err(BackendError::Io)?; + (tmp.path().to_path_buf(), Some(tmp)) + } + }; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(BackendError::Io)?; + let local_addr = listener.local_addr().map_err(BackendError::Io)?; + debug!(?vivado_path, ?shim_path, ?cwd, %local_addr, "spawning vivado worker"); + + let pty_system = native_pty_system(); + let pair = pty_system + .openpty(PtySize { + rows: 24, + cols: 80, + pixel_width: 0, + pixel_height: 0, + }) + .map_err(|e| { + BackendError::Worker(format!("openpty failed: {e}")) + })?; + + // `-mode tcl` keeps Vivado alive as a long-running TCL + // interpreter; once the shim's socket loop takes over, Vivado + // never returns to its interactive prompt. + let mut cmd = CommandBuilder::new(&vivado_path); + cmd.arg("-mode"); + cmd.arg("tcl"); + cmd.arg("-nojournal"); + cmd.arg("-nolog"); + cmd.arg("-source"); + cmd.arg(&shim_path); + cmd.env("VW_PROTOCOL_ADDR", local_addr.to_string()); + // Read by the shim's `is_vivado_message` / stack-attach + // logic to decide whether INFO-level messages get stacks + // attached. WARNING / ERROR / CRITICAL always do. + cmd.env( + "VW_INFO_WITH_STACK", + if config.info_with_stack { "1" } else { "0" }, + ); + // Diagnostic knobs — propagate whatever the shell set so + // `VW_TRACE_STACK_CAPTURE=1 vw run …` (or the REPL + // equivalent) reaches the shim. portable-pty inherits by + // default on unix, but pass-through explicitly keeps the + // contract obvious. + if let Ok(v) = std::env::var("VW_TRACE_STACK_CAPTURE") { + cmd.env("VW_TRACE_STACK_CAPTURE", v); + } + cmd.cwd(&cwd); + + let child = pair.slave.spawn_command(cmd).map_err(|e| { + BackendError::Worker(format!( + "failed to spawn vivado at {}: {}", + vivado_path.display(), + e + )) + })?; + // Release our handle to the slave so the master sees EOF + // when the child exits. + drop(pair.slave); + + let reader = pair.master.try_clone_reader().map_err(|e| { + BackendError::Worker(format!("pty reader clone failed: {e}")) + })?; + // Open the raw byte-log if the caller asked for one. + // Parent-directory creation is the caller's job — see + // `raw_log_path_for_workspace`; here we just open (creating + // the file, truncating on repeat) and wrap for buffered + // per-chunk flush. Failure to open bubbles as a spawn + // error rather than silently disabling the log — the + // caller opted in, so a broken opt-in should be loud. + let raw_log: Option> = config + .raw_log + .as_ref() + .map(|p| File::create(p).map(BufWriter::new)) + .transpose() + .map_err(BackendError::Io)?; + let (pty_tx, pty_rx) = tokio::sync::mpsc::unbounded_channel::(); + let stdout_pump = spawn_stdout_pump(reader, pty_tx, raw_log); + + // Wait for the shim to connect. Borrow `&mut listener` so + // it stays alive for [`Self::respawn_shim`] to re-accept + // on the same port after a cancel-induced shim teardown. + let accept_result = + tokio::time::timeout(SHIM_CONNECT_TIMEOUT, listener.accept()).await; + let stream = match accept_result { + Ok(Ok((stream, _peer))) => stream, + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(_) => { + return Err(BackendError::Worker( + "timed out waiting for shim to connect".into(), + )); + } + }; + stream.set_nodelay(true).map_err(BackendError::Io)?; + debug!("shim connected"); + + let (read_half, write_half) = stream.into_split(); + // Move BufReader::read_line onto a dedicated task so its + // cancellation-unsafe nature can't corrupt state when the + // eval loop's `select!` fires a different branch mid-read. + // See the field-doc on `proto_read` for the failure mode. + let (proto_tx, proto_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut proto_buf = BufReader::new(read_half); + let _proto_read_task = tokio::spawn(async move { + let mut line = String::new(); + loop { + line.clear(); + match proto_buf.read_line(&mut line).await { + Ok(0) => break, + Ok(_) => { + if proto_tx.send(Ok(line.clone())).is_err() { + break; + } + } + Err(e) => { + let _ = proto_tx.send(Err(e)); + break; + } + } + } + }); + let trace_message_sources = std::env::var("VW_TRACE_MESSAGE_SOURCES") + .map(|v| { + let v = v.trim(); + !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false") + }) + .unwrap_or(false); + + let verbose_log = config + .verbose_log + .as_ref() + .map(|p| File::create(p).map(BufWriter::new)) + .transpose() + .map_err(BackendError::Io)?; + + let mut backend = Self { + child: Some(child), + _master: pair.master, + proto_read: proto_rx, + _proto_read_task: Some(_proto_read_task), + proto_write: write_half, + rpc_handler: config.rpc_handler.clone(), + next_id: AtomicU64::new(1), + stdout_pump: Some(stdout_pump), + stdout_sink: None, + pty_rx, + verbose: config.verbose, + verbose_log, + trace_message_sources, + pty_classifier: PtyClassifier::new(PTY_CONTINUATION_WINDOW), + pty_context_stack: Vec::new(), + building_pty_context: Vec::new(), + _shim_dir: shim_dir, + _scratch_dir: scratch_dir, + shim_dead: false, + shim_path: shim_path.clone(), + listener: Some(listener), + }; + // Auto-create (or reopen) the Vivado project if the caller + // specified one. Runs BEFORE we return to the caller so the + // first user eval sees an existing project — `ip::check` / + // `get_ipdefs` / etc. work without special-casing. + // + // Three branches keyed on `persist_dir`: + // * `None` → legacy `create_project -in_memory`. Kept for + // `vw test` isolation. + // * `Some(dir)` with an existing `//.xpr` + // → `open_project` (warm path — Vivado restores the + // full BD/XCI/`ipshared` graph without us hand- + // serializing it). If Vivado rejects the file (version + // mismatch, truncated write, etc.) we log a warning, + // wipe the dir, and fall through to the fresh-create + // branch in the same session so users don't have to + // intervene manually. + // * `Some(dir)` otherwise → `create_project -name + // -dir `. Immediately follow with + // `set_property source_mgmt_mode None + // [current_project]`: default is `All`, which would + // auto-import every `.vhd` Vivado finds under the + // project dir into `xil_defaultlib` — including + // `vw::make_wrapper`'s out-of-fileset + // `target/ip//wrapper.vhd` files. That fights the + // "deliberately do NOT `-import`" model make_wrapper + // relies on. + // + // All branches then force `TARGET_LANGUAGE VHDL` — `vw` is + // VHDL-first, and the default Verilog would silently emit + // `_wrapper.v` files that downstream tools looking for + // `.vhd` fail on with an unhelpful "no wrapper found". + if let Some(ap) = &config.auto_project { + use vw_eda::EdaBackend as _; + let tcl = match &ap.persist_dir { + None => format!( + "create_project -in_memory -name {{{name}}} \ + -part {{{part}}}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + name = ap.name, + part = ap.part, + ), + Some(dir) => { + // Vivado's `create_project -dir D -name N` + // places `N.xpr` + `N.srcs` + `N.cache` + + // `N.hw` + `N.ip_user_files` + `N.gen` + // DIRECTLY under D — flat, not nested in + // `D/N/`. Nesting is what we want (clean + // `remove_dir_all` on invalidation, doesn't + // fight sibling artifacts), so we pass `D/N` + // as the `-dir` argument and let Vivado + // scatter its per-project files inside that. + let per_name = dir.join(&ap.name); + let xpr = per_name.join(format!("{}.xpr", ap.name)); + if xpr.exists() { + // Warm path. On failure we `catch` and fall + // through to a fresh create in the same eval + // so the session stays usable even if the + // on-disk project got corrupted. + // + // Intentionally NO `set_property + // source_mgmt_mode None`: the earlier + // migration set it defensively to keep + // `vw::make_wrapper`'s workspace-side + // wrapper (`target/ip//wrapper.vhd`) + // from being auto-imported into + // `xil_defaultlib`, but that's already + // prevented by `-import false` in + // `vw::make_wrapper` itself. With `None`, + // Vivado also stops adding IP-generated + // RTL under `.gen/sources_1/ip//synth/` + // to the synth fileset, so `synth_design` + // fails to find `entity xil_defaultlib.` + // — regressing top-level IPs like + // `primary_clock`. + format!( + "if {{[catch {{open_project {{{xpr}}}}} err]}} {{\n \ + puts \"WARNING: open_project failed ({xpr}): \ + $err — recreating\"\n \ + file delete -force {{{per_name}}}\n \ + file mkdir {{{per_name}}}\n \ + create_project -name {{{name}}} -part {{{part}}} \ + -dir {{{per_name}}}\n \ + }}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + xpr = xpr.display(), + per_name = per_name.display(), + name = ap.name, + part = ap.part, + ) + } else { + format!( + "file mkdir {{{per_name}}}\n\ + create_project -name {{{name}}} -part {{{part}}} \ + -dir {{{per_name}}}\n\ + set_property TARGET_LANGUAGE VHDL [current_project]", + per_name = per_name.display(), + name = ap.name, + part = ap.part, + ) + } + } + }; + backend.eval(&tcl).await.map_err(|e| { + BackendError::Worker(format!( + "auto-creating Vivado project (name={}, part={}, \ + persist_dir={:?}): {e}", + ap.name, ap.part, ap.persist_dir, + )) + })?; + } + Ok(backend) + } + + /// The Vivado child process's OS pid, if the child is still + /// alive. Callers use this to send SIGINT for eval + /// cancellation — see [`Self::interrupt`] for the mechanism + /// and design assumptions. + pub fn child_pid(&self) -> Option { + self.child.as_ref().and_then(|c| c.process_id()) + } + + /// Cancel the current in-flight Tcl eval WITHOUT killing the + /// Vivado session. Sends `SIGINT` to the child; Vivado's Tcl + /// runtime installs a `SIGINT` handler that traps the signal + /// into `interp cancel`, so the mechanism is: + /// + /// 1. Vivado receives `SIGINT` in the middle of executing the + /// current eval body (a `synth_ip`, a long `get_pins + /// -hierarchical`, whatever). + /// 2. Tcl catches the signal, sets the "cancel" flag on the + /// active interpreter, and the currently-executing script + /// returns with `TCL_ERROR` / message `"interrupted"`. + /// 3. Our shim wraps every user eval in `catch`, so the error + /// is caught, wrapped into a protocol response, and sent + /// back over the wire. The eval returns from the caller's + /// perspective as a normal [`vw_eda::BackendError::Tcl`] + /// with the interrupted message. + /// 4. Vivado's outer read loop is untouched — it goes right + /// back to reading the next request from the shim's + /// protocol socket. Project state, IP synth results, all + /// open designs survive. + /// + /// Same semantics as pressing Ctrl-C on an interactive + /// `vivado -mode tcl` console: the running command aborts, + /// the Tcl prompt returns. + /// + /// **Signals the process group, not the process.** Vivado on + /// disk is a bash wrapper (`~/Xilinx/…/bin/vivado`) that runs + /// `loader -exec vivado …` without `exec`, so bash sticks + /// around as the child we spawned, and the actual Vivado + /// runtime is a grandchild via loader. If we `kill(pid, + /// SIGINT)` we hit bash-running-a-script, which normally does + /// NOT forward SIGINT to its children — the signal dies with + /// bash's SIGINT-default handling and Vivado never sees it. + /// + /// portable-pty sets each spawned command up as its own + /// session leader with a fresh PGID = its own pid, and + /// descendants inherit that PGID unless they explicitly call + /// `setpgrp`. So `kill(-pid, SIGINT)` — i.e. signalling the + /// negative-of-pid, which POSIX interprets as "the process + /// group with this ID" — reaches bash, loader, and Vivado + /// all at once. Bash is a no-op recipient (script mode + /// SIGINT); loader relays; Vivado's Tcl runtime catches it + /// and does the `interp cancel`. + /// + /// **Design assumption**: SIGINT-under-PTY behaves the same + /// as SIGINT-on-console. Historically true for Vivado 2020+ + /// (we're on 2025.1). If a future release drops that + /// handling, callers see `interrupt()` return without + /// actually cancelling — the current eval keeps running. We + /// have no clean fallback in that world; the eval would have + /// to be killed via full backend restart. + /// + /// Returns the pid (== pgid) we signalled so callers can log + /// something meaningful ("interrupted eval on pid 12345"). + /// Returns `None` if the child is already gone — in which + /// case there's nothing to interrupt anyway. + /// + /// Unix-only. Windows has no analogue that reliably pumps + /// into the child's Tcl signal handler, and every vw + /// interactive user is on Unix; when Windows matters we'll + /// add a ConPTY-native cancellation path. + #[cfg(unix)] + pub fn interrupt(&self) -> Option { + let pid = self.child_pid()?; + interrupt_process_group(pid); + Some(pid) + } + + /// Windows stub: no-op. See the unix variant's doc for the + /// eval-cancellation model. + #[cfg(not(unix))] + pub fn interrupt(&self) -> Option { + None + } + + fn alloc_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) + } + + async fn write_request( + &mut self, + req: &Request, + ) -> Result<(), BackendError> { + let mut line = serde_json::to_string(req)?; + line.push('\n'); + self.proto_write.write_all(line.as_bytes()).await?; + self.proto_write.flush().await?; + Ok(()) + } + + /// Read messages until we get the response that matches + /// `expected_id`. Stream notifications for the same id are routed + /// to [`Self::stdout_sink`] if set, or accumulated into the + /// returned `String` if not. + /// + /// While waiting, the worker also drains the PTY channel — so + /// Vivado's own `send_msg_id` output (ERROR/WARNING/CRITICAL + /// WARNING/INFO lines printed to the process stdout, not through + /// the shim's `puts` interception) reaches the same sink and + /// gets attributed to the in-flight eval. Without this, the + /// "earlier errors" Vivado refers to when a command fails are + /// invisible to the caller. + async fn read_response_for( + &mut self, + expected_id: u64, + ) -> Result<(Response, String), BackendError> { + let mut accumulated = String::new(); + let mut line = String::new(); + loop { + line.clear(); + // Race the protocol socket against the PTY channel. The + // protocol path eventually terminates the loop (a + // Response arrives); PTY lines are forwarded as + // best-effort context until then. + tokio::select! { + biased; + pty = self.pty_rx.recv() => { + let Some(pty_line) = pty else { + // Pump exited: Vivado's PTY closed. + // Stop trying to drain it but keep waiting + // for a Response on the protocol socket — + // most teardown sequences send the + // shutdown ack before the PTY EOFs. + continue; + }; + vw_timing_log( + "pty_rx_recv", + pty_line.len(), + &pty_line, + ); + // Shim-died detector. When Vivado catches Ctrl-C + // during a long C++ command (`place_design`, + // `phys_opt_design`, ...), its signal handler + // does `interp cancel -unwind` — and -unwind + // rips through EVERYTHING, including the shim's + // `while {1}` dispatch loop AND the `source` + // that loaded it. Vivado then drops to its + // interactive `Vivado%` prompt with no shim + // running, so our protocol socket is orphaned: + // no response will ever come for this id, and + // `read_response_for` would hang forever. + // + // Vivado announces the shim-source unwind with a + // very specific `[Common 17-344] 'source' was + // cancelled` line — precise (per-command + // cancellation like `'opt_design' was cancelled` + // is a different scope; only `'source'` means + // the shim's own `source $shim_path` was + // unwound). Match on that substring rather + // than on the `Vivado% ` prompt itself — the + // prompt has no trailing newline, so the + // line-buffered PTY pump never delivers it + // to us. + if pty_line.contains("'source' was cancelled") + { + // Shim died. Try to re-establish it in + // place — Vivado's C++ state (project, + // IP catalog, loaded designs) survives + // an `interp cancel -unwind`, only the + // Tcl-level shim/user state is lost. + // Fast respawn keeps the Vivado process + // warm (avoids the ~minutes-long + // startup) at the cost of user Tcl + // bindings needing to be reloaded. + self.shim_dead = true; + let respawn = self.respawn_shim().await; + let recovery = match respawn { + Ok(()) => "shim was respawned; \ + Vivado process + project state \ + preserved, but Tcl-level proc \ + bindings and session vars are \ + gone — re-source your entry \ + htcl to restore them", + Err(_) => "shim respawn failed; \ + session is unusable — exit vw \ + and re-launch (`:exit` in the \ + REPL, then `vw repl` / `vw run` \ + again)", + }; + return Err(BackendError::Tcl { + message: format!( + "vivado shim was torn down by \ + Ctrl-C cancellation (Vivado's \ + `interp cancel -unwind` blew \ + through the shim's dispatch \ + loop). {recovery}." + ), + code: Some("VW_SHIM_DEAD".to_string()), + info: None, + stdout: std::mem::take(&mut accumulated), + }); + } + self.handle_pty_line_during_eval( + &pty_line, + &mut accumulated, + ); + continue; + } + read = self.proto_read.recv() => { + // `recv()` on `UnboundedReceiver` is + // cancellation-safe (per tokio's docs on + // mpsc::Receiver::recv), so a losing branch + // in this `select!` just drops a poll — no + // in-flight bytes to corrupt. + let Some(res) = read else { + return Err(BackendError::Worker( + "vivado shim closed protocol socket".into(), + )); + }; + let Ok(l) = res else { + return Err(BackendError::Io(res.unwrap_err())); + }; + line = l; + } + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let msg: WireMessage = + serde_json::from_str(trimmed).map_err(|e| { + BackendError::Worker(format!( + "malformed message from shim: {e}; payload={trimmed}" + )) + })?; + match msg { + WireMessage::Stream(s) if s.id == expected_id => { + if let Some(sink) = self.stdout_sink.as_mut() { + // Default: shim-stream chunks are user + // `puts` output (StreamKind::Stdout). But + // our send_msg_id override in the shim + // also emits via this path — those chunks + // start with a Vivado-standard severity + // prefix (`WARNING:`/`ERROR:`/etc.) which + // we re-use the PTY-line classifier to + // detect. + let kind = classify_chunk_for_sink(&s.data); + // Provenance marker (opt-in via + // VW_TRACE_MESSAGE_SOURCES), only on + // classified (non-Stdout) chunks — plain + // user output doesn't benefit from a + // "came via the shim" tag, but a warning + // does so the user can tell it from a + // PTY-routed one. + if self.trace_message_sources + && kind != StreamKind::Stdout + { + sink( + StreamKind::Info, + &format!( + "[vw-shim-stream] \ + classified-as={kind:?}\n" + ), + ); + } + sink(kind, &s.data); + } else { + // Same separator rule as `emit_pty_chunk` — + // if the shim's incoming chunk is a + // classified WARNING/ERROR and the tail of + // `accumulated` isn't newline-terminated, + // inject one so the failure-block renderer + // sees line-bounded messages instead of + // `…clk_in1ERROR:…` smashes. + let kind = classify_chunk_for_sink(&s.data); + if matches!( + kind, + StreamKind::Warning | StreamKind::Error + ) && !accumulated.is_empty() + && !accumulated.ends_with('\n') + { + accumulated.push('\n'); + } + accumulated.push_str(&s.data); + } + } + WireMessage::Stream(s) => { + warn!( + got = s.id, + expected = expected_id, + "stream id mismatch; discarding" + ); + } + WireMessage::Response(r) if r.id == expected_id => { + vw_timing_log( + "response_arrival", + trimmed.len(), + &format!( + "id={} kind={}", + r.id, + match &r.result { + ResponseResult::Ok { .. } => "Ok", + ResponseResult::Err { .. } => "Err", + } + ), + ); + // Force-flush any buffered PTY message — a + // classified line that arrived right before + // the Vivado response would otherwise linger + // in the classifier until the next eval's + // drain. Flushing here means the user always + // sees every classified message attributable + // to the eval before the eval's result. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut accumulated); + } + return Ok((r, accumulated)); + } + WireMessage::Response(r) => { + warn!( + got = r.id, + expected = expected_id, + "response id mismatch; discarding" + ); + } + WireMessage::Rpc(call) => { + // Shim-initiated RPC (a `vw::…` proc reaching + // back into Rust). Dispatch synchronously here + // — we're the only reader of `proto_read` + // during an eval, and we're the writer of + // `proto_write` too, so replying inline keeps + // ownership simple and avoids a second writer + // task. The htcl caller is blocked on + // `gets $sock` waiting for exactly this + // response; other in-flight RPCs on other + // evals aren't a concern (only one eval runs + // at a time on the shim). + self.dispatch_rpc_call(call).await?; + } + } + } + } + + /// Handle one inbound [`RpcCall`], write the response back to + /// the shim. + async fn dispatch_rpc_call( + &mut self, + call: vw_eda::protocol::RpcCall, + ) -> Result<(), BackendError> { + let result = match &self.rpc_handler { + Some(handler) => handler.call(&call.method, call.args).await, + None => Err(format!( + "no RPC handler configured; can't answer '{}'", + call.method + )), + }; + let resp = match result { + Ok(value) => Response::ok(call.id, value), + Err(msg) => Response::err( + call.id, + vw_eda::protocol::ErrorPayload { + message: msg, + code: None, + info: None, + }, + ), + }; + // Reuse the same write path Rust's own Requests take. + // Writing to `proto_write` while `proto_read` is being + // consumed here (we own both, we're inside `read_ + // response_for`) is safe: TCP is full-duplex and no other + // task holds `proto_write` while `eval` is in progress. + let mut line = serde_json::to_string(&resp)?; + line.push('\n'); + self.proto_write.write_all(line.as_bytes()).await?; + self.proto_write.flush().await?; + Ok(()) + } + + /// Filter a PTY line received during an in-flight eval. Lines + /// matching Vivado's standard message format are forwarded to + /// the stdout sink (or accumulated when there's no sink); + /// everything else (banner, source-echo, idle chatter) is + /// dropped — or stderr-mirrored when `verbose` is set, so a + /// user diagnosing a flaky eval can still get the full firehose. + fn handle_pty_line_during_eval( + &mut self, + line: &str, + accumulated: &mut String, + ) { + if self.consume_ctx_marker(line) { + return; + } + if is_vivado_known_noise(line) { + // Verbose still logs it so the transcript is intact + // when someone's debugging what Vivado emitted. + if self.verbose { + self.write_verbose_line(line); + } + return; + } + let outcome = + self.pty_classifier.handle(line, std::time::Instant::now()); + for (kind, text) in outcome.chunks { + self.emit_pty_chunk(kind, &text, accumulated); + } + if !outcome.absorbed { + // Unclassified PTY output DURING an eval — the tail + // of a multi-line Vivado error (`[BD 41-758] … valid + // clock source:` followed by unindented `/pin` list), + // or arbitrary chatter the user's command triggered. + // Route as Stdout so the user can SEE it in the + // scrollback without it inheriting the previous + // warning/error's styling. Verbose-log too, for + // parity with the previous behavior. + self.emit_pty_chunk(StreamKind::Stdout, line, accumulated); + if self.verbose { + self.write_verbose_line(line); + } + } + } + + /// Recognize one of the `__VW_CTX_*` lines the shim emits + /// around wrapped commands and user proc bodies. Returns + /// `true` if the line was a marker (and should be swallowed); + /// `false` if it's a normal PTY line for the classifier. + /// + /// The marker protocol is stack-based: BEGIN opens a new + /// entry, FRAME lines accumulate into it, READY seals it onto + /// `pty_context_stack`, END pops the top entry. Nested proc + /// calls produce nested BEGIN/END pairs, and the top of the + /// stack — the innermost wrap in flight — is what tags any + /// traceless warning/error that arrives while it's active. + fn consume_ctx_marker(&mut self, line: &str) -> bool { + let stripped = line.trim_end_matches(['\r', '\n']); + match stripped { + "__VW_CTX_BEGIN__" => { + self.building_pty_context.clear(); + true + } + "__VW_CTX_READY__" => { + let frames = std::mem::take(&mut self.building_pty_context); + self.pty_context_stack.push(frames); + true + } + "__VW_CTX_END__" => { + self.pty_context_stack.pop(); + // Defensively clear building too — a stray FRAME + // that arrived after END shouldn't leak into the + // next window. + self.building_pty_context.clear(); + true + } + _ => { + if let Some(frame) = stripped.strip_prefix("__VW_CTX_FRAME__:") + { + self.building_pty_context.push(frame.to_string()); + true + } else { + false + } + } + } + } + + /// Drain whatever PTY lines have queued up between evals. Two + /// classes of line show up here in practice: + /// + /// 1. **Shim startup logs** (`[vw-shim] ...`) — emitted by + /// Vivado-startup-time code before the user's first eval. + /// Routed via the sink as Info so the user can see + /// whether the send_msg_id override installed. + /// 2. **Vivado messages emitted between evals** (e.g., delayed + /// `WARNING:` from a previous eval's async work, or + /// initialization messages fired by Vivado without any + /// eval in flight). Same deal — route to sink so the user + /// sees them in scrollback. + /// + /// Unclassified lines (banner, source-echo, the Vivado prompt) + /// still drop on the floor — or stderr-mirror if `verbose`. + /// Forwarding those would flood scrollback. + /// Post-Err PTY drain — see the call site in [`Self::eval`] for + /// the timing rationale. Polls `pty_rx` with a 10 ms per-wait + /// timeout so a quiescent channel exits fast; caps total wait + /// at 50 ms so a truly stuck Vivado can't hold the REPL longer + /// than that. Each retrieved line is fed through + /// [`Self::handle_pty_line_during_eval`] — same code path as an + /// in-flight eval — so marker BEGIN/END events pop the stack + /// coherently and error messages carry the failed eval's + /// frames when they reach the REPL sink. + /// Recover from a cancelled-shim state. Called by + /// [`Self::read_response_for`] when it sees the `Vivado%` + /// prompt in the PTY — that means Vivado's SIGINT handler + /// did `interp cancel -unwind` and the shim's dispatch loop + /// unwound all the way through the `source` that loaded it, + /// dropping Vivado back to interactive Tcl. + /// + /// Recovery steps: + /// 1. Drop the orphaned protocol socket + its reader task. + /// 2. Write `source \n` to the PTY master — + /// Vivado's interactive prompt sees it and re-runs the + /// shim. `VW_PROTOCOL_ADDR` is still set in the child + /// env, so the re-sourced shim connects back on the + /// same listener. + /// 3. Accept the new connection (with the same connect + /// timeout as the initial spawn) and rewire + /// [`Self::proto_read`] / [`Self::proto_write`]. + /// 4. Clear [`Self::shim_dead`] so subsequent evals can + /// run. + /// + /// Post-condition: Vivado is warm — project, IP catalog, + /// C++-level state all survive — but every Tcl-level user + /// binding (proc definitions from htcl `src`, `set` vars, + /// namespace state) is gone. Callers that had lowered + /// procs into the session need to reload them. + /// + /// Returns `Ok(())` on successful respawn. On failure + /// leaves `shim_dead` latched so future evals fast-fail + /// with a clear error. + async fn respawn_shim(&mut self) -> Result<(), BackendError> { + // Cancel the old reader task so it doesn't fight the + // new one when the old socket goes away. + if let Some(handle) = self._proto_read_task.take() { + handle.abort(); + } + // Drop the old socket ends — closing our side lets any + // buffered writes from the dead shim get discarded, and + // the listener stays open on the same port for accept. + // We can't literally "close" via `drop` here because + // `proto_write` is not Option; replace it below when we + // wire up the new stream. + // Write the `source` line to the PTY. `_master.take_writer()` + // returns a `Box`; each call gives a fresh + // handle so we can grab one just for this write. + let mut writer = self._master.take_writer().map_err(|e| { + BackendError::Worker(format!( + "respawn_shim: cannot obtain PTY writer: {e}" + )) + })?; + let source_cmd = format!("source {}\n", self.shim_path.display()); + use std::io::Write as _; + writer.write_all(source_cmd.as_bytes()).map_err(|e| { + BackendError::Worker(format!("respawn_shim: PTY write failed: {e}")) + })?; + writer.flush().map_err(|e| { + BackendError::Worker(format!("respawn_shim: PTY flush failed: {e}")) + })?; + drop(writer); + // Await the new shim's connection on the retained + // listener. + let listener = self.listener.as_ref().ok_or_else(|| { + BackendError::Worker( + "respawn_shim: no listener retained (backend was \ + constructed without one)" + .into(), + ) + })?; + let accept_result = + tokio::time::timeout(SHIM_CONNECT_TIMEOUT, listener.accept()).await; + let stream = match accept_result { + Ok(Ok((s, _peer))) => s, + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(_) => { + return Err(BackendError::Worker( + "respawn_shim: timed out waiting for re-sourced \ + shim to connect back" + .into(), + )); + } + }; + stream.set_nodelay(true).map_err(BackendError::Io)?; + let (read_half, write_half) = stream.into_split(); + let (proto_tx, proto_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut proto_buf = BufReader::new(read_half); + let task = tokio::spawn(async move { + let mut line = String::new(); + loop { + line.clear(); + match proto_buf.read_line(&mut line).await { + Ok(0) => break, + Ok(_) => { + if proto_tx.send(Ok(line.clone())).is_err() { + break; + } + } + Err(e) => { + let _ = proto_tx.send(Err(e)); + break; + } + } + } + }); + self.proto_read = proto_rx; + self.proto_write = write_half; + self._proto_read_task = Some(task); + self.shim_dead = false; + Ok(()) + } + + async fn settle_late_pty_after_err(&mut self) { + use std::time::{Duration, Instant}; + let mut sink_void = String::new(); + let hard_deadline = Instant::now() + Duration::from_millis(50); + loop { + let now = Instant::now(); + if now >= hard_deadline { + break; + } + let per_wait = Duration::from_millis(10); + match tokio::time::timeout(per_wait, self.pty_rx.recv()).await { + Ok(Some(line)) => { + self.handle_pty_line_during_eval(&line, &mut sink_void); + } + _ => break, // idle window elapsed or channel closed + } + } + // Flush any classifier-pending message so it emits with + // THIS eval's context, not the next one's. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + } + + fn drain_pty_between_evals(&mut self) { + // Force-flush any pending PTY message from the previous + // eval first — if the eval ended right after a classified + // line and before any continuation could arrive, we want + // it surfaced before whatever the drain finds. + let mut sink_void = String::new(); + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + while let Ok(line) = self.pty_rx.try_recv() { + if self.consume_ctx_marker(&line) { + continue; + } + let outcome = + self.pty_classifier.handle(&line, std::time::Instant::now()); + for (kind, text) in outcome.chunks { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + if !outcome.absorbed && self.verbose { + self.write_verbose_line(&line); + } + } + // Flush again at end of drain — a classified line that + // landed right before the drain stopped might still be + // buffered. No new lines will arrive before the next + // eval's read_response_for, so we'd rather surface this + // now than wait. + if let Some((kind, text)) = self.pty_classifier.flush() { + self.emit_pty_chunk(kind, &text, &mut sink_void); + } + } + + /// Write one verbose-firehose line — to the log file when + /// configured, otherwise to vw's stderr. Used for unclassified + /// PTY lines we'd otherwise discard. Errors silently because + /// dropping a verbose line shouldn't break the eval. + fn write_verbose_line(&mut self, line: &str) { + if let Some(w) = self.verbose_log.as_mut() { + let _ = writeln!(w, "{line}"); + let _ = w.flush(); + } else { + let _ = writeln!(std::io::stderr(), "{line}"); + } + } + + /// Emit one classified PTY chunk: the optional gray provenance + /// marker (when `trace_message_sources` is on) followed by the + /// chunk itself. Used by both the in-eval and between-eval + /// classification paths so the marker / sink-vs-accumulator + /// rule lives in exactly one place. + /// + /// `[vw-*]` self-diagnostic chunks (shim install logs, future + /// internal tracers) are suppressed unless trace is on. Most + /// users don't care that the shim connected to a port and + /// installed an override — that's housekeeping noise. When + /// something goes wrong, set `VW_TRACE_MESSAGE_SOURCES=1` to + /// surface both these chunks AND the per-message provenance + /// markers. + fn emit_pty_chunk( + &mut self, + kind: StreamKind, + text: &str, + accumulated: &mut String, + ) { + if !self.trace_message_sources && is_vw_log_chunk(text) { + return; + } + // Tag warnings/errors that arrived without a trace with the + // innermost active context (the top of `pty_context_stack` + // — frames captured by the shim around the in-flight C++ + // call or user proc body). This is the path that resolves + // "IP_Flow 19-7090" and friends — they go straight from + // Vivado's C++ to the PTY, bypassing every Tcl-side + // stack-capture hook. + let tagged: String; + let payload: &str = if let Some(frames) = self + .pty_context_stack + .last() + .filter(|_| { + matches!(kind, StreamKind::Warning | StreamKind::Error) + && !text.contains("\n at ") + }) + .filter(|f| !f.is_empty()) + { + let trimmed = text.trim_end_matches('\n'); + let mut buf = String::with_capacity(text.len() + 80); + buf.push_str(trimmed); + for frame in frames { + buf.push_str("\n at "); + buf.push_str(frame); + } + // Restore the trailing newline if the caller had one + // — downstream chunk handling assumes line-terminated. + if text.ends_with('\n') { + buf.push('\n'); + } + tagged = buf; + &tagged + } else { + text + }; + if let Some(sink) = self.stdout_sink.as_mut() { + if self.trace_message_sources { + sink( + StreamKind::Info, + &format!("[vw-pty] classified-as={kind:?}\n"), + ); + } + sink(kind, payload); + } else { + // A classified WARNING/ERROR is always the start of a + // new logical message from Vivado. The PTY pump can + // split chunks at arbitrary byte boundaries, so the + // preceding chunk may lack a trailing newline — if we + // just push_str, we get e.g. `/clocky/clk_in1ERROR: [BD + // 41-1031]…` in the failure block. Inject a separator + // so downstream renderers see line-bounded messages. + if matches!(kind, StreamKind::Warning | StreamKind::Error) + && !accumulated.is_empty() + && !accumulated.ends_with('\n') + { + accumulated.push('\n'); + } + accumulated.push_str(payload); + } + } +} + +/// True when the chunk's first non-whitespace content matches one +/// of our `[vw-*]` self-diagnostic prefixes (see [`VW_LOG_PREFIXES`]). +/// Used by [`VivadoBackend::emit_pty_chunk`] to suppress these +/// chunks when trace isn't enabled. +pub(crate) fn is_vw_log_chunk(text: &str) -> bool { + let trimmed = text.trim_start(); + VW_LOG_PREFIXES.iter().any(|p| trimmed.starts_with(p)) +} + +/// True for lines Vivado is known to emit but which carry no signal +/// the user needs at REPL / `vw run` level. Filtered out during +/// eval (verbose mode still records them in the log). Keep this +/// list tight — filter only stuff Vivado has no knob for and no +/// downstream tool cares about. +/// +/// Current entries: +/// +/// - `Wrote : ` — status echo emitted by `save_bd_design`, +/// `write_bd_tcl`, `write_xci`, etc. No Vivado parameter +/// suppresses it (searched `list_param` for wrote / write / +/// save / persist / log / quiet / silent / banner / bd. / +/// verbose / echo / message / command — zero matches). +/// +/// If a genuine Vivado parameter shows up in a later release, drop +/// the corresponding pattern here. +pub(crate) fn is_vivado_known_noise(line: &str) -> bool { + // Vivado uses two spaces between "Wrote" and ":" — match on + // the whole `Wrote : <` prefix rather than just "Wrote" so + // a genuine user `puts "Wrote foo"` doesn't get filtered. + let trimmed = line.trim_start(); + trimmed.starts_with("Wrote : <") +} + +#[async_trait] +impl EdaBackend for VivadoBackend { + fn name(&self) -> &str { + "vivado" + } + + async fn eval(&mut self, tcl: &str) -> Result { + if self.shim_dead { + // Fast-fail after a prior eval detected the shim was + // torn down by cancellation — writing to the orphaned + // socket would either buffer silently or block, and + // reading a response would hang forever. Surface the + // same message the initial detection did so every + // subsequent eval says the same clear thing. + return Err(BackendError::Tcl { + message: "vivado shim is dead (torn down by an \ + earlier Ctrl-C cancellation). The session \ + cannot recover — exit vw and re-launch \ + (`:exit` in the REPL, then `vw repl` / \ + `vw run` again)." + .to_string(), + code: Some("VW_SHIM_DEAD".to_string()), + info: None, + stdout: String::new(), + }); + } + self.drain_pty_between_evals(); + let id = self.alloc_id(); + let req = Request { + id, + op: RequestOp::Eval { tcl: tcl.into() }, + }; + self.write_request(&req).await?; + let (resp, stdout) = self.read_response_for(id).await?; + // Vivado writes the Err response to the protocol socket + // *before* the underlying error text and marker cleanup + // reach the PTY — empirical: 65 μs between an Err response + // arrival and the [BD 41-71] error line landing on our + // pump thread's `read(2)`; another ~180 μs before the + // pump forwards it into `pty_rx`. Returning here without + // draining those bytes leaves them queued in `pty_rx` + // until the NEXT eval's `drain_pty_between_evals`, which + // runs after `pending_eval_index` has advanced — so the + // origin fallback in the REPL tags them with a completely + // unrelated command. Poll briefly for the tail so the + // late messages get emitted with THIS eval's marker + // context still on the stack. Err-only: successful evals + // don't have this misattribution risk (any tail markers + // just adjust the stack idempotently), and running it on + // every eval would add ~1 ms to each of the ~1800 + // auto-load evals. + if matches!(resp.result, ResponseResult::Err { .. }) { + self.settle_late_pty_after_err().await; + } + match resp.result { + ResponseResult::Ok { result, .. } => { + let value = match result { + serde_json::Value::String(s) => s, + other => other.to_string(), + }; + Ok(EvalOutput { value, stdout }) + } + ResponseResult::Err { error, .. } => Err(BackendError::Tcl { + message: error.message, + code: error.code, + info: error.info, + stdout, + }), + } + } + + async fn send( + &mut self, + mut request: Request, + ) -> Result { + self.drain_pty_between_evals(); + if request.id == 0 { + request.id = self.alloc_id(); + } + let id = request.id; + self.write_request(&request).await?; + let (resp, _stdout) = self.read_response_for(id).await?; + Ok(resp) + } + + fn set_stdout_sink(&mut self, sink: StdoutSink) { + self.stdout_sink = Some(sink); + } + + async fn shutdown(&mut self) -> Result<(), BackendError> { + if self.child.is_none() { + return Ok(()); + } + let id = self.alloc_id(); + let req = Request { + id, + op: RequestOp::Shutdown, + }; + let _ = self.write_request(&req).await; + let _ = self.read_response_for(id).await; + if let Some(mut child) = self.child.take() { + // Vivado's tear-down is slow; bound it. + let waited = tokio::task::spawn_blocking(move || { + let deadline = + std::time::Instant::now() + Duration::from_secs(10); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => { + if std::time::Instant::now() >= deadline { + let _ = child.kill(); + return child.wait(); + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } + }) + .await; + match waited { + Ok(Ok(status)) => debug!(?status, "vivado exited"), + Ok(Err(e)) => return Err(BackendError::Io(e)), + Err(e) => warn!(?e, "vivado wait join error"), + } + } + if let Some(handle) = self.stdout_pump.take() { + let _ = handle.join(); + } + Ok(()) + } +} + +impl Drop for VivadoBackend { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + } + // The pump thread will exit on its own when the PTY master is + // dropped and the read returns EOF. + } +} + +/// Cancel whatever the vivado at `pid` is running, without killing it. +/// +/// Split out from [`VivadoBackend::interrupt`] because the two callers cannot +/// both hold the backend. An interactive session interrupts from the UI while +/// an eval has the backend borrowed, and an agent interrupts on behalf of a +/// developer whose keyboard is on another continent — neither can take a +/// `&mut`, and the part that is easy to get wrong should exist once. +/// +/// Unix-only. Windows has no analogue that reliably reaches the child's Tcl +/// signal handler, and every vw interactive user is on Unix. +#[cfg(unix)] +pub fn interrupt_process_group(pid: u32) { + // SAFETY: `libc::kill(-pid, SIGINT)` on a process group we spawned. + // Negative pid selects the pgid — necessary because vivado's on-disk + // binary is a bash wrapper that forks loader+vivado as children without + // `exec`, so signalling the pid alone would only reach bash, which does + // not forward it. The documented failures are EPERM (we don't own the + // group — impossible, we are the parent) and ESRCH (the group is empty — + // a benign race with the child exiting). + unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGINT); + } +} + +/// Windows stub. See the unix variant. +#[cfg(not(unix))] +pub fn interrupt_process_group(_pid: u32) {} + +/// Pump Vivado's PTY output in the background. +/// +/// Pump Vivado's process stdout into the worker as a stream of +/// newline-split lines. Runs on a blocking std thread because +/// `portable_pty` only exposes a synchronous `Read`. +/// +/// We always *read* the bytes — otherwise the PTY backpressures and +/// Vivado eventually blocks. Splitting on `\n` here (rather than +/// shipping raw chunks) keeps line semantics consistent for the +/// downstream message-line filter, which works one line at a time. +/// `\r` is stripped — Vivado's PTY output sometimes contains CRLF. +/// +/// The thread exits when the PTY closes (Vivado died) or the +/// receiver is dropped (the worker shut down). +fn spawn_stdout_pump( + mut reader: Box, + tx: tokio::sync::mpsc::UnboundedSender, + mut raw_log: Option>, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + let mut line = String::new(); + loop { + match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + // Raw byte-log tee. Write BEFORE any line + // processing so the log is a byte-perfect + // record of what Vivado emitted — CR/LF + // preserved, non-UTF-8 bytes preserved, tables + // and banners preserved. Flush per chunk so a + // `tail -f` from another terminal stays near + // real-time. Errors are ignored: dropping a log + // write should never take out the eval channel. + if let Some(w) = raw_log.as_mut() { + let _ = w.write_all(&buf[..n]); + let _ = w.flush(); + } + // DIAGNOSTIC (temporary): log every `read(2)` + // completion so we can see when Vivado actually + // wrote bytes to the PTY. Paired with the + // response-arrival and pty_rx.recv() logs in + // `read_response_for`, this tells us whether a + // late message is a pump-forwarding lag or a + // Vivado-flush lag. + let preview: String = std::str::from_utf8(&buf[..n]) + .unwrap_or("") + .chars() + .take(140) + .collect(); + vw_timing_log("pump_read", n, &preview); + for &b in &buf[..n] { + if b == b'\n' { + let send = std::mem::take(&mut line); + vw_timing_log("pump_send", send.len(), &send); + if tx.send(send).is_err() { + return; + } + } else if b != b'\r' { + // Best-effort UTF-8 — replace bytes that + // aren't valid mid-line. Vivado's stdout + // is ASCII in practice. + line.push(b as char); + } + } + } + Err(e) => { + debug!(error = %e, "pty read error"); + break; + } + } + } + // Flush any partial trailing line so EOF doesn't swallow the + // last unterminated message. + if !line.is_empty() { + let _ = tx.send(line); + } + }) +} + +/// Classify `line` as a Vivado standard-format message and +/// translate the prefix into the [`StreamKind`] the sink should +/// receive. Returns `None` for lines that don't match the +/// `common::send_msg_id` prefix set — those are banner / +/// source-echo / idle chatter and don't reach the sink. +/// +/// Conservative on purpose: a false-negative just means a useful +/// line is dropped (recoverable with `verbose=true` for users who +/// need the full firehose), but a false-positive injects banner / +/// source-echo noise into every eval's output, which would degrade +/// the REPL experience for everyone. Leading whitespace is allowed +/// because Vivado occasionally indents within scripted blocks. +/// Classify a multi-line shim-stream chunk for sink routing. The +/// chunk's first line determines its kind: a Vivado-standard +/// severity prefix routes to the matching [`StreamKind`]; +/// anything else falls back to [`StreamKind::Stdout`] (the +/// chunk is treated as user `puts` output). +/// +/// Continuation lines (the `at file:line in proc` frames the +/// send_msg_id override appends) inherit the first-line kind by +/// virtue of being part of the same chunk. The downstream +/// renderer treats each chunk as one scrollback entry. +pub(crate) fn classify_chunk_for_sink(chunk: &str) -> StreamKind { + let first = chunk.lines().next().unwrap_or(""); + classify_vivado_message_line(first).unwrap_or(StreamKind::Stdout) +} + +pub(crate) fn classify_vivado_message_line(line: &str) -> Option { + let l = line.trim_start(); + // `CRITICAL WARNING:` must be checked BEFORE `WARNING:` because + // the latter is a prefix of the former when leading whitespace + // is trimmed. `CRITICAL WARNING:` routes to Error rather than + // Warning — in Vivado's severity hierarchy it's between WARNING + // and ERROR but semantically means "your run may fail because + // of this" (bad connection, missing property, etc.), which + // reads like a hard failure to the user. Rendering it with the + // same red ✗ style as ERROR matches how a reader treats it. + if l.starts_with("CRITICAL WARNING:") { + Some(StreamKind::CriticalWarning) + } else if l.starts_with("ERROR:") { + Some(StreamKind::Error) + } else if l.starts_with("WARNING:") { + Some(StreamKind::Warning) + } else if l.starts_with("INFO:") { + Some(StreamKind::Info) + } else if VW_LOG_PREFIXES.iter().any(|p| l.starts_with(p)) { + // Our own diagnostics — see [`VW_LOG_PREFIXES`] for the + // canonical list. All route as Info: they're gray "where + // did this come from" markers, not warnings. The + // allowlist (rather than a generic `starts_with("[vw-")`) + // prevents a user's `puts "[vw-mystuff] hi"` from getting + // accidentally absorbed. + Some(StreamKind::Info) + } else { + None + } +} + +/// Allowlist of prefixes our shim and worker emit for self- +/// diagnostics. Any line starting with one of these classifies +/// as [`StreamKind::Info`]. +pub(crate) const VW_LOG_PREFIXES: &[&str] = + &["[vw-shim]", "[vw-pty]", "[vw-shim-stream]"]; + +/// Window during which a classified PTY line will absorb a +/// following unclassified line as a continuation. Vivado +/// occasionally emits multi-line messages where the severity +/// prefix only sits on the first line; treating an +/// immediately-following unclassified line as part of the same +/// message renders the warning as one scrollback entry instead +/// of two. +/// +/// 20ms is well above the inter-line latency our PTY pump sees +/// for a single Vivado write (which is sub-millisecond) but +/// well below human reaction time, so a real follow-up message +/// from a *different* call site can't be misattributed. +pub(crate) const PTY_CONTINUATION_WINDOW: std::time::Duration = + std::time::Duration::from_millis(20); + +/// Per-message buffer the worker uses to merge a multi-line PTY +/// warning into one chunk. See [`PtyClassifier`] for the merge +/// semantics. +#[derive(Debug, Clone)] +struct PendingPtyMessage { + kind: StreamKind, + text: String, + arrived_at: std::time::Instant, +} + +/// Outcome of feeding one PTY line through [`PtyClassifier`]. +#[derive(Debug, Default)] +pub(crate) struct ClassifyOutcome { + /// Chunks ready for the sink (or accumulator) in arrival + /// order. At most one *new* classified chunk per call; an + /// additional preceding entry appears only when this call + /// flushed a previously-pending message (either because a + /// new classified line arrived or the window expired). + pub chunks: Vec<(StreamKind, String)>, + /// True when the classifier took responsibility for the + /// input line (stored it as pending, or appended it to a + /// pending message). False when the line was an unclassified + /// non-continuation — caller may stderr-mirror it. + pub absorbed: bool, +} + +/// Brief-buffer classifier for PTY lines. Holds one classified +/// message at a time; an unclassified line arriving within +/// [`PTY_CONTINUATION_WINDOW`] gets folded into it, so a multi- +/// line Vivado warning whose first line carries the severity +/// prefix renders as a single chunk (and thus a single scrollback +/// entry on the App side). +/// +/// Pure / time-injected so it's unit-testable without setting up +/// a worker. +#[derive(Debug)] +pub(crate) struct PtyClassifier { + pending: Option, + window: std::time::Duration, +} + +impl PtyClassifier { + pub fn new(window: std::time::Duration) -> Self { + Self { + pending: None, + window, + } + } + + /// Feed one PTY line. `now` is when the line arrived (taken + /// as a parameter so tests can drive the clock). + pub fn handle( + &mut self, + line: &str, + now: std::time::Instant, + ) -> ClassifyOutcome { + let mut out = ClassifyOutcome::default(); + if let Some(kind) = classify_vivado_message_line(line) { + // A new classified line starts a new pending. Flush + // whatever was pending first — same path as the + // window-expired case below. + if let Some(prev) = self.pending.take() { + out.chunks + .push((prev.kind, with_trailing_newline(&prev.text))); + } + self.pending = Some(PendingPtyMessage { + kind, + text: line.to_string(), + arrived_at: now, + }); + out.absorbed = true; + return out; + } + // Unclassified. Maybe a continuation of the current + // pending warning/error? We only fold for Warning/Error + // kinds because Info messages (`[vw-shim] ...` and + // Vivado `INFO:`) are always single-line in practice — + // and absorbing into them swallowed Vivado's source-echo + // of our shim script (`# catch {...}`, `# while {1} {`, + // etc.) which arrived inside the window during boot. + // Restricting to Warning|Error covers the case we + // actually care about (Vivado occasionally emits multi- + // line WARNING/ERROR text with `\n` between the header + // and a body) without the noise. + if let Some(p) = self.pending.as_mut() { + let merges = matches!( + p.kind, + StreamKind::Warning + | StreamKind::CriticalWarning + | StreamKind::Error + ); + // Only a leading-whitespace non-empty line is a + // real continuation. Vivado indents every body line + // of a multi-line message (e.g. the `.xci` path + // that follows `[Vivado_Tcl 4-393]`), so an + // unindented line — or a blank line acting as a + // separator — signals a new, unrelated message + // (Vivado's `Attempting to get a license…` status + // echoes are the motivating case). Reject them + // even inside the merge window; otherwise those + // status lines glom onto the previous warning and + // inherit its orange/red styling. Non-continuation + // unclassified lines return `absorbed=false` — the + // during-eval caller routes them to Stdout so they + // still surface (e.g. the `/pin` list following + // `[BD 41-758] … clock source:`), just without the + // pending's severity style. + let looks_like_continuation = + line.chars().next().is_some_and(char::is_whitespace); + if merges + && looks_like_continuation + && now.duration_since(p.arrived_at) < self.window + { + p.text.push('\n'); + p.text.push_str(line); + // Refresh the arrival time so a chain of + // continuation lines all qualifies, not just the + // first. + p.arrived_at = now; + out.absorbed = true; + return out; + } + // Either kind doesn't merge, the shape doesn't + // qualify as a continuation, or the window expired — + // flush the pending. The current line itself is not + // absorbed; caller may stderr-mirror it. + let p = self.pending.take().unwrap(); + out.chunks.push((p.kind, with_trailing_newline(&p.text))); + } + out + } + + /// Force-flush any pending message. Called at eval end so a + /// message buffered right before the Vivado response doesn't + /// linger unseen. + pub fn flush(&mut self) -> Option<(StreamKind, String)> { + self.pending + .take() + .map(|p| (p.kind, with_trailing_newline(&p.text))) + } +} + +fn with_trailing_newline(s: &str) -> String { + if s.ends_with('\n') { + s.to_string() + } else { + format!("{s}\n") + } +} + +fn resolve_vivado(config: &VivadoConfig) -> Result { + if let Some(path) = &config.vivado { + return Ok(path.clone()); + } + if let Ok(env) = std::env::var("VW_VIVADO") { + if !env.is_empty() { + return Ok(PathBuf::from(env)); + } + } + if let Some(paths) = std::env::var_os("PATH") { + for dir in std::env::split_paths(&paths) { + let candidate = dir.join("vivado"); + if candidate.is_file() { + return Ok(candidate); + } + } + } + Err(BackendError::Worker( + "could not find `vivado` on PATH; set $VW_VIVADO or pass \ + VivadoConfig::vivado" + .into(), + )) +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{ + classify_chunk_for_sink, classify_vivado_message_line, is_vw_log_chunk, + PtyClassifier, StreamKind, + }; + + fn classifier(window_ms: u64) -> PtyClassifier { + PtyClassifier::new(Duration::from_millis(window_ms)) + } + + #[test] + fn classifier_emits_classified_line_only_on_flush() { + // A classified line gets buffered, not emitted, until + // either a new classified line arrives or flush() is + // called. This is what enables the multi-line-message + // merge that follows. + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] hi", t0); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + let flushed = c.flush().expect("pending must flush"); + assert_eq!(flushed.0, StreamKind::Warning); + assert_eq!(flushed.1, "WARNING: [X 1-1] hi\n"); + } + + #[test] + fn classifier_absorbs_indented_continuation_within_window() { + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] header", t0); + assert!(out.absorbed); + // Real Vivado multi-line messages indent every body line. + let out = c.handle(" second body line", t0 + Duration::from_millis(5)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + // A third continuation works too (window refreshes on + // each absorb). + let out = c.handle(" third line", t0 + Duration::from_millis(15)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + let (kind, text) = c.flush().unwrap(); + assert_eq!(kind, StreamKind::Warning); + assert_eq!( + text, + "WARNING: [X 1-1] header\n second body line\n third line\n" + ); + } + + /// Regression: an unindented follow-up line (e.g. Vivado's + /// `Attempting to get a license for feature 'Synthesis'…` + /// status echo) must NOT be absorbed as a continuation of the + /// preceding WARNING. Real Vivado multi-line messages indent + /// every body line — an unindented line is a separate message, + /// even when it lands inside the merge window. + #[test] + fn classifier_does_not_absorb_unindented_line_as_continuation() { + let mut c = classifier(20); + let t0 = Instant::now(); + assert!(c.handle("WARNING: [X 1-1] header", t0).absorbed); + // Indented body line: legitimate continuation. + let out = + c.handle(" /path/to/file.xci", t0 + Duration::from_millis(2)); + assert!(out.chunks.is_empty(), "{:?}", out.chunks); + assert!(out.absorbed); + // Unindented follow-up: NOT a continuation. Pending + // flushes, current line is reported as not-absorbed. + let out = c.handle( + "Attempting to get a license for feature 'Synthesis'", + t0 + Duration::from_millis(4), + ); + assert_eq!(out.chunks.len(), 1, "{:?}", out.chunks); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!( + out.chunks[0].1, + "WARNING: [X 1-1] header\n /path/to/file.xci\n" + ); + assert!(!out.absorbed, "unindented follow-up must not be absorbed"); + } + + /// Regression: a blank line between the warning body and the + /// next status message must terminate the pending, not extend + /// it. Blank lines are visual separators, not warning bodies. + #[test] + fn classifier_treats_blank_line_as_message_separator() { + let mut c = classifier(20); + let t0 = Instant::now(); + assert!(c.handle("WARNING: [X 1-1] header", t0).absorbed); + assert!(c.handle(" body", t0 + Duration::from_millis(1)).absorbed); + let out = c.handle("", t0 + Duration::from_millis(2)); + assert_eq!(out.chunks.len(), 1, "{:?}", out.chunks); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] header\n body\n"); + assert!(!out.absorbed); + } + + #[test] + fn classifier_flushes_pending_when_window_expires() { + let mut c = classifier(20); + let t0 = Instant::now(); + let out = c.handle("WARNING: [X 1-1] one", t0); + assert!(out.absorbed); + // Unclassified line arrives well past the window: + // pending flushes, line itself is reported as + // not-absorbed so the caller may stderr-mirror it. + let out = c.handle("a", t0 + Duration::from_millis(50)); + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] one\n"); + assert!(!out.absorbed); + // Nothing further pending. + assert!(c.flush().is_none()); + } + + #[test] + fn classifier_flushes_previous_pending_on_new_classified() { + // Two classified lines back-to-back: the first flushes + // as soon as the second arrives, and the second becomes + // the new pending. + let mut c = classifier(20); + let t0 = Instant::now(); + c.handle("WARNING: [X 1-1] one", t0); + let out = c.handle("ERROR: [Y 1-1] two", t0 + Duration::from_millis(5)); + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Warning); + assert_eq!(out.chunks[0].1, "WARNING: [X 1-1] one\n"); + assert!(out.absorbed); + let (kind, text) = c.flush().unwrap(); + assert_eq!(kind, StreamKind::Error); + assert_eq!(text, "ERROR: [Y 1-1] two\n"); + } + + #[test] + fn vw_log_chunk_detection() { + // Recognized chunks for emit-time suppression. + assert!(is_vw_log_chunk( + "[vw-shim] installed send_msg_id override\n" + )); + assert!(is_vw_log_chunk("[vw-pty] classified-as=Warning\n")); + assert!(is_vw_log_chunk(" [vw-shim-stream] classified-as=Error\n")); + // Real message content stays — never accidentally + // suppress a Vivado warning that mentions our tag in its + // body. + assert!(!is_vw_log_chunk("WARNING: [Common 17-1] no\n")); + assert!(!is_vw_log_chunk( + "INFO: [Common 17-1] something about [vw-shim]\n" + )); + assert!(!is_vw_log_chunk("")); + } + + #[test] + fn classifier_info_kind_does_not_absorb_continuations() { + // Regression guard: Info-kind pending (typically a + // [vw-shim] log line or a Vivado INFO line) must NOT + // absorb subsequent unclassified lines, because those + // are almost always unrelated content arriving in the + // same time window (Vivado source-echo during boot, + // banner lines, etc.). Only Warning/Error kinds merge. + let mut c = classifier(20); + let t0 = Instant::now(); + c.handle("[vw-shim] installed send_msg_id override", t0); + let out = c.handle( + "# catch {::vw::install_send_msg_override}", + t0 + Duration::from_millis(5), + ); + // The pending Info flushed as its own chunk; the source- + // echo line was NOT absorbed. + assert_eq!(out.chunks.len(), 1); + assert_eq!(out.chunks[0].0, StreamKind::Info); + assert_eq!( + out.chunks[0].1, + "[vw-shim] installed send_msg_id override\n" + ); + assert!(!out.absorbed); + } + + #[test] + fn classifier_drops_unclassified_lines_when_no_pending() { + let mut c = classifier(20); + let out = c.handle("plain output, no prefix", Instant::now()); + assert!(out.chunks.is_empty()); + assert!(!out.absorbed); + assert!(c.flush().is_none()); + } + + #[test] + fn classifies_each_standard_prefix_to_its_stream_kind() { + let cases = [ + ( + "ERROR: [Common 17-53] No open project. ...", + StreamKind::Error, + ), + ( + "WARNING: [Coretcl 2-1184] no open project", + StreamKind::Warning, + ), + ( + "CRITICAL WARNING: [Vivado 12-180] ...", + // Critical warnings classify as their own kind so + // log-level filtering can treat them distinctly from + // ERROR. Downstream renderers may still paint them + // with the red ✗ treatment, but the classifier keeps + // the semantic distinction intact. + StreamKind::CriticalWarning, + ), + ( + "INFO: [Vivado 12-3661] auto-pinning enabled", + StreamKind::Info, + ), + ]; + for (line, expected) in cases { + assert_eq!( + classify_vivado_message_line(line), + Some(expected), + "wrong classification for: {line}" + ); + } + } + + #[test] + fn classifies_lines_with_leading_whitespace() { + // Scripted blocks sometimes indent messages — still a + // Vivado-formatted line, still useful to surface. + assert_eq!( + classify_vivado_message_line(" ERROR: [X 1-2] indented"), + Some(StreamKind::Error) + ); + assert_eq!( + classify_vivado_message_line("\tWARNING: [X 1-2] tabbed"), + Some(StreamKind::Warning) + ); + } + + #[test] + fn drops_banner_and_source_echo_and_chatter() { + for line in [ + "", + "Vivado v2024.2 (64-bit)", + "SW Build 5095499 on Wed Nov 13 22:37:05 MST 2024", + "Copyright 1986-2024 Xilinx, Inc.", + "Vivado% ", // The interactive prompt + "source /tmp/vw-vivado-shim/vivado-shim.tcl -notrace", + "create_bd_design metroid", + "errors not at start of line: ERROR: foo", + // Has the substring but not at the start — looks like + // shell or log output, not a message-system line. + "[2024-01-01 12:00] INFO: forwarded by other tool", + ] { + assert_eq!( + classify_vivado_message_line(line), + None, + "should drop: {line:?}" + ); + } + } + + #[test] + fn does_not_match_partial_prefixes() { + // `ERRORS:` would be a hypothetical other label and we + // shouldn't false-positive on it. + assert_eq!(classify_vivado_message_line("ERRORS: bogus prefix"), None); + assert_eq!( + classify_vivado_message_line("INFOMERCIAL: not a message"), + None + ); + } + + #[test] + fn plain_chunk_routes_to_stdout() { + // User `puts hi` produces an ordinary stdout chunk — + // nothing matches the Vivado prefix set so it falls + // through to Stdout. + assert_eq!(classify_chunk_for_sink("hi\n"), StreamKind::Stdout); + assert_eq!( + classify_chunk_for_sink("progress: 42%\n"), + StreamKind::Stdout + ); + assert_eq!(classify_chunk_for_sink(""), StreamKind::Stdout); + } + + #[test] + fn classified_chunk_with_stack_inherits_first_line_kind() { + // The exact shape our send_msg_id override produces: a + // severity-prefixed first line followed by `at ...` + // continuation frames. The whole chunk should route to + // the kind matching the first line, so the warning and + // its stack stay together as one orange (or red) entry. + let warning_with_stack = "WARNING: [Common 17-1496] tclapp out of date\n\ + \x20\x20at /opt/Vivado/foo.tcl:42 in ::tclapp::loader\n\ + \x20\x20at /opt/Vivado/init.tcl:10\n"; + assert_eq!( + classify_chunk_for_sink(warning_with_stack), + StreamKind::Warning + ); + + let error_with_stack = "ERROR: [BD 5-148] no open project\n\ + \x20\x20at /opt/Vivado/bd.tcl:99 in ::bd::create\n"; + assert_eq!( + classify_chunk_for_sink(error_with_stack), + StreamKind::Error + ); + } + + #[test] + fn shim_log_lines_route_as_info() { + // Our shim's own log lines start with `[vw-shim]`. They're + // diagnostic-level info — we don't want them to look like + // a hot error, just a "here's what the worker said" + // notice in the scrollback. + assert_eq!( + classify_vivado_message_line( + "[vw-shim] installed send_msg_id override" + ), + Some(StreamKind::Info) + ); + assert_eq!( + classify_vivado_message_line( + "[vw-shim] ::common::send_msg_id not present; skipping override" + ), + Some(StreamKind::Info) + ); + // The matcher uses an explicit allowlist — see + // VW_LOG_PREFIXES. Each member routes as Info; anything + // outside the list does NOT match, even when it bears + // the `[vw-` shape. + for prefix in super::VW_LOG_PREFIXES { + let line = format!("{prefix} something"); + assert_eq!( + classify_vivado_message_line(&line), + Some(StreamKind::Info), + "should classify our known prefix: {prefix}" + ); + } + // Look-alike inside our namespace that we DIDN'T sanction + // (a future shim subsystem nobody added to the allowlist, + // or a user's puts that happens to bracket `[vw-*]`) is + // rejected — keeps the classifier conservative. + assert_eq!(classify_vivado_message_line("[vw-mystuff] foo"), None); + assert_eq!(classify_vivado_message_line("[other] foo"), None); + } + + #[test] + fn classified_chunk_with_leading_indent_still_routes() { + // `classify_vivado_message_line` tolerates leading + // whitespace; `classify_chunk_for_sink` should inherit + // that — Vivado occasionally indents scripted messages. + let chunk = " WARNING: [X 1-2] indented\n at foo:1\n"; + assert_eq!(classify_chunk_for_sink(chunk), StreamKind::Warning); + } + + /// Regression test for `VivadoBackend::interrupt`'s + /// process-group signalling. + /// + /// The `vivado` on-disk binary is a bash wrapper that runs + /// `loader -exec vivado ...` WITHOUT `exec` — so bash stays + /// as our direct child (which is what `Child::process_id` + /// returns) and the actual Vivado runtime is a grandchild. + /// A naive `kill(pid, SIGINT)` hits bash-running-a-script, + /// which does NOT forward SIGINT to its child; Vivado never + /// sees it. `interrupt` fixes this by signalling the process + /// GROUP (`kill(-pid, SIGINT)`) — under portable-pty every + /// spawned command becomes its own session leader with a + /// fresh PGID equal to its pid, and children inherit that + /// PGID, so the group signal reaches everyone. + /// + /// This test mirrors the shape without needing Vivado: bash + /// spawns a long `sleep` in the background then waits for + /// it. Same lack-of-`exec` as the Vivado wrapper. The group + /// signal must terminate both bash AND the sleep within a + /// small bounded time (a signal that lands takes + /// milliseconds; anything past 3s means we missed the + /// target). + /// + /// If this test starts failing under a future portable-pty + /// release, the fix in `interrupt` needs revisiting — some + /// alternate mechanism (writing ETX to the PTY, using + /// `process_group_leader`, or an out-of-band `interp cancel` + /// through the shim protocol) will be needed instead. + #[cfg(unix)] + #[test] + fn interrupt_reaches_wrapped_grandchild_via_process_group() { + use portable_pty::{native_pty_system, CommandBuilder, PtySize}; + use std::time::{Duration, Instant}; + + let pair = native_pty_system() + .openpty(PtySize { + rows: 24, + cols: 80, + pixel_width: 0, + pixel_height: 0, + }) + .expect("openpty"); + + // Mirror the Vivado wrapper's shape: bash spawns a + // long-running child WITHOUT `exec`, then waits. The + // background `sleep` is the stand-in for Vivado's + // `loader`/runtime; `wait $!` is what keeps bash in the + // foreground so it can propagate the eventual exit. + let mut cmd = CommandBuilder::new("bash"); + cmd.arg("-c"); + cmd.arg("sleep 300 & wait $!"); + let mut child = + pair.slave.spawn_command(cmd).expect("spawn bash wrapper"); + // Match VivadoBackend::spawn: drop the slave so the + // master sees EOF at child exit. + drop(pair.slave); + let pid = child.process_id().expect("child pid"); + + // Give bash a beat to fork off the `sleep` before we + // signal. Without this the test races between spawn and + // signal delivery. + std::thread::sleep(Duration::from_millis(300)); + + // The exact call `VivadoBackend::interrupt` makes. + unsafe { + libc::kill(-(pid as libc::pid_t), libc::SIGINT); + } + + let deadline = Instant::now() + Duration::from_secs(3); + loop { + match child.try_wait() { + Ok(Some(_status)) => return, // pass + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + panic!( + "wrapper + grandchild did not exit \ + within 3s of group SIGINT — signalling \ + mechanism is broken" + ); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => panic!("try_wait error: {e}"), + } + } + } +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000..1c3f739 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "xtask" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Development tasks for the vw workspace" +publish = false + +[dependencies] +rcgen = "0.14.8" +camino.workspace = true +clap.workspace = true +thiserror.workspace = true +time = "0.3" diff --git a/xtask/src/devcerts.rs b/xtask/src/devcerts.rs new file mode 100644 index 0000000..8a08d69 --- /dev/null +++ b/xtask/src/devcerts.rs @@ -0,0 +1,158 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Generate a self-signed certificate for running `vw-svc --tls` locally. +//! +//! ```text +//! cargo xtask devcerts +//! cargo run -p vw-svc -- serve --tls \ +//! --cert-file target/devcert/cert.pem \ +//! --key-file target/devcert/key.pem +//! vw cloud --url https://localhost:2727 --insecure list +//! ``` +//! +//! The certificate is its own issuer and is trusted by nothing, so clients +//! still have to be told to accept it — `--insecure` for `vw cloud`, `-k` for +//! curl. It exists so the TLS path can be exercised, not to provide any real +//! assurance. + +use std::fs; + +use camino::{Utf8Path, Utf8PathBuf}; +use rcgen::{ + CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, + IsCa, KeyPair, KeyUsagePurpose, +}; +use time::{Duration, OffsetDateTime}; + +use crate::DevcertArgs; + +/// Where the certificate goes when the caller does not say. +/// +/// Under `target/` so it is disposable and already ignored by git. +pub const DEFAULT_DIR: &str = "target/devcert"; + +/// Names every generated certificate is valid for, so that a service reached +/// as `localhost` and one reached as `127.0.0.1` both work out of the box. +const DEFAULT_SUBJECT_ALT_NAMES: [&str; 3] = ["localhost", "127.0.0.1", "::1"]; + +/// What the certificate calls itself, so it is recognizable in a browser or in +/// `openssl x509 -text` output. +const COMMON_NAME: &str = "vw development certificate"; + +const CERT_FILE: &str = "cert.pem"; +const KEY_FILE: &str = "key.pem"; + +#[derive(Debug, thiserror::Error)] +pub enum DevcertError { + #[error("{0} already exists; pass --force to replace it")] + Exists(Utf8PathBuf), + #[error("creating {0}: {1}")] + CreateDir(Utf8PathBuf, #[source] std::io::Error), + #[error("writing {0}: {1}")] + Write(Utf8PathBuf, #[source] std::io::Error), + #[error("generating certificate: {0}")] + Generate(#[from] rcgen::Error), + #[error("a certificate cannot be valid for {0} days")] + Validity(u16), +} + +pub fn run(args: DevcertArgs) -> Result<(), DevcertError> { + let cert_path = args.dir.join(CERT_FILE); + let key_path = args.dir.join(KEY_FILE); + + // Check both before writing either, so a refusal does not leave behind a + // certificate whose key was never replaced. + if !args.force { + for path in [&cert_path, &key_path] { + if path.exists() { + return Err(DevcertError::Exists(path.clone())); + } + } + } + + let subject_alt_names: Vec = DEFAULT_SUBJECT_ALT_NAMES + .iter() + .map(|name| (*name).to_owned()) + .chain(args.subject_alt_names.iter().cloned()) + .collect(); + + let key = KeyPair::generate()?; + let cert = params(&subject_alt_names, args.days)?.self_signed(&key)?; + + fs::create_dir_all(&args.dir) + .map_err(|e| DevcertError::CreateDir(args.dir.clone(), e))?; + write(&cert_path, cert.pem().as_bytes(), false)?; + // The key is only good for a throwaway service, but there is no reason to + // leave it world readable. + write(&key_path, key.serialize_pem().as_bytes(), true)?; + + println!("wrote {cert_path}"); + println!("wrote {key_path}"); + println!("valid for {} days, for:", args.days); + for name in &subject_alt_names { + println!(" {name}"); + } + println!(); + println!("run the service with it:"); + println!( + " cargo run -p vw-svc -- serve --tls \\\n \ + --cert-file {cert_path} --key-file {key_path}" + ); + println!("clients must be told to accept it, e.g. `vw cloud --insecure`"); + + Ok(()) +} + +/// Certificate parameters for a server certificate valid for `days` days. +fn params( + subject_alt_names: &[String], + days: u16, +) -> Result { + let mut params = CertificateParams::new(subject_alt_names.to_vec())?; + + let now = OffsetDateTime::now_utc(); + params.not_before = now; + params.not_after = now + .checked_add(Duration::days(days.into())) + .ok_or(DevcertError::Validity(days))?; + + let mut name = DistinguishedName::new(); + name.push(DnType::CommonName, COMMON_NAME); + params.distinguished_name = name; + + // Say outright that this is a leaf, not an authority. Certificates that + // omit this are rejected by rustls as `CaUsedAsEndEntity`, which is a + // confusing way to learn that a hand-rolled cert was wrong. + params.is_ca = IsCa::ExplicitNoCa; + params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyEncipherment, + ]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + + Ok(params) +} + +/// Write `contents` to `path`, restricting it to the current user when +/// `private`. +fn write( + path: &Utf8Path, + contents: &[u8], + private: bool, +) -> Result<(), DevcertError> { + fs::write(path, contents) + .map_err(|e| DevcertError::Write(path.to_owned(), e))?; + + #[cfg(unix)] + if private { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)) + .map_err(|e| DevcertError::Write(path.to_owned(), e))?; + } + #[cfg(not(unix))] + let _ = private; + + Ok(()) +} diff --git a/xtask/src/external.rs b/xtask/src/external.rs new file mode 100644 index 0000000..2ac857c --- /dev/null +++ b/xtask/src/external.rs @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Tasks that live in another crate. +//! +//! Every development task should be findable by running `cargo xtask`, but not +//! every one should be paid for by everybody. The OpenAPI manager pulls in the +//! whole API surface and its schema machinery; somebody who only wants a +//! development certificate should not wait for that to compile. +//! +//! So the subcommand is here and the work is elsewhere. [`External`] swallows +//! every argument it is given — including `--help`, so the real tool answers +//! that rather than this one — and hands them to `cargo run` for the crate +//! that does the work. The process is replaced rather than spawned, so exit +//! codes, signals and terminal behaviour are the tool's own. +//! +//! The pattern is maghemite's, which took it from omicron. + +use std::ffi::OsString; +use std::os::unix::process::CommandExt; +use std::process::Command; + +use clap::Parser; + +/// Argument parser for a task implemented in another crate. +#[derive(Debug, Parser)] +#[command( + disable_help_flag(true), + disable_help_subcommand(true), + disable_version_flag(true) +)] +pub struct External { + #[arg(trailing_var_arg(true), allow_hyphen_values(true))] + args: Vec, +} + +impl External { + /// Run `bin` from `package`, passing on everything this was given. + /// + /// Only returns if the tool could not be started at all; otherwise this + /// process becomes that one. + pub fn exec(self, package: &str, bin: &str) -> Result<(), ExternalError> { + // The same cargo that invoked this xtask, so a `+toolchain` or a + // rustup shim in play stays in play. + let cargo = std::env::var_os("CARGO") + .unwrap_or_else(|| OsString::from("cargo")); + + let error = Command::new(&cargo) + .args(["run", "--quiet", "--package", package, "--bin", bin]) + .arg("--") + .args(self.args) + .exec(); + + Err(ExternalError { + bin: bin.to_owned(), + source: error, + }) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("cannot run {bin}")] +pub struct ExternalError { + bin: String, + #[source] + source: std::io::Error, +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000..3f08c61 --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Development tasks for the vw workspace. +//! +//! Run through the workspace alias: +//! +//! ```text +//! cargo xtask devcerts # generate a self-signed cert for `vw-svc --tls` +//! cargo xtask openapi # manage the checked-in OpenAPI documents +//! ``` +//! +//! One entry point, so `cargo xtask` on its own lists everything a developer +//! can do here. Tasks with heavy dependencies live in their own crates and are +//! reached through [`external`], so having them listed costs nothing to +//! somebody who does not run them. + +use camino::Utf8PathBuf; +use clap::{Parser, Subcommand}; + +mod devcerts; +mod external; + +#[derive(Parser)] +#[command(name = "xtask")] +#[command(about = "Development tasks for the vw workspace")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + #[command(about = "Generate a self-signed certificate for local https")] + Devcerts(DevcertArgs), + + #[command( + about = "Manage the checked-in OpenAPI documents", + long_about = "Manage the checked-in OpenAPI documents under \ + `openapi/`. Run `cargo xtask openapi --help` for what \ + it can do; `generate` writes them from the API traits \ + and `check` fails if what is on disk is out of date." + )] + Openapi(external::External), +} + +#[derive(Parser)] +pub struct DevcertArgs { + /// Directory to write `cert.pem` and `key.pem` into. + #[arg(default_value = devcerts::DEFAULT_DIR)] + pub dir: Utf8PathBuf, + + /// Additional name the certificate should be valid for. A DNS name or an + /// IP address; may be given more than once. + #[arg(long = "san", value_name = "NAME")] + pub subject_alt_names: Vec, + + /// How many days the certificate is valid for. + #[arg(long, default_value_t = 365)] + pub days: u16, + + /// Replace an existing certificate and key in the target directory. + #[arg(long)] + pub force: bool, +} + +/// Whatever went wrong, from whichever task. +#[derive(Debug, thiserror::Error)] +enum XtaskError { + #[error(transparent)] + Devcerts(#[from] devcerts::DevcertError), + #[error(transparent)] + External(#[from] external::ExternalError), +} + +fn main() { + let cli = Cli::parse(); + let result: Result<(), XtaskError> = match cli.command { + Command::Devcerts(args) => devcerts::run(args).map_err(Into::into), + // Replaces this process, so it only returns if it could not start. + Command::Openapi(external) => external + .exec("vw-openapi-manager", "vw-openapi-manager") + .map_err(Into::into), + }; + + if let Err(e) = result { + eprintln!("error: {e}"); + // The source chain matters here: "cannot run vw-openapi-manager" on + // its own does not say whether cargo is missing or the crate failed + // to build. + let mut source = std::error::Error::source(&e); + while let Some(cause) = source { + eprintln!(" caused by: {cause}"); + source = cause.source(); + } + std::process::exit(1); + } +}