diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index a78fcd7..24f178c 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -14,4 +14,5 @@ Refer to [agents.md](../agents.md) at the repository root for full architecture,
- Tests use `MockEmbeddingProvider` and in-memory SQLite (no sqlite-vec in tests).
- Run `npm run typecheck && npm run test:coverage && npm run lint` before considering work complete. Use `test:coverage` (not `test`) — CI enforces coverage thresholds (statements ≥ 75%, branches ≥ 74%, functions ≥ 75%, lines ≥ 75%) and will reject PRs that drop below them.
- Before creating a PR, use a `code-review` sub-agent to self-review your diff. Fix any issues it finds before opening the PR.
+- **Branch workflow:** All feature branches and PRs target `development`. Only `development` can be merged into `main`. When creating branches, branch from `development`. When creating PRs, set the base to `development`.
- **PR lifecycle is mandatory.** After pushing a PR, always: (1) wait for CI/CD to complete, (2) check if it passed, (3) fix failures and re-push if needed, (4) read and address all review comments, (5) verify CI is green again. A PR is not done until all checks pass and all review comments are resolved. See the "Pull Request Lifecycle" section in `agents.md` for the full workflow.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 7f94626..4c13cec 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -2,6 +2,7 @@ version: 2
updates:
- package-ecosystem: "npm"
directory: "/"
+ target-branch: "development"
schedule:
interval: "weekly"
groups:
@@ -9,14 +10,17 @@ updates:
update-types: ["minor", "patch"]
- package-ecosystem: "pip"
directory: "/sdk/python"
+ target-branch: "development"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/sdk/go"
+ target-branch: "development"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
+ target-branch: "development"
schedule:
interval: "weekly"
groups:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2058500..b9b3589 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,9 +2,9 @@ name: CI
on:
push:
- branches: [main]
+ branches: [main, development]
pull_request:
- branches: [main]
+ branches: [main, development]
workflow_call:
concurrency:
@@ -15,8 +15,8 @@ jobs:
lint-and-typecheck:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
with:
node-version: 22
cache: npm
@@ -31,8 +31,8 @@ jobs:
matrix:
node-version: [20, 22]
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
cache: npm
@@ -40,7 +40,7 @@ jobs:
- run: npm run test:coverage
- name: Upload coverage
if: matrix.node-version == 22
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: coverage
path: coverage/
@@ -51,7 +51,7 @@ jobs:
permissions:
contents: read
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: actions/dependency-review-action@v4
with:
fail-on-severity: high
@@ -60,8 +60,8 @@ jobs:
runs-on: ubuntu-latest
needs: [lint-and-typecheck, test]
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
with:
node-version: 22
cache: npm
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index f1359a0..5c57320 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -2,9 +2,9 @@ name: "CodeQL"
on:
push:
- branches: [main]
+ branches: [main, development]
pull_request:
- branches: [main]
+ branches: [main, development]
schedule:
- cron: "0 6 * * 1"
@@ -21,7 +21,7 @@ jobs:
matrix:
language: [javascript-typescript]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
with:
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 02b50c4..56ad55d 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -1,7 +1,7 @@
name: Docker
on:
push:
- branches: [main]
+ branches: [main, development]
tags: ["v*"]
pull_request:
paths: ["Dockerfile", "docker-compose.yml", ".dockerignore"]
@@ -21,7 +21,7 @@ jobs:
contents: read
packages: write
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
if: github.event_name != 'pull_request'
diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml
new file mode 100644
index 0000000..ace7c31
--- /dev/null
+++ b/.github/workflows/merge-gate.yml
@@ -0,0 +1,19 @@
+name: Merge Gate
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ enforce-source-branch:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Verify PR source is development or release-please branch
+ run: |
+ HEAD="${{ github.head_ref }}"
+ if [ "$HEAD" = "development" ] || [[ "$HEAD" == release-please--* ]]; then
+ echo "✅ Source branch '$HEAD' — merge allowed."
+ else
+ echo "::error::Only the 'development' branch (or release-please branches) can be merged into main. This PR is from '$HEAD'."
+ exit 1
+ fi
diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
index 922f668..af8c40f 100644
--- a/.github/workflows/release-please.yml
+++ b/.github/workflows/release-please.yml
@@ -19,6 +19,8 @@ jobs:
id: release
with:
release-type: node
+ target-branch: main
+ token: ${{ secrets.GH_TOKEN || github.token }}
publish:
runs-on: ubuntu-latest
@@ -28,8 +30,8 @@ jobs:
contents: read
id-token: write
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
with:
node-version: 22
cache: npm
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
index 88817fe..8aedd35 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -14,9 +14,9 @@ jobs:
runs-on: ubuntu-latest
environment: pypi
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- - uses: actions/setup-python@v5
+ - uses: actions/setup-python@v6
with:
python-version: "3.12"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d2d133b..3efe2fd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest
needs: ci
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
diff --git a/.github/workflows/sdk-go.yml b/.github/workflows/sdk-go.yml
index f091577..22b136c 100644
--- a/.github/workflows/sdk-go.yml
+++ b/.github/workflows/sdk-go.yml
@@ -11,8 +11,8 @@ jobs:
matrix:
go-version: ["1.21", "1.22"]
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-go@v5
+ - uses: actions/checkout@v6
+ - uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go-version }}
- run: cd sdk/go && go test ./... -v
diff --git a/.github/workflows/sdk-python.yml b/.github/workflows/sdk-python.yml
index cfd2e3b..eb3d89f 100644
--- a/.github/workflows/sdk-python.yml
+++ b/.github/workflows/sdk-python.yml
@@ -11,8 +11,8 @@ jobs:
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-python@v5
+ - uses: actions/checkout@v6
+ - uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- run: cd sdk/python && pip install -e ".[dev]" && pytest -v
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5cdfd6c..a48aca0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,47 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [1.3.0](https://github.com/RobertLD/libscope/compare/v1.2.3...v1.3.0) (2026-03-05)
+
+
+### Features
+
+* add allowSelfSignedCerts config for corporate TLS ([#239](https://github.com/RobertLD/libscope/issues/239)) ([858ad1c](https://github.com/RobertLD/libscope/commit/858ad1ca9e55b8f31d4878433e745e470bd1be11))
+* add source-type filter to search ([#246](https://github.com/RobertLD/libscope/issues/246)) ([#268](https://github.com/RobertLD/libscope/issues/268)) ([cb05ded](https://github.com/RobertLD/libscope/commit/cb05dedbdb028cd9749add0a9f5852e01cb9c2e5))
+* auto-suggest tags based on content analysis ([#243](https://github.com/RobertLD/libscope/issues/243)) ([#273](https://github.com/RobertLD/libscope/issues/273)) ([da158ff](https://github.com/RobertLD/libscope/commit/da158ff5f55947a39712bd7f5ae4b0d6b421bfcd))
+* bulk operations for documents ([#170](https://github.com/RobertLD/libscope/issues/170)) ([#272](https://github.com/RobertLD/libscope/issues/272)) ([7e28d2d](https://github.com/RobertLD/libscope/commit/7e28d2d03aee02059473ddac09c97fe90785abee))
+* **confluence:** add --type flag for cloud vs server auth ([#241](https://github.com/RobertLD/libscope/issues/241)) ([f29029e](https://github.com/RobertLD/libscope/commit/f29029e63b201bfa446d92c7dca7901f6032b37e))
+* context chunk expansion for search results ([#266](https://github.com/RobertLD/libscope/issues/266)) ([d087485](https://github.com/RobertLD/libscope/commit/d08748592d4bccf4bb726fe05f6d3ae79a97df80)), closes [#247](https://github.com/RobertLD/libscope/issues/247)
+* deduplicate search results by document ([#245](https://github.com/RobertLD/libscope/issues/245)) ([#269](https://github.com/RobertLD/libscope/issues/269)) ([3dd70de](https://github.com/RobertLD/libscope/commit/3dd70ded96b12fc8808bc116f28509adefe41c53))
+* document cross-references and relationship links ([#267](https://github.com/RobertLD/libscope/issues/267)) ([3151a73](https://github.com/RobertLD/libscope/commit/3151a73d1bea36106968ab5c3c99325366d8f9df)), closes [#169](https://github.com/RobertLD/libscope/issues/169)
+* saved searches with filters ([#166](https://github.com/RobertLD/libscope/issues/166)) ([#271](https://github.com/RobertLD/libscope/issues/271)) ([508d82a](https://github.com/RobertLD/libscope/commit/508d82a0d20381469c251692e96ca3e26e744f1e))
+* scheduled connector sync with cron expressions ([#178](https://github.com/RobertLD/libscope/issues/178)) ([#276](https://github.com/RobertLD/libscope/issues/276)) ([557cd3f](https://github.com/RobertLD/libscope/commit/557cd3f2c783a42e1711b161facbdb9ca18db9db))
+* support additional document formats (PDF, Word, CSV, YAML, JSON) ([#249](https://github.com/RobertLD/libscope/issues/249)) ([#275](https://github.com/RobertLD/libscope/issues/275)) ([6c4b589](https://github.com/RobertLD/libscope/commit/6c4b58913e0733ce5a8075a0f3d1ed80233132ae))
+* webhook system for document events ([#187](https://github.com/RobertLD/libscope/issues/187)) ([#274](https://github.com/RobertLD/libscope/issues/274)) ([4713d2b](https://github.com/RobertLD/libscope/commit/4713d2b20dde5dfac5d6b05dcc693b44d5fe06a7))
+* wire document update to MCP, CLI, and REST API ([#182](https://github.com/RobertLD/libscope/issues/182)) ([#270](https://github.com/RobertLD/libscope/issues/270)) ([96ced04](https://github.com/RobertLD/libscope/commit/96ced04b5c4c78b47b4e74d35cc2a20628ef9fdd))
+* wire up web dashboard via `libscope serve --dashboard` ([#265](https://github.com/RobertLD/libscope/issues/265)) ([cf16afd](https://github.com/RobertLD/libscope/commit/cf16afd0de285454f53fd101d6cfcda66d475c7c)), closes [#259](https://github.com/RobertLD/libscope/issues/259)
+
+
+### Bug Fixes
+
+* add input validation for search params, CLI options, and API responses ([#252](https://github.com/RobertLD/libscope/issues/252)) ([#260](https://github.com/RobertLD/libscope/issues/260)) ([d84de47](https://github.com/RobertLD/libscope/commit/d84de47ebbf1a11ed742d31d6fa428a1257a326a))
+* add missing database cleanup and stream reader cancellation ([#254](https://github.com/RobertLD/libscope/issues/254)) ([#262](https://github.com/RobertLD/libscope/issues/262)) ([7a3f650](https://github.com/RobertLD/libscope/commit/7a3f650b0193ec297b5b2bafcc45c16b5007fd36))
+* add request timeouts to embedding providers and RAG ([#258](https://github.com/RobertLD/libscope/issues/258)) ([e52e97f](https://github.com/RobertLD/libscope/commit/e52e97f89f6b98d6bc053a1e34f5dcd161716e0a))
+* address 5 CodeQL security alerts (SSRF, TLS, ReDoS) ([#279](https://github.com/RobertLD/libscope/issues/279)) ([e2339cd](https://github.com/RobertLD/libscope/commit/e2339cd790979ad8bb03e7ffb7eb8290c493bbd3))
+* address HIGH and MEDIUM audit findings ([#280](https://github.com/RobertLD/libscope/issues/280)) ([1e93987](https://github.com/RobertLD/libscope/commit/1e939879e1e15281e8e71f6abeb5e6013f1afd6f))
+* apply allowSelfSignedCerts to connector fetch calls ([#240](https://github.com/RobertLD/libscope/issues/240)) ([3b7b281](https://github.com/RobertLD/libscope/commit/3b7b2813e8cb2b3798cab1232dc085e7b2766d96))
+* code quality improvements from comprehensive audit ([#278](https://github.com/RobertLD/libscope/issues/278)) ([e988c63](https://github.com/RobertLD/libscope/commit/e988c63f04c497449ddfc50b55b0453c76fb9bc4))
+* comprehensive audit fixes — security, performance, resilience, API hardening ([#316](https://github.com/RobertLD/libscope/issues/316)) ([5585db5](https://github.com/RobertLD/libscope/commit/5585db5ff96304cca318b6765a743a0ee85ebb04))
+* **confluence:** use correct REST API paths for Server/Data Center ([#242](https://github.com/RobertLD/libscope/issues/242)) ([d1afeab](https://github.com/RobertLD/libscope/commit/d1afeabbc3c0da33de0fa756fa8748f0672b2bcb))
+* connector bugs — pagination, recursion, rate limiting, auth ([#257](https://github.com/RobertLD/libscope/issues/257)) ([49fef8b](https://github.com/RobertLD/libscope/commit/49fef8b88363fbe57b4437fc38bd26675b8f1f3e))
+* error handling gaps — counters, silent failures, version pruning ([#255](https://github.com/RobertLD/libscope/issues/255)) ([#264](https://github.com/RobertLD/libscope/issues/264)) ([b20cebe](https://github.com/RobertLD/libscope/commit/b20cebedf05532e3c1f83fa4984e9349400ee54f))
+* fall back to github.token if GH_TOKEN secret not set ([5bf987d](https://github.com/RobertLD/libscope/commit/5bf987d03733c3431b2bae51abd8f21d3b45899b))
+* **mcp:** expose relevance scores in search results ([#248](https://github.com/RobertLD/libscope/issues/248)) ([760ce21](https://github.com/RobertLD/libscope/commit/760ce213f21adc59be67453ea773800ad2fbf866))
+* **mcp:** use workspace-aware database path ([#244](https://github.com/RobertLD/libscope/issues/244)) ([0ddba5a](https://github.com/RobertLD/libscope/commit/0ddba5ad5a4490f630912eed933b5e3a1acfb61e))
+* move saveVersion inside transaction in updateDocument ([#256](https://github.com/RobertLD/libscope/issues/256)) ([#263](https://github.com/RobertLD/libscope/issues/263)) ([2afeae1](https://github.com/RobertLD/libscope/commit/2afeae190d1e1231f822ddd2c1f07a0b7a950b69))
+* remove unsafe non-null assertions and fix type casts ([#253](https://github.com/RobertLD/libscope/issues/253)) ([#261](https://github.com/RobertLD/libscope/issues/261)) ([e549a89](https://github.com/RobertLD/libscope/commit/e549a89eb562b01cb43d4d36e4db38cda09fd4ff))
+* use OS DNS resolver fallback for internal hostnames ([#237](https://github.com/RobertLD/libscope/issues/237)) ([3beb234](https://github.com/RobertLD/libscope/commit/3beb234a8365906552869a69c1a906acdd0cf993))
+
## [1.2.3](https://github.com/RobertLD/libscope/compare/v1.2.2...v1.2.3) (2026-03-02)
### Features
diff --git a/README.md b/README.md
index fa5577d..79add05 100644
--- a/README.md
+++ b/README.md
@@ -36,13 +36,15 @@ On first run with the default embedding provider, LibScope downloads the [all-Mi
LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats are available via optional dependencies:
-| Format | Extension | Optional Dependency | Node.js Requirement |
-| ------ | --------- | ------------------- | ------------------- |
-| PDF | `.pdf` | `pdf-parse` (v2) | ≥ 20.16 or ≥ 22.3 |
-| Word | `.docx` | `mammoth` | Any |
-| CSV | `.csv` | Built-in | Any |
+| Format | Extension | Optional Dependency | Node.js Requirement |
+| ------------ | --------- | ------------------- | ------------------- |
+| PDF | `.pdf` | `pdf-parse` (v2) | ≥ 20.16 or ≥ 22.3 |
+| Word | `.docx` | `mammoth` | Any |
+| EPUB | `.epub` | `epub2` | Any |
+| PowerPoint | `.pptx` | `pizzip` | Any |
+| CSV | `.csv` | Built-in | Any |
-The `pdf-parse` and `mammoth` packages are listed as `optionalDependencies` and install automatically when the Node.js version is compatible.
+The `pdf-parse`, `mammoth`, `epub2`, and `pizzip` packages are listed as `optionalDependencies` and install automatically when the Node.js version is compatible. Note: binary `.ppt` files are not supported — only `.pptx`.
## Using with AI Assistants
@@ -72,25 +74,34 @@ Once connected, your assistant can search docs, submit new documents, rate conte
Full list of MCP tools
-| Tool | What it does |
-| --------------------- | --------------------------------------------------------- |
-| `search-docs` | Semantic search with topic/library/version/rating filters |
-| `get-document` | Retrieve a document by ID |
-| `delete-document` | Remove a document |
-| `submit-document` | Index new content (raw text or a URL to fetch) |
-| `rate-document` | Rate a doc 1–5 with optional feedback and corrections |
-| `list-documents` | List docs with filters |
-| `list-topics` | Browse the topic hierarchy |
-| `ask-question` | RAG question-answering with source citations |
-| `reindex-documents` | Re-embed chunks (useful after switching providers) |
-| `health-check` | DB status, doc/chunk counts |
-| `sync-obsidian-vault` | Sync an Obsidian vault |
-| `sync-onenote` | Sync OneNote notebooks via Microsoft Graph |
-| `sync-notion` | Sync Notion pages and databases |
-| `sync-confluence` | Sync Confluence spaces |
-| `sync-slack` | Sync Slack channels and threads |
-| `install-pack` | Install a knowledge pack |
-| `list-packs` | List installed or registry packs |
+| Tool | What it does |
+| ---------------------- | --------------------------------------------------------- |
+| `search-docs` | Semantic search with topic/library/version/rating filters |
+| `ask-question` | RAG question-answering with source citations |
+| `get-document` | Retrieve a document by ID |
+| `list-documents` | List docs with filters |
+| `list-topics` | Browse the topic hierarchy |
+| `submit-document` | Index new content (raw text or a URL to fetch) |
+| `update-document` | Update a document's title, content, or metadata |
+| `delete-document` | Remove a document |
+| `rate-document` | Rate a doc 1–5 with optional feedback and corrections |
+| `suggest-tags` | Auto-suggest tags based on content analysis |
+| `save-search` | Save a named search query with filters |
+| `list-saved-searches` | List all saved searches |
+| `run-saved-search` | Execute a saved search by name or ID |
+| `delete-saved-search` | Delete a saved search |
+| `link-documents` | Create a cross-reference between two documents |
+| `get-document-links` | List all incoming and outgoing links for a document |
+| `delete-link` | Remove a cross-reference link |
+| `reindex-documents` | Re-embed chunks (useful after switching providers) |
+| `health-check` | DB status, doc/chunk counts |
+| `sync-obsidian-vault` | Sync an Obsidian vault |
+| `sync-onenote` | Sync OneNote notebooks via Microsoft Graph |
+| `sync-notion` | Sync Notion pages and databases |
+| `sync-confluence` | Sync Confluence spaces |
+| `sync-slack` | Sync Slack channels and threads |
+| `install-pack` | Install a knowledge pack |
+| `list-packs` | List installed or registry packs |
@@ -157,7 +168,22 @@ libscope ask "How do I configure OAuth2?" --library my-lib
libscope repl
```
-Search uses sqlite-vec for vector similarity when available, with FTS5 full-text search as a fallback.
+Search uses sqlite-vec for vector similarity when available, with FTS5 full-text search as a fallback. Results automatically get a 1.5× title boost when the document title matches query words. You can also pass `--diversity 0.5` (0–1) for MMR-based diversity reranking.
+
+### Programmatic SDK
+
+LibScope also exports a `LibScope` class for use as a library:
+
+```ts
+import { LibScope } from "libscope";
+
+const scope = LibScope.create();
+await scope.index({ title: "My Doc", content: "..." });
+const results = await scope.search("query");
+scope.close();
+```
+
+See the [Programmatic Usage](/guide/programmatic-usage) guide for details on the SDK, batch search, and document TTL/expiry.
## Organizing Content
@@ -194,17 +220,33 @@ libscope serve --api --port 3378
OpenAPI 3.0 spec at `GET /openapi.json`. Key endpoints:
-| Method | Endpoint | Description |
-| ------------ | ----------------------- | ------------------------ |
-| `GET` | `/api/v1/search?q=...` | Semantic search |
-| `GET/POST` | `/api/v1/documents` | List or create documents |
-| `GET/DELETE` | `/api/v1/documents/:id` | Get or remove a document |
-| `POST` | `/api/v1/documents/url` | Index from a URL |
-| `POST` | `/api/v1/ask` | RAG question-answering |
-| `GET/POST` | `/api/v1/topics` | List or create topics |
-| `GET` | `/api/v1/tags` | List tags |
-| `GET` | `/api/v1/stats` | Usage statistics |
-| `GET` | `/api/v1/health` | Health check |
+| Method | Endpoint | Description |
+| --------------- | --------------------------------- | ---------------------------------- |
+| `GET` | `/api/v1/search?q=...` | Semantic search |
+| `POST` | `/api/v1/batch-search` | Batch search (up to 20 queries) |
+| `POST` | `/api/v1/ask` | RAG question-answering |
+| `GET/POST` | `/api/v1/documents` | List or create documents |
+| `GET/PATCH/DELETE` | `/api/v1/documents/:id` | Get, update, or delete a document |
+| `POST` | `/api/v1/documents/url` | Index from a URL |
+| `POST` | `/api/v1/documents/:id/tags` | Add tags |
+| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags |
+| `GET/POST` | `/api/v1/documents/:id/links` | List or create cross-references |
+| `DELETE` | `/api/v1/links/:id` | Delete a cross-reference |
+| `GET/POST` | `/api/v1/topics` | List or create topics |
+| `GET` | `/api/v1/tags` | List tags |
+| `GET/POST` | `/api/v1/searches` | List or create saved searches |
+| `POST` | `/api/v1/searches/:id/run` | Run a saved search |
+| `DELETE` | `/api/v1/searches/:id` | Delete a saved search |
+| `POST` | `/api/v1/bulk/delete` | Bulk delete documents |
+| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags |
+| `POST` | `/api/v1/bulk/move` | Bulk move to a topic |
+| `GET/POST` | `/api/v1/webhooks` | List or create webhooks |
+| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook |
+| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping to a webhook |
+| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps|
+| `GET` | `/api/v1/connectors/status` | Connector sync status and history |
+| `GET` | `/api/v1/stats` | Usage statistics |
+| `GET` | `/api/v1/health` | Health check |
## Configuration
@@ -230,7 +272,7 @@ export LIBSCOPE_OPENAI_API_KEY=sk-...
The `ask` command and `ask-question` MCP tool need an LLM. Configure one with:
```bash
-export LIBSCOPE_LLM_PROVIDER=openai # or ollama
+export LIBSCOPE_LLM_PROVIDER=openai # or ollama, anthropic
export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional
```
@@ -242,8 +284,9 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional
| `LIBSCOPE_EMBEDDING_PROVIDER` | `local`, `ollama`, or `openai` | `local` |
| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — |
| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
-| `LIBSCOPE_LLM_PROVIDER` | LLM for RAG (`openai` / `ollama`) | — |
+| `LIBSCOPE_LLM_PROVIDER` | LLM for RAG (`openai` / `ollama` / `anthropic`) | — |
| `LIBSCOPE_LLM_MODEL` | LLM model override | — |
+| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — |
| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` |
| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` |
| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — |
@@ -268,7 +311,8 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional
},
"llm": {
"provider": "openai",
- "model": "gpt-4o-mini"
+ "model": "gpt-4o-mini",
+ "anthropicApiKey": "sk-ant-..."
},
"database": {
"path": "~/.libscope/libscope.db"
@@ -304,6 +348,28 @@ export LIBSCOPE_ALLOW_PRIVATE_URLS=true
export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true
```
+## Webhooks
+
+LibScope can push events to any HTTP endpoint. Useful for triggering CI pipelines, Slack notifications, or custom workflows whenever documents are created or updated.
+
+```bash
+libscope serve --api # webhooks require the REST API
+```
+
+```bash
+# Create a webhook
+curl -X POST http://localhost:3378/api/v1/webhooks \
+ -H "Content-Type: application/json" \
+ -d '{"url": "https://hooks.example.com/libscope", "events": ["document.created", "document.updated"], "secret": "my-hmac-secret"}'
+
+# Send a test ping
+curl -X POST http://localhost:3378/api/v1/webhooks//test
+```
+
+Webhook payloads are signed with HMAC-SHA256 when a secret is set. The signature is in the `X-LibScope-Signature` header.
+
+Supported events: `document.created`, `document.updated`, `document.deleted`.
+
## Other Tools
LibScope ships with a few more utilities beyond the core index-and-search loop:
@@ -352,27 +418,54 @@ There's also a web dashboard at `http://localhost:3377` when you run `libscope s
**Documents**
-| Command | Description |
-| ----------------------------------- | ----------------- |
-| `libscope docs list` | List documents |
-| `libscope docs show ` | Show a document |
-| `libscope docs delete ` | Delete a document |
-| `libscope docs history ` | Version history |
-| `libscope docs rollback ` | Roll back |
+| Command | Description |
+| --------------------------------------- | ---------------------------- |
+| `libscope docs list` | List documents |
+| `libscope docs show ` | Show a document |
+| `libscope docs update ` | Update title/content/metadata|
+| `libscope docs delete ` | Delete a document |
+| `libscope docs history ` | Version history |
+| `libscope docs rollback ` | Roll back to a prior version |
**Organization**
-| Command | Description |
-| ---------------------------------- | ---------------- |
-| `libscope topics list` | List topics |
-| `libscope topics create ` | Create a topic |
-| `libscope tag add ` | Add tags |
-| `libscope tag remove ` | Remove a tag |
-| `libscope tag list` | List tags |
-| `libscope workspace create ` | Create workspace |
-| `libscope workspace list` | List workspaces |
-| `libscope workspace use ` | Switch workspace |
-| `libscope workspace delete ` | Delete workspace |
+| Command | Description |
+| ------------------------------------ | -------------------------------- |
+| `libscope topics list` | List topics |
+| `libscope topics create ` | Create a topic |
+| `libscope tag add ` | Add tags |
+| `libscope tag remove ` | Remove a tag |
+| `libscope tag list` | List tags |
+| `libscope workspace create ` | Create workspace |
+| `libscope workspace list` | List workspaces |
+| `libscope workspace use ` | Switch workspace |
+| `libscope workspace delete ` | Delete workspace |
+
+**Saved Searches**
+
+| Command | Description |
+| ------------------------------------ | ----------------------------- |
+| `libscope searches list` | List all saved searches |
+| `libscope searches run ` | Re-run a saved search |
+| `libscope searches delete ` | Delete a saved search |
+| `libscope search --save ` | Save a search while running it|
+
+**Document Links**
+
+| Command | Description |
+| ------------------------------------- | -------------------------------- |
+| `libscope link ` | Create a cross-reference |
+| `libscope links ` | Show all links for a document |
+| `libscope unlink ` | Remove a link |
+| `libscope prereqs ` | Show prerequisite reading chain |
+
+**Bulk Operations**
+
+| Command | Description |
+| ---------------------------- | ----------------------------------- |
+| `libscope bulk delete` | Delete all matching documents |
+| `libscope bulk retag` | Add/remove tags on matching docs |
+| `libscope bulk move` | Move matching docs to a topic |
**Connectors**
@@ -432,4 +525,4 @@ npm run lint # lint
## License
-MIT — see [LICENSE](LICENSE).
+[Business Source License 1.1](LICENSE) — see [LICENSE](LICENSE) for full terms.
diff --git a/agents.md b/agents.md
index 3148a5b..1b21baa 100644
--- a/agents.md
+++ b/agents.md
@@ -35,14 +35,32 @@ LibScope is an **AI-powered knowledge base with MCP (Model Context Protocol) int
src/
├── cli/index.ts # CLI entry point (commander). All commands in one file.
├── mcp/server.ts # MCP server (stdio transport, @modelcontextprotocol/sdk)
+├── api/
+│ ├── server.ts # HTTP server bootstrap (createServer, listen)
+│ ├── routes.ts # All REST route handlers in one function (~700 lines)
+│ ├── middleware.ts # Auth (checkApiKey), rate limiting, body parsing, sendError/sendJson
+│ └── openapi.ts # OpenAPI spec generation
├── core/ # Business logic — framework-agnostic, no side effects
│ ├── indexing.ts # Document parsing, chunking by heading, embedding + storage
│ ├── search.ts # Semantic (vector) + FTS5 + LIKE fallback search
│ ├── ratings.ts # Rating storage, aggregation, correction suggestions
│ ├── documents.ts # Document CRUD
│ ├── topics.ts # Topic hierarchy management
-│ ├── url-fetcher.ts # Fetch URL → convert HTML to markdown-like text
+│ ├── bulk.ts # Bulk delete/move/retag operations with selector resolution
+│ ├── tags.ts # Tag CRUD and document–tag associations
+│ ├── dedup.ts # Content deduplication helpers
+│ ├── rag.ts # Retrieval-augmented generation (ask a question over the index)
+│ ├── url-fetcher.ts # Fetch URL → convert HTML to markdown-like text (SSRF-protected)
│ └── index.ts # Public re-exports (barrel file)
+├── connectors/ # External-service sync connectors
+│ ├── obsidian.ts # Obsidian vault sync (reads .md files + YAML frontmatter)
+│ ├── confluence.ts # Confluence Cloud sync (REST API)
+│ ├── notion.ts # Notion sync (official API)
+│ ├── onenote.ts # OneNote sync (Microsoft Graph API)
+│ ├── slack.ts # Slack channel sync (Web API)
+│ ├── sync-tracker.ts # Tracks last-synced state per connector in SQLite
+│ ├── http-utils.ts # Authenticated fetch helper shared by connectors (respects allowSelfSignedCerts)
+│ └── index.ts # Re-exports
├── db/
│ ├── connection.ts # SQLite connection + sqlite-vec extension loading
│ ├── schema.ts # Migrations (versioned) + vector table creation
@@ -146,6 +164,64 @@ Environment variables > project `.libscope.json` > user `~/.libscope/config.json
Env vars: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_OPENAI_API_KEY`, `LIBSCOPE_OLLAMA_URL`, `LIBSCOPE_ALLOW_PRIVATE_URLS`, `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS`.
+## Security Patterns
+
+### Authentication — use constant-time comparison
+
+The API key check in `src/api/middleware.ts` must use `crypto.timingSafeEqual` to prevent timing attacks. Direct string equality (`===` / `!==`) short-circuits on the first differing byte, leaking information about the key length and value.
+
+```typescript
+import { timingSafeEqual } from "node:crypto";
+
+// ✅ Correct
+const tokenBuf = Buffer.from(token);
+const keyBuf = Buffer.from(apiKey);
+if (tokenBuf.length !== keyBuf.length || !timingSafeEqual(tokenBuf, keyBuf)) {
+ return sendError(res, 401, "UNAUTHORIZED", "Invalid API key");
+}
+
+// ❌ Wrong — timing-attack vulnerable
+if (token !== apiKey) { ... }
+```
+
+### TLS — use per-request undici Agent, not process-wide env var
+
+When self-signed certificates must be accepted (controlled by `config.indexing.allowSelfSignedCerts`), configure TLS per-request via an `undici.Agent` passed as `dispatcher`. Do **not** mutate `process.env["NODE_TLS_REJECT_UNAUTHORIZED"]` — it is process-global and creates a race condition with concurrent requests.
+
+```typescript
+import { Agent } from "undici";
+
+// ✅ Correct — scoped to this request chain only
+let _insecureAgent: Agent | undefined;
+const getInsecureAgent = (): Agent =>
+ (_insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }));
+
+const response = await fetch(url, {
+ // @ts-expect-error — Node.js undici-based fetch accepts dispatcher for per-request TLS config
+ dispatcher: allowSelfSigned ? getInsecureAgent() : undefined,
+});
+
+// ❌ Wrong — affects all concurrent requests until restored
+process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+```
+
+### SSE streaming — check backpressure
+
+`res.write()` returns `false` when the socket buffer is full or the client has disconnected. Ignoring the return value wastes compute and holds open connections indefinitely.
+
+```typescript
+// ✅ Correct
+for await (const event of stream) {
+ const ok = res.write(`data: ${JSON.stringify(event)}\n\n`);
+ if (!ok) break;
+}
+
+// ❌ Wrong — no backpressure handling
+for await (const event of stream) {
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
+}
+```
+
## Testing
### Framework
diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts
index 59ca38a..44d8cf1 100644
--- a/docs/.vitepress/config.ts
+++ b/docs/.vitepress/config.ts
@@ -1,4 +1,8 @@
import { defineConfig } from "vitepress";
+import { createRequire } from "module";
+
+const require = createRequire(import.meta.url);
+const { version } = require("../../package.json");
export default defineConfig({
title: "LibScope",
@@ -23,7 +27,7 @@ export default defineConfig({
{ text: "Guide", link: "/guide/getting-started" },
{ text: "Reference", link: "/reference/cli" },
{
- text: "v1.1.0",
+ text: `v${version}`,
items: [
{
text: "Changelog",
@@ -52,6 +56,17 @@ export default defineConfig({
{ text: "MCP Setup", link: "/guide/mcp-setup" },
{ text: "Connectors", link: "/guide/connectors" },
{ text: "Knowledge Packs", link: "/guide/knowledge-packs" },
+ {
+ text: "Programmatic Usage",
+ link: "/guide/programmatic-usage",
+ },
+ ],
+ },
+ {
+ text: "Deep Dives",
+ items: [
+ { text: "How Search Works", link: "/guide/how-search-works" },
+ { text: "Troubleshooting", link: "/guide/troubleshooting" },
],
},
],
@@ -83,7 +98,7 @@ export default defineConfig({
},
footer: {
- message: "Released under the MIT License.",
+ message: "Released under the Business Source License 1.1.",
copyright: "Copyright © 2026 RobertLD",
},
},
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index 50c11fa..7a47ed8 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -76,7 +76,18 @@ export LIBSCOPE_LLM_PROVIDER=openai
export LIBSCOPE_LLM_MODEL=gpt-4o-mini
```
-Supported providers: `openai`, `ollama`.
+Supported providers: `openai`, `ollama`, `anthropic`, `passthrough`.
+
+The `anthropic` provider uses Anthropic's Claude models. Set the API key via config or environment variable:
+
+```bash
+export LIBSCOPE_LLM_PROVIDER=anthropic
+export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-...
+```
+
+Or in your config file, set `llm.provider` to `"anthropic"` and `llm.anthropicApiKey` to your key. You can optionally set `llm.model` to choose a specific Claude model.
+
+The `passthrough` provider is for advanced integrations where you supply your own LLM responses externally. When set, the `ask` command emits an event stream that your application handles rather than calling an LLM directly.
## Environment Variables
@@ -85,8 +96,9 @@ Supported providers: `openai`, `ollama`.
| `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider (`local` / `ollama` / `openai`) | `local` |
| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — |
| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
-| `LIBSCOPE_LLM_PROVIDER` | LLM provider for RAG (`openai` / `ollama`) | — |
+| `LIBSCOPE_LLM_PROVIDER` | LLM provider for RAG (`openai` / `ollama` / `anthropic`) | — |
| `LIBSCOPE_LLM_MODEL` | LLM model override | — |
+| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — |
| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` |
| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` |
| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — |
diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md
index 916fe77..d332585 100644
--- a/docs/guide/getting-started.md
+++ b/docs/guide/getting-started.md
@@ -10,7 +10,7 @@ It also runs as an [MCP server](/guide/mcp-setup), so AI assistants like Claude
npm install -g libscope
```
-Requires Node.js 18 or later.
+Requires Node.js 20 or later.
## Initialize
@@ -35,6 +35,8 @@ libscope add https://docs.example.com/guide
libscope import ./docs/ --library my-lib --extensions .md,.mdx
```
+LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats — **PDF** (`.pdf`), **Word** (`.docx`), **EPUB** (`.epub`), and **PowerPoint** (`.pptx`) — are available via optional dependencies that install automatically. See the [README](/) for the full format table.
+
Each document gets chunked by heading, embedded into vectors, and stored in the database.
## Search
@@ -51,7 +53,7 @@ Results are ranked by vector similarity, with optional FTS5 boosting for keyword
## Ask Questions
-If you have an LLM provider configured (OpenAI or Ollama), you can ask questions and get synthesized answers with source citations:
+If you have an LLM provider configured (OpenAI, Ollama, or Anthropic), you can ask questions and get synthesized answers with source citations:
```bash
libscope ask "What is the recommended auth flow?"
@@ -67,9 +69,56 @@ libscope serve
This starts a stdio-based MCP server that any compatible AI assistant can connect to. See [MCP Setup](/guide/mcp-setup) for integration instructions.
+## Web Dashboard
+
+Run the local web dashboard to browse, search, and manage your knowledge base in a browser:
+
+```bash
+libscope serve --dashboard
+# opens at http://localhost:3377
+```
+
+The dashboard includes full-text search, document browsing, topic navigation, and a knowledge graph visualization at `/graph`.
+
+## Organize and Annotate
+
+Once you have content indexed you can enrich it:
+
+```bash
+# Tag documents
+libscope tag add typescript,api,v2
+
+# Group into topics
+libscope topics create "backend"
+libscope topics create "auth" --parent backend
+
+# Save frequent searches
+libscope search "auth best practices" --save "Auth Docs"
+libscope searches run "Auth Docs"
+
+# Cross-reference documents
+libscope link --type prerequisite
+
+# Bulk operations
+libscope bulk retag --library react --add-tags deprecated --dry-run
+libscope bulk move --library react --to new-topic-id
+```
+
+## REST API
+
+For programmatic access, start the REST API instead of the MCP server:
+
+```bash
+libscope serve --api --port 3378
+```
+
+The OpenAPI 3.0 spec is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation.
+
## What's Next
- [Configuration](/guide/configuration) — embedding providers, LLM setup, environment variables
- [MCP Setup](/guide/mcp-setup) — connect LibScope to Claude, Cursor, or VS Code
- [Connectors](/guide/connectors) — sync from Obsidian, Notion, Confluence, Slack, and more
- [CLI Reference](/reference/cli) — full list of commands and options
+- [REST API Reference](/reference/rest-api) — full API endpoint documentation
+- [Programmatic Usage](/guide/programmatic-usage) — use LibScope as a Node.js library
diff --git a/docs/guide/how-search-works.md b/docs/guide/how-search-works.md
new file mode 100644
index 0000000..4f2e906
--- /dev/null
+++ b/docs/guide/how-search-works.md
@@ -0,0 +1,79 @@
+# How Search Works
+
+LibScope uses a hybrid search strategy combining vector (semantic) search with full-text search (FTS5), merged via Reciprocal Rank Fusion (RRF).
+
+## Search Pipeline
+
+### 1. Query Embedding
+Your search query is converted to a vector embedding using the configured embedding provider (local model or OpenAI). This captures the semantic meaning of your query.
+
+### 2. Vector Search (ANN)
+The query vector is matched against all indexed chunk embeddings using approximate nearest-neighbour (ANN) search via `sqlite-vec`. Results are ranked by cosine similarity — chunks semantically related to your query rank highest, even if they use different words.
+
+### 3. Full-Text Search (FTS5)
+Simultaneously, SQLite's FTS5 (BM25 ranking) searches for chunks containing your query terms. LibScope first tries AND logic (all terms must match) for precision, then falls back to OR logic if no results are found.
+
+### 4. Hybrid Fusion (RRF)
+Vector and FTS5 results are merged using **Reciprocal Rank Fusion (RRF)** — a technique that combines ranked lists without needing calibrated scores:
+
+```
+RRF_score(chunk) = Σ 1 / (k + rank_in_list)
+```
+
+where `k = 60` (standard constant). Chunks that rank well in *both* vector and FTS5 lists get the highest fused scores.
+
+### 5. Title Boost
+Chunks whose document title contains any query word receive a 1.5× score multiplier, lifting exact-title matches to the top.
+
+### 6. MMR Diversity Reranking
+
+When you set the `diversity` option (0–1), results are reranked using **Maximal Marginal Relevance (MMR)**. This penalizes results that are too similar to already-selected results, pushing diverse content higher in the list.
+
+- `diversity: 0` — pure relevance (no reranking)
+- `diversity: 0.5` — balanced relevance and diversity
+- `diversity: 1` — maximum diversity
+
+MMR is applied after title boost and score sorting. It's useful when you want to cover different aspects of a topic rather than getting multiple chunks from the same document.
+
+```bash
+libscope search "authentication" --diversity 0.5
+```
+
+### 7. Pagination & Deduplication
+Results are optionally deduplicated by document (`maxChunksPerDocument`) and paginated. Use `offset` and `limit` in your search options for pagination.
+
+## Search Methods
+
+| Method | When Used | Best For |
+|--------|-----------|---------|
+| `hybrid` | sqlite-vec available + FTS5 match | Most queries — best precision & recall |
+| `vector` | sqlite-vec available, FTS5 returns nothing | Conceptual/semantic queries |
+| `fts5` | Part of hybrid pipeline | Keyword-heavy queries |
+| `keyword` | sqlite-vec unavailable | Fallback — exact word matching only |
+
+The active method is returned in each result's `scoreExplanation.method` field.
+
+## Score Explanation
+
+Every search result includes a `scoreExplanation` object:
+
+```typescript
+{
+ method: "hybrid" | "vector" | "fts5" | "keyword",
+ rawScore: number, // raw score before boosts
+ boostFactors: string[], // e.g. ["title_match:x1.5"]
+ details: string // human-readable scoring breakdown
+}
+```
+
+## Tuning Search
+
+| Option | Default | Effect |
+|--------|---------|--------|
+| `limit` | 10 | Results per page |
+| `offset` | 0 | Pagination offset |
+| `maxChunksPerDocument` | unlimited | Max chunks returned per document |
+| `contextChunks` | 0 | Adjacent chunks to include for context (max 2) |
+| `diversity` | 0 | MMR diversity factor (0 = relevance only, 1 = max diversity) |
+| `minRating` | none | Filter by minimum avg document rating |
+| `tags` | none | Filter by document tags (AND logic) |
diff --git a/docs/guide/mcp-setup.md b/docs/guide/mcp-setup.md
index 63594e6..5db4c40 100644
--- a/docs/guide/mcp-setup.md
+++ b/docs/guide/mcp-setup.md
@@ -80,13 +80,38 @@ If you're using [workspaces](/guide/configuration#workspaces), pass the workspac
## Available Tools
-Once connected, your AI assistant gets access to all of LibScope's MCP tools. See the [MCP Tools Reference](/reference/mcp-tools) for the full list.
-
-The most commonly used ones:
-
-- **`search-docs`** — semantic search across your knowledge base
-- **`ask-question`** — RAG Q&A with synthesized answers
-- **`submit-document`** — index new content (by text or URL)
-- **`list-topics`** — browse what's in the knowledge base
+Once connected, your AI assistant gets access to all 26 of LibScope's MCP tools. See the [MCP Tools Reference](/reference/mcp-tools) for full parameter details.
+
+**Search & Q&A**
+- **`search-docs`** — semantic search with topic/library/version/rating filters
+- **`ask-question`** — RAG Q&A with synthesized answers and source citations
+
+**Document Management**
+- **`submit-document`** — index new content by text or URL
+- **`update-document`** — update title, content, or metadata
+- **`get-document`** — retrieve a document by ID
+- **`list-documents`** — list docs with filters
+- **`delete-document`** — remove a document
+- **`rate-document`** — rate 1–5 with feedback
+- **`suggest-tags`** — auto-suggest tags based on content
+
+**Organization**
+- **`list-topics`** — browse the topic hierarchy
+- **`link-documents`** — create cross-references between docs
+- **`get-document-links`** — list a document's incoming and outgoing links
+- **`delete-link`** — remove a cross-reference
+
+**Saved Searches**
+- **`save-search`** — save a named query with filters
+- **`list-saved-searches`** — list saved searches
+- **`run-saved-search`** — execute a saved search
+
+**Connectors** — trigger syncs directly from your AI assistant:
+- **`sync-obsidian-vault`**, **`sync-notion`**, **`sync-confluence`**, **`sync-slack`**, **`sync-onenote`**
+
+**Packs & Maintenance**
+- **`install-pack`**, **`list-packs`** — manage knowledge packs
+- **`reindex-documents`** — re-embed after switching providers
+- **`health-check`** — DB status and doc/chunk counts
Your AI assistant will call these tools automatically when it needs information from your docs.
diff --git a/docs/guide/programmatic-usage.md b/docs/guide/programmatic-usage.md
new file mode 100644
index 0000000..e5cb353
--- /dev/null
+++ b/docs/guide/programmatic-usage.md
@@ -0,0 +1,133 @@
+# Programmatic Usage
+
+LibScope can be used as a Node.js library via the `LibScope` SDK class.
+
+## Setup
+
+```ts
+import { LibScope } from "libscope";
+
+const scope = LibScope.create();
+```
+
+You can pass options to `create()`:
+
+```ts
+const scope = LibScope.create({
+ workspace: "my-project",
+ config: {
+ embedding: { provider: "openai" },
+ llm: { provider: "anthropic" },
+ },
+});
+```
+
+This initializes the database, runs migrations, and sets up embedding/LLM providers automatically.
+
+## Indexing Documents
+
+```ts
+const doc = await scope.index({
+ title: "Auth Guide",
+ content: "# Authentication\n\nUse OAuth2 for all API access...",
+ library: "my-lib",
+ version: "2.0.0",
+});
+
+console.log(doc.id); // document ID
+```
+
+### Document TTL / Auto-Expiry
+
+Set `expiresAt` to an ISO 8601 timestamp to mark a document for automatic expiry:
+
+```ts
+await scope.index({
+ title: "Sprint 42 Notes",
+ content: "...",
+ expiresAt: "2026-04-01T00:00:00Z",
+});
+```
+
+Expired documents are not removed automatically — call `pruneExpiredDocuments()` to clean them up:
+
+```ts
+import { pruneExpiredDocuments } from "libscope";
+
+// Using the low-level function (requires a db handle)
+const { pruned } = pruneExpiredDocuments(db);
+console.log(`Removed ${pruned} expired documents`);
+```
+
+## Searching
+
+```ts
+const { results } = await scope.search("how to authenticate", {
+ library: "my-lib",
+ limit: 10,
+ diversity: 0.3, // MMR diversity reranking (0 = pure relevance, 1 = max diversity)
+});
+
+for (const result of results) {
+ console.log(result.title, result.score);
+}
+```
+
+## Batch Search
+
+Run up to 20 search queries concurrently:
+
+```ts
+const { results } = await scope.searchBatch([
+ { query: "authentication" },
+ { query: "deployment", options: { library: "my-lib", limit: 5 } },
+]);
+
+// Results are keyed by query string
+console.log(results["authentication"].results.length);
+console.log(results["deployment"].results.length);
+```
+
+## RAG (Ask Questions)
+
+```ts
+const answer = await scope.ask("How does OAuth2 work?", {
+ library: "my-lib",
+ topK: 5,
+});
+
+console.log(answer.text);
+console.log(answer.sources); // cited chunks
+```
+
+For streaming responses:
+
+```ts
+for await (const event of scope.askStream("How does OAuth2 work?")) {
+ if (event.type === "text") process.stdout.write(event.text);
+}
+```
+
+## Other Operations
+
+```ts
+// Get stats
+const stats = scope.stats();
+
+// List documents
+const docs = scope.list({ library: "my-lib" });
+
+// Get a single document
+const doc = scope.get("doc-id");
+
+// Delete a document
+scope.delete("doc-id");
+```
+
+## Cleanup
+
+Always close the database connection when done:
+
+```ts
+scope.close();
+```
diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md
new file mode 100644
index 0000000..f69cb95
--- /dev/null
+++ b/docs/guide/troubleshooting.md
@@ -0,0 +1,92 @@
+# Troubleshooting
+
+## Installation Issues
+
+### `no such module: vec0` or `sqlite-vec` not loading
+
+**Symptom:** Errors mentioning `vec0`, `vec_each`, or `no such module` when indexing or searching.
+
+**Cause:** `sqlite-vec` requires a native Node.js addon that must be built for your platform.
+
+**Fix:**
+```bash
+npm rebuild sqlite-vec
+# or
+npm install --force
+```
+
+If you're on an unsupported platform, LibScope will fall back to keyword-only search automatically.
+
+---
+
+### Model download fails or hangs
+
+**Symptom:** First run hangs at "Downloading model..." or fails with a network error.
+
+**Cause:** The local embedding model (~80MB) downloads from Hugging Face on first use.
+
+**Fix:**
+- Check your internet connection
+- If behind a proxy, set `HTTPS_PROXY` environment variable
+- To use OpenAI embeddings instead (no download required): `libscope config set embedding.provider openai`
+
+---
+
+### Embedding dimension mismatch after switching providers
+
+**Symptom:** Error like `expected N dimensions, got M` when searching after changing the embedding provider.
+
+**Cause:** Changing embedding providers produces vectors of different dimensions. Existing embeddings in the database are incompatible.
+
+**Fix:** Re-index all documents after switching providers:
+```bash
+libscope db reset # clears all indexed content
+libscope pack install ... # re-install packs
+```
+
+---
+
+## Search Issues
+
+### Search returns no results
+
+1. Check that documents are indexed: `libscope list`
+2. Try a simpler query — FTS5 AND logic requires all terms to match
+3. Check your filters — `--library`, `--topic`, `--tags` may be too restrictive
+4. Run `libscope search "test" --limit 5` to verify basic search works
+
+### Results seem irrelevant
+
+- The local embedding model is smaller and less accurate than OpenAI — consider switching: `libscope config set embedding.provider openai`
+- Ensure documents were indexed after the embedding provider was configured
+- Try adding more context to your query
+
+---
+
+## API / MCP Issues
+
+### `401 Unauthorized` from API
+
+The REST API requires an `X-API-Key` header. Find your key: `libscope config show`
+
+### MCP tools not appearing in Claude / Cursor
+
+1. Verify the MCP server is running: `libscope mcp start`
+2. Check the MCP config path in your client settings points to the libscope server
+3. Restart your AI client after adding the MCP server
+
+---
+
+## Database Issues
+
+### Database locked errors
+
+LibScope uses SQLite WAL mode which supports concurrent reads but only one writer. If you see lock errors:
+- Ensure only one libscope process is running
+- Check for stuck processes: `ps aux | grep libscope`
+
+### How to reset the database
+
+```bash
+libscope db reset # removes all indexed content (keeps config)
+```
diff --git a/docs/index.md b/docs/index.md
index d15556a..e85e880 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -19,7 +19,7 @@ features:
details: Vector similarity search powered by sqlite-vec, with FTS5 full-text fallback. Find what you need, even when you don't know the exact words.
- icon: 🤖
title: MCP Integration
- details: 17 tools for AI assistants out of the box. Works with Claude, Cursor, VS Code, and any MCP-compatible client.
+ details: 26 tools for AI assistants out of the box. Works with Claude, Cursor, VS Code, and any MCP-compatible client.
- icon: 🔗
title: Connectors
details: Pull in docs from Obsidian, Notion, Confluence, Slack, OneNote, and GitHub/GitLab. Keep everything in one place.
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 9b0a644..e7f4821 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -283,7 +283,7 @@ libscope link --type see_also --label "Background context"
| `libscope config set ` | Set a configuration value |
| `libscope config show` | Show current configuration |
-Supported config keys for `set`: `embedding.provider`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`.
+Supported config keys for `set`: `embedding.provider`, `embedding.ollamaUrl`, `embedding.ollamaModel`, `embedding.openaiModel`, `llm.provider`, `llm.model`, `database.path`, `logging.level`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`.
## Global Options
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 13d6589..30e06aa 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -24,10 +24,12 @@ Complete reference for all configuration options.
### LLM (for RAG)
-| Key | Type | Default | Description |
-| -------------- | ------ | ------- | -------------------- |
-| `llm.provider` | string | — | `openai` or `ollama` |
-| `llm.model` | string | — | Model name override |
+| Key | Type | Default | Description |
+| ----------------- | ------ | ------- | ---------------------------------------------- |
+| `llm.provider` | string | — | `openai`, `ollama`, `anthropic`, or `passthrough` |
+| `llm.model` | string | — | Model name override |
+| `llm.ollamaUrl` | string | — | Ollama server URL (overrides embedding URL) |
+| `llm.anthropicApiKey` | string | — | Anthropic API key (or use `LIBSCOPE_ANTHROPIC_API_KEY` env var) |
### Database
@@ -57,6 +59,7 @@ Complete reference for all configuration options.
| `LIBSCOPE_OLLAMA_URL` | `embedding.ollamaUrl` | `http://localhost:11434` |
| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | — |
| `LIBSCOPE_LLM_MODEL` | `llm.model` | — |
+| `LIBSCOPE_ANTHROPIC_API_KEY` | `llm.anthropicApiKey` | — |
| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` |
| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` |
| `ONENOTE_CLIENT_ID` | OneNote app client ID | — |
@@ -69,9 +72,24 @@ Complete reference for all configuration options.
## Setting Values
```bash
-# Via CLI
-libscope config set embedding.provider ollama
-libscope config set llm.provider openai
+# Embedding
+libscope config set embedding.provider ollama # local | ollama | openai
+libscope config set embedding.ollamaUrl http://localhost:11434
+libscope config set embedding.ollamaModel nomic-embed-text
+libscope config set embedding.openaiModel text-embedding-3-small
+
+# LLM (for RAG)
+libscope config set llm.provider openai # openai | ollama | passthrough
+libscope config set llm.model gpt-4o-mini
+
+# Database
+libscope config set database.path /custom/path/libscope.db
+
+# Logging
+libscope config set logging.level debug # debug | info | warn | error | silent
+
+# Network
+libscope config set indexing.allowPrivateUrls true
libscope config set indexing.allowSelfSignedCerts true
# View current config
diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md
index c0fe6e4..71706f5 100644
--- a/docs/reference/mcp-tools.md
+++ b/docs/reference/mcp-tools.md
@@ -16,6 +16,17 @@ Semantic search across your knowledge base.
| `limit` | number | | Max results (default: 10) |
| `offset` | number | | Pagination offset |
+**Search results** include a `scoreExplanation` object on each result:
+
+```typescript
+{
+ method: "hybrid" | "vector" | "fts5" | "keyword",
+ rawScore: number, // raw score before boosts
+ boostFactors: string[], // e.g. ["title_match:x1.5"]
+ details: string // human-readable scoring breakdown
+}
+```
+
## get-document
Retrieve a document by its ID, including ratings and metadata.
@@ -45,6 +56,46 @@ Index a new document. You can provide content directly, or a URL to fetch automa
| `version` | string | | Library version |
| `topic` | string | | Topic to categorize under |
| `sourceType` | string | | `library`, `topic`, `manual`, or `model-generated` |
+| `dedup` | string | | Duplicate detection behaviour (see below) |
+| `dedupOptions` | object | | Fine-tune duplicate detection (see below) |
+
+**`dedup`** *(optional)*: Controls duplicate detection behaviour.
+- `"skip"` — If a duplicate is detected, return the existing document without re-indexing
+- `"warn"` — Log a warning about the duplicate but index anyway
+- `"force"` — Skip duplicate checking entirely and always index
+- *(omitted)* — Default behaviour: reject exact duplicates by title+content-length, allow similar content
+
+**`dedupOptions`** *(optional)*: Fine-tune duplicate detection.
+- `threshold` *(number, 0–1)*: Similarity threshold for semantic dedup (default 0.95)
+- `strategy` *(string)*: `"exact"` (hash-based) or `"semantic"` (embedding-based)
+
+## update-document
+
+Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding.
+
+| Parameter | Type | Required | Description |
+| ------------ | ------ | -------- | --------------------------------------- |
+| `documentId` | string | ✅ | The document ID to update |
+| `title` | string | | New title |
+| `content` | string | | New content (triggers re-chunking) |
+| `library` | string | | New library name (pass `null` to clear) |
+| `version` | string | | New version (pass `null` to clear) |
+| `url` | string | | New source URL (pass `null` to clear) |
+| `topicId` | string | | New topic ID (pass `null` to clear) |
+
+## update-document
+
+Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding.
+
+| Parameter | Type | Required | Description |
+| ------------ | ------ | -------- | --------------------------------------- |
+| `documentId` | string | ✅ | The document ID to update |
+| `title` | string | | New title |
+| `content` | string | | New content (triggers re-chunking) |
+| `library` | string | | New library name (pass `null` to clear) |
+| `version` | string | | New version (pass `null` to clear) |
+| `url` | string | | New source URL (pass `null` to clear) |
+| `topicId` | string | | New topic ID (pass `null` to clear) |
## rate-document
@@ -171,6 +222,42 @@ List installed or available knowledge packs.
| `available` | boolean | | If true, list from registry instead of installed |
| `registryUrl` | string | | Custom registry URL |
+## suggest-tags
+
+Suggest tags for a document based on content analysis (compares against existing tags in the knowledge base).
+
+| Parameter | Type | Required | Description |
+| ---------------- | ------ | -------- | ------------------------------------ |
+| `documentId` | string | ✅ | The document ID |
+| `maxSuggestions` | number | | Maximum suggestions to return (1–20, default: 5) |
+
+## link-documents
+
+Create a typed cross-reference relationship between two documents.
+
+| Parameter | Type | Required | Description |
+| ---------- | ------ | -------- | -------------------------------------------------------------------- |
+| `sourceId` | string | ✅ | The source document ID |
+| `targetId` | string | ✅ | The target document ID |
+| `linkType` | string | ✅ | Relationship type: `see_also`, `prerequisite`, `supersedes`, `related` |
+| `label` | string | | Optional human-readable description of the relationship |
+
+## get-document-links
+
+Get all cross-reference links for a document, both outgoing and incoming.
+
+| Parameter | Type | Required | Description |
+| ------------ | ------ | -------- | --------------- |
+| `documentId` | string | ✅ | The document ID |
+
+## delete-link
+
+Remove a cross-reference link between documents.
+
+| Parameter | Type | Required | Description |
+| --------- | ------ | -------- | ------------------------ |
+| `linkId` | string | ✅ | The link ID to delete |
+
## save-search
Save a search query with optional filters for later re-use.
diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md
index 3aa3d81..272317d 100644
--- a/docs/reference/rest-api.md
+++ b/docs/reference/rest-api.md
@@ -10,29 +10,79 @@ The OpenAPI 3.0 spec is available at `GET /openapi.json`.
## Endpoints
-| Method | Endpoint | Description |
-| -------- | ---------------------------- | -------------------------------- |
-| `GET` | `/api/v1/health` | Health check with document count |
-| `GET` | `/api/v1/search?q=...` | Semantic search |
-| `GET` | `/api/v1/documents` | List documents (with filters) |
-| `POST` | `/api/v1/documents` | Index a new document |
-| `GET` | `/api/v1/documents/:id` | Get a single document |
-| `DELETE` | `/api/v1/documents/:id` | Delete a document |
-| `POST` | `/api/v1/documents/url` | Index a document from a URL |
-| `POST` | `/api/v1/documents/:id/tags` | Add tags to a document |
-| `POST` | `/api/v1/ask` | RAG question answering |
-| `GET` | `/api/v1/topics` | List all topics |
-| `POST` | `/api/v1/topics` | Create a topic |
-| `GET` | `/api/v1/tags` | List all tags |
-| `GET` | `/api/v1/stats` | Usage statistics |
-| `GET` | `/api/v1/searches` | List saved searches |
-| `POST` | `/api/v1/searches` | Create a saved search |
-| `POST` | `/api/v1/searches/:id/run` | Run a saved search |
-| `DELETE` | `/api/v1/searches/:id` | Delete a saved search |
-| `GET` | `/openapi.json` | OpenAPI 3.0 specification |
-| `POST` | `/api/v1/bulk/delete` | Bulk delete documents |
-| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags |
-| `POST` | `/api/v1/bulk/move` | Bulk move documents to a topic |
+### Search & Q&A
+
+| Method | Endpoint | Description |
+| ------ | ------------------------- | ------------------------------------- |
+| `GET` | `/api/v1/search?q=...` | Semantic search |
+| `POST` | `/api/v1/batch-search` | Run up to 20 search queries at once |
+| `POST` | `/api/v1/ask` | RAG question-answering |
+
+### Documents
+
+| Method | Endpoint | Description |
+| -------- | ------------------------------------- | ------------------------------------- |
+| `GET` | `/api/v1/documents` | List documents (with filters) |
+| `POST` | `/api/v1/documents` | Index a new document |
+| `GET` | `/api/v1/documents/:id` | Get a single document |
+| `PATCH` | `/api/v1/documents/:id` | Update a document |
+| `DELETE` | `/api/v1/documents/:id` | Delete a document |
+| `POST` | `/api/v1/documents/url` | Index from a URL |
+| `POST` | `/api/v1/documents/:id/tags` | Add tags to a document |
+| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags based on content |
+| `GET` | `/api/v1/documents/:id/links` | List cross-reference links |
+| `POST` | `/api/v1/documents/:id/links` | Create a cross-reference link |
+
+### Document Links
+
+| Method | Endpoint | Description |
+| -------- | --------------------- | ---------------------- |
+| `DELETE` | `/api/v1/links/:id` | Delete a link |
+
+### Topics & Tags
+
+| Method | Endpoint | Description |
+| ------ | ----------------- | -------------------- |
+| `GET` | `/api/v1/topics` | List all topics |
+| `POST` | `/api/v1/topics` | Create a topic |
+| `GET` | `/api/v1/tags` | List all tags |
+
+### Saved Searches
+
+| Method | Endpoint | Description |
+| -------- | --------------------------- | ---------------------------- |
+| `GET` | `/api/v1/searches` | List saved searches |
+| `POST` | `/api/v1/searches` | Create a saved search |
+| `POST` | `/api/v1/searches/:id/run` | Run a saved search |
+| `DELETE` | `/api/v1/searches/:id` | Delete a saved search |
+
+### Bulk Operations
+
+| Method | Endpoint | Description |
+| ------ | ----------------------- | ------------------------------ |
+| `POST` | `/api/v1/bulk/delete` | Bulk delete documents |
+| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags |
+| `POST` | `/api/v1/bulk/move` | Bulk move documents to a topic |
+
+### Webhooks
+
+| Method | Endpoint | Description |
+| -------- | ------------------------------- | ----------------------------- |
+| `GET` | `/api/v1/webhooks` | List webhooks |
+| `POST` | `/api/v1/webhooks` | Create a webhook |
+| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook |
+| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping |
+
+### System
+
+| Method | Endpoint | Description |
+| ------ | -------------------------------- | ---------------------------------- |
+| `GET` | `/api/v1/health` | Health check with document count |
+| `GET` | `/api/v1/stats` | Usage statistics |
+| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps|
+| `GET` | `/api/v1/connectors/status` | Connector sync status and history |
+| `GET` | `/api/v1/connectors/schedules` | Scheduled connector entries |
+| `GET` | `/openapi.json` | OpenAPI 3.0 specification |
## Examples
@@ -44,7 +94,7 @@ curl -X POST http://localhost:3378/api/v1/documents \
-d '{
"title": "Auth Guide",
"content": "# Authentication\n\nUse OAuth2...",
- "tags": ["auth"]
+ "tags": ["auth", "security"]
}'
```
@@ -54,6 +104,27 @@ curl -X POST http://localhost:3378/api/v1/documents \
curl "http://localhost:3378/api/v1/search?q=authentication&limit=5"
```
+### Search with filters
+
+```bash
+curl "http://localhost:3378/api/v1/search?q=deploy&library=my-lib&topic=backend&limit=10"
+```
+
+### Batch search
+
+Run multiple search queries concurrently (up to 20). Results are keyed by query string.
+
+```bash
+curl -X POST http://localhost:3378/api/v1/batch-search \
+ -H "Content-Type: application/json" \
+ -d '{
+ "requests": [
+ { "query": "authentication" },
+ { "query": "deployment", "options": { "library": "my-lib", "limit": 5 } }
+ ]
+ }'
+```
+
### Ask a question
```bash
@@ -75,3 +146,73 @@ curl -X POST http://localhost:3378/api/v1/documents/url \
"library": "my-lib"
}'
```
+
+### Update a document
+
+```bash
+curl -X PATCH http://localhost:3378/api/v1/documents/ \
+ -H "Content-Type: application/json" \
+ -d '{
+ "title": "Updated Title",
+ "library": "my-lib",
+ "version": "2.0.0"
+ }'
+```
+
+### Bulk retag
+
+```bash
+curl -X POST http://localhost:3378/api/v1/bulk/retag \
+ -H "Content-Type: application/json" \
+ -d '{
+ "selector": {"library": "react"},
+ "addTags": ["v18"],
+ "removeTags": ["v17"],
+ "dryRun": false
+ }'
+```
+
+### Create a webhook
+
+```bash
+curl -X POST http://localhost:3378/api/v1/webhooks \
+ -H "Content-Type: application/json" \
+ -d '{
+ "url": "https://hooks.example.com/libscope",
+ "events": ["document.created", "document.updated"],
+ "secret": "my-hmac-secret"
+ }'
+```
+
+Webhook payloads are signed with HMAC-SHA256 when a secret is provided. The signature is sent in the `X-LibScope-Signature` header.
+
+Supported events: `document.created`, `document.updated`, `document.deleted`.
+
+### Create a cross-reference link
+
+```bash
+curl -X POST http://localhost:3378/api/v1/documents//links \
+ -H "Content-Type: application/json" \
+ -d '{
+ "targetId": "",
+ "linkType": "prerequisite",
+ "label": "Read this first"
+ }'
+```
+
+Valid `linkType` values: `see_also`, `prerequisite`, `supersedes`, `related`.
+
+### Create a saved search
+
+```bash
+curl -X POST http://localhost:3378/api/v1/searches \
+ -H "Content-Type: application/json" \
+ -d '{
+ "name": "Auth Docs",
+ "query": "authentication best practices",
+ "filters": {"library": "my-lib"}
+ }'
+
+# Run it later
+curl -X POST http://localhost:3378/api/v1/searches//run
+```
diff --git a/package-lock.json b/package-lock.json
index 8a6f4c7..41ceda2 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,17 +9,21 @@
"version": "1.2.3",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
+ "@anthropic-ai/sdk": "^0.78.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@xenova/transformers": "^2.17.2",
- "better-sqlite3": "^11.0.0",
+ "better-sqlite3": "^12.6.2",
"commander": "^14.0.3",
"csv-parse": "^6.1.0",
+ "epub2": "^3.0.2",
"js-yaml": "^4.1.1",
"node-cron": "^4.2.1",
"node-html-markdown": "^2.0.0",
"openai": "^6.25.0",
"pino": "^10.3.1",
+ "pizzip": "^3.2.0",
"sqlite-vec": "^0.1.0",
+ "undici": "^7.22.0",
"zod": "^4.3.6"
},
"bin": {
@@ -28,14 +32,14 @@
"devDependencies": {
"@types/better-sqlite3": "^7.6.0",
"@types/js-yaml": "^4.0.9",
- "@types/node": "^22.0.0",
+ "@types/node": "^25.3.3",
"@types/node-cron": "^3.0.11",
"@types/pdf-parse": "^1.1.5",
"@typescript-eslint/eslint-plugin": "^8.0.0",
"@typescript-eslint/parser": "^8.0.0",
"@vitest/coverage-v8": "^4.0.18",
- "eslint": "^9.0.0",
- "eslint-config-prettier": "^9.0.0",
+ "eslint": "^10.0.2",
+ "eslint-config-prettier": "^10.1.8",
"husky": "^9.0.0",
"lint-staged": "^16.3.1",
"prettier": "^3.0.0",
@@ -309,6 +313,26 @@
"node": ">= 14.0.0"
}
},
+ "node_modules/@anthropic-ai/sdk": {
+ "version": "0.78.0",
+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.78.0.tgz",
+ "integrity": "sha512-PzQhR715td/m1UaaN5hHXjYB8Gl2lF9UVhrrGrZeysiF6Rb74Wc9GCB8hzLdzmQtBd1qe89F9OptgB9Za1Ib5w==",
+ "license": "MIT",
+ "dependencies": {
+ "json-schema-to-ts": "^3.1.1"
+ },
+ "bin": {
+ "anthropic-ai-sdk": "bin/cli"
+ },
+ "peerDependencies": {
+ "zod": "^3.25.0 || ^4.0.0"
+ },
+ "peerDependenciesMeta": {
+ "zod": {
+ "optional": true
+ }
+ }
+ },
"node_modules/@babel/helper-string-parser": {
"version": "7.27.1",
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz",
@@ -345,6 +369,15 @@
"node": ">=6.0.0"
}
},
+ "node_modules/@babel/runtime": {
+ "version": "7.28.6",
+ "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz",
+ "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=6.9.0"
+ }
+ },
"node_modules/@babel/types": {
"version": "7.29.0",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz",
@@ -450,207 +483,74 @@
}
},
"node_modules/@eslint/config-array": {
- "version": "0.21.1",
- "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz",
- "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==",
+ "version": "0.23.2",
+ "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.2.tgz",
+ "integrity": "sha512-YF+fE6LV4v5MGWRGj7G404/OZzGNepVF8fxk7jqmqo3lrza7a0uUcDnROGRBG1WFC1omYUS/Wp1f42i0M+3Q3A==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
- "@eslint/object-schema": "^2.1.7",
+ "@eslint/object-schema": "^3.0.2",
"debug": "^4.3.1",
- "minimatch": "^3.1.2"
- },
- "engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
- }
- },
- "node_modules/@eslint/config-array/node_modules/balanced-match": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
- "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/@eslint/config-array/node_modules/brace-expansion": {
- "version": "1.1.12",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
- "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "balanced-match": "^1.0.0",
- "concat-map": "0.0.1"
- }
- },
- "node_modules/@eslint/config-array/node_modules/minimatch": {
- "version": "3.1.5",
- "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
- "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
- "dev": true,
- "license": "ISC",
- "dependencies": {
- "brace-expansion": "^1.1.7"
+ "minimatch": "^10.2.1"
},
"engines": {
- "node": "*"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
}
},
"node_modules/@eslint/config-helpers": {
- "version": "0.4.2",
- "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz",
- "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.2.tgz",
+ "integrity": "sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
- "@eslint/core": "^0.17.0"
+ "@eslint/core": "^1.1.0"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
}
},
"node_modules/@eslint/core": {
- "version": "0.17.0",
- "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz",
- "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==",
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz",
+ "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@types/json-schema": "^7.0.15"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
- }
- },
- "node_modules/@eslint/eslintrc": {
- "version": "3.3.4",
- "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.4.tgz",
- "integrity": "sha512-4h4MVF8pmBsncB60r0wSJiIeUKTSD4m7FmTFThG8RHlsg9ajqckLm9OraguFGZE4vVdpiI1Q4+hFnisopmG6gQ==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "ajv": "^6.14.0",
- "debug": "^4.3.2",
- "espree": "^10.0.1",
- "globals": "^14.0.0",
- "ignore": "^5.2.0",
- "import-fresh": "^3.2.1",
- "js-yaml": "^4.1.1",
- "minimatch": "^3.1.3",
- "strip-json-comments": "^3.1.1"
- },
- "engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
- },
- "funding": {
- "url": "https://opencollective.com/eslint"
- }
- },
- "node_modules/@eslint/eslintrc/node_modules/ajv": {
- "version": "6.14.0",
- "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
- "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "fast-deep-equal": "^3.1.1",
- "fast-json-stable-stringify": "^2.0.0",
- "json-schema-traverse": "^0.4.1",
- "uri-js": "^4.2.2"
- },
- "funding": {
- "type": "github",
- "url": "https://github.com/sponsors/epoberezkin"
- }
- },
- "node_modules/@eslint/eslintrc/node_modules/balanced-match": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
- "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/@eslint/eslintrc/node_modules/brace-expansion": {
- "version": "1.1.12",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
- "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "balanced-match": "^1.0.0",
- "concat-map": "0.0.1"
- }
- },
- "node_modules/@eslint/eslintrc/node_modules/ignore": {
- "version": "5.3.2",
- "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
- "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">= 4"
- }
- },
- "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": {
- "version": "0.4.1",
- "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
- "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/@eslint/eslintrc/node_modules/minimatch": {
- "version": "3.1.5",
- "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
- "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
- "dev": true,
- "license": "ISC",
- "dependencies": {
- "brace-expansion": "^1.1.7"
- },
- "engines": {
- "node": "*"
- }
- },
- "node_modules/@eslint/js": {
- "version": "9.39.3",
- "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.3.tgz",
- "integrity": "sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
- },
- "funding": {
- "url": "https://eslint.org/donate"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
}
},
"node_modules/@eslint/object-schema": {
- "version": "2.1.7",
- "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz",
- "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==",
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.2.tgz",
+ "integrity": "sha512-HOy56KJt48Bx8KmJ+XGQNSUMT/6dZee/M54XyUyuvTvPXJmsERRvBchsUVx1UMe1WwIH49XLAczNC7V2INsuUw==",
"dev": true,
"license": "Apache-2.0",
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
}
},
"node_modules/@eslint/plugin-kit": {
- "version": "0.4.1",
- "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz",
- "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==",
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz",
+ "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
- "@eslint/core": "^0.17.0",
+ "@eslint/core": "^1.1.0",
"levn": "^0.4.1"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
}
},
"node_modules/@hono/node-server": {
- "version": "1.19.9",
- "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz",
- "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==",
+ "version": "1.19.10",
+ "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.10.tgz",
+ "integrity": "sha512-hZ7nOssGqRgyV3FVVQdfi+U4q02uB23bpnYpdvNXkYTRRyWx84b7yf1ans+dnJ/7h41sGL3CeQTfO+ZGxuO+Iw==",
"license": "MIT",
"engines": {
"node": ">=18.14.1"
@@ -1532,6 +1432,13 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/@types/esrecurse": {
+ "version": "4.3.1",
+ "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",
+ "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/estree": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
@@ -1605,12 +1512,12 @@
"license": "MIT"
},
"node_modules/@types/node": {
- "version": "22.19.13",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.13.tgz",
- "integrity": "sha512-akNQMv0wW5uyRpD2v2IEyRSZiR+BeGuoB6L310EgGObO44HSMNT8z1xzio28V8qOrgYaopIDNA18YgdXd+qTiw==",
+ "version": "25.3.3",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-25.3.3.tgz",
+ "integrity": "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==",
"license": "MIT",
"dependencies": {
- "undici-types": "~6.21.0"
+ "undici-types": "~7.18.0"
}
},
"node_modules/@types/node-cron": {
@@ -2364,6 +2271,15 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
+ "node_modules/adm-zip": {
+ "version": "0.5.16",
+ "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
+ "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12.0"
+ }
+ },
"node_modules/ajv": {
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
@@ -2452,28 +2368,22 @@
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
}
},
- "node_modules/ansi-styles": {
- "version": "4.3.0",
- "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
- "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "color-convert": "^2.0.1"
- },
- "engines": {
- "node": ">=8"
- },
- "funding": {
- "url": "https://github.com/chalk/ansi-styles?sponsor=1"
- }
- },
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"license": "Python-2.0"
},
+ "node_modules/array-hyper-unique": {
+ "version": "2.1.6",
+ "resolved": "https://registry.npmjs.org/array-hyper-unique/-/array-hyper-unique-2.1.6.tgz",
+ "integrity": "sha512-BdlHRqjKSYs88WFaVNVEc6Kv8ln/FdzCKPbcDPuWs4/EXkQFhnjc8TyR7hnPxRjcjo5LKOhUMGUWpAqRgeJvpA==",
+ "license": "ISC",
+ "dependencies": {
+ "deep-eql": "= 4.0.0",
+ "lodash": "^4.17.21"
+ }
+ },
"node_modules/assertion-error": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
@@ -2637,14 +2547,17 @@
"license": "MIT"
},
"node_modules/better-sqlite3": {
- "version": "11.10.0",
- "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",
- "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==",
+ "version": "12.6.2",
+ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.6.2.tgz",
+ "integrity": "sha512-8VYKM3MjCa9WcaSAI3hzwhmyHVlH8tiGFwf0RlTsZPWJ1I5MkzjiudCo4KC4DxOaL/53A5B1sI/IbldNFDbsKA==",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
"bindings": "^1.5.0",
"prebuild-install": "^7.1.1"
+ },
+ "engines": {
+ "node": "20.x || 22.x || 23.x || 24.x || 25.x"
}
},
"node_modules/bindings": {
@@ -2802,16 +2715,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/callsites": {
- "version": "3.1.0",
- "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
- "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=6"
- }
- },
"node_modules/ccount": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz",
@@ -2833,23 +2736,6 @@
"node": ">=18"
}
},
- "node_modules/chalk": {
- "version": "4.1.2",
- "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
- "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "ansi-styles": "^4.1.0",
- "supports-color": "^7.1.0"
- },
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/chalk/chalk?sponsor=1"
- }
- },
"node_modules/character-entities-html4": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz",
@@ -2979,13 +2865,6 @@
"node": ">=20"
}
},
- "node_modules/concat-map": {
- "version": "0.0.1",
- "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
- "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
- "dev": true,
- "license": "MIT"
- },
"node_modules/content-disposition": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz",
@@ -3066,6 +2945,15 @@
"url": "https://opencollective.com/express"
}
},
+ "node_modules/crlf-normalize": {
+ "version": "1.0.20",
+ "resolved": "https://registry.npmjs.org/crlf-normalize/-/crlf-normalize-1.0.20.tgz",
+ "integrity": "sha512-h/rBerTd3YHQGfv7tNT25mfhWvRq2BBLCZZ80GFarFxf6HQGbpW6iqDL3N+HBLpjLfAdcBXfWAzVlLfHkRUQBQ==",
+ "license": "ISC",
+ "dependencies": {
+ "ts-type": ">=2"
+ }
+ },
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -3153,6 +3041,18 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/deep-eql": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.0.0.tgz",
+ "integrity": "sha512-GxJC5MOg2KyQlv6WiUF/VAnMj4MWnYiXo4oLgeptOELVoknyErb4Z8+5F/IM/K4g9/80YzzatxmWcyRwUseH0A==",
+ "license": "MIT",
+ "dependencies": {
+ "type-detect": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
@@ -3360,6 +3260,26 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/epub2": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/epub2/-/epub2-3.0.2.tgz",
+ "integrity": "sha512-rhvpt27CV5MZfRetfNtdNwi3XcNg1Am0TwfveJkK8YWeHItHepQ8Js9J06v8XRIjuTrCW/NSGYMTy55Of7BfNQ==",
+ "license": "ISC",
+ "dependencies": {
+ "adm-zip": "^0.5.10",
+ "array-hyper-unique": "^2.1.4",
+ "bluebird": "^3.7.2",
+ "crlf-normalize": "^1.0.19",
+ "tslib": "^2.6.2",
+ "xml2js": "^0.6.2"
+ }
+ },
+ "node_modules/epub2/node_modules/bluebird": {
+ "version": "3.7.2",
+ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+ "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
+ "license": "MIT"
+ },
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -3417,33 +3337,30 @@
}
},
"node_modules/eslint": {
- "version": "9.39.3",
- "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.3.tgz",
- "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==",
+ "version": "10.0.2",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.0.2.tgz",
+ "integrity": "sha512-uYixubwmqJZH+KLVYIVKY1JQt7tysXhtj21WSvjcSmU5SVNzMus1bgLe+pAt816yQ8opKfheVVoPLqvVMGejYw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/eslint-utils": "^4.8.0",
- "@eslint-community/regexpp": "^4.12.1",
- "@eslint/config-array": "^0.21.1",
- "@eslint/config-helpers": "^0.4.2",
- "@eslint/core": "^0.17.0",
- "@eslint/eslintrc": "^3.3.1",
- "@eslint/js": "9.39.3",
- "@eslint/plugin-kit": "^0.4.1",
+ "@eslint-community/regexpp": "^4.12.2",
+ "@eslint/config-array": "^0.23.2",
+ "@eslint/config-helpers": "^0.5.2",
+ "@eslint/core": "^1.1.0",
+ "@eslint/plugin-kit": "^0.6.0",
"@humanfs/node": "^0.16.6",
"@humanwhocodes/module-importer": "^1.0.1",
"@humanwhocodes/retry": "^0.4.2",
"@types/estree": "^1.0.6",
- "ajv": "^6.12.4",
- "chalk": "^4.0.0",
+ "ajv": "^6.14.0",
"cross-spawn": "^7.0.6",
"debug": "^4.3.2",
"escape-string-regexp": "^4.0.0",
- "eslint-scope": "^8.4.0",
- "eslint-visitor-keys": "^4.2.1",
- "espree": "^10.4.0",
- "esquery": "^1.5.0",
+ "eslint-scope": "^9.1.1",
+ "eslint-visitor-keys": "^5.0.1",
+ "espree": "^11.1.1",
+ "esquery": "^1.7.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
"file-entry-cache": "^8.0.0",
@@ -3453,8 +3370,7 @@
"imurmurhash": "^0.1.4",
"is-glob": "^4.0.0",
"json-stable-stringify-without-jsonify": "^1.0.1",
- "lodash.merge": "^4.6.2",
- "minimatch": "^3.1.2",
+ "minimatch": "^10.2.1",
"natural-compare": "^1.4.0",
"optionator": "^0.9.3"
},
@@ -3462,7 +3378,7 @@
"eslint": "bin/eslint.js"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
},
"funding": {
"url": "https://eslint.org/donate"
@@ -3477,30 +3393,35 @@
}
},
"node_modules/eslint-config-prettier": {
- "version": "9.1.2",
- "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.2.tgz",
- "integrity": "sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ==",
+ "version": "10.1.8",
+ "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz",
+ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==",
"dev": true,
"license": "MIT",
"bin": {
"eslint-config-prettier": "bin/cli.js"
},
+ "funding": {
+ "url": "https://opencollective.com/eslint-config-prettier"
+ },
"peerDependencies": {
"eslint": ">=7.0.0"
}
},
"node_modules/eslint-scope": {
- "version": "8.4.0",
- "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz",
- "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==",
+ "version": "9.1.1",
+ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.1.tgz",
+ "integrity": "sha512-GaUN0sWim5qc8KVErfPBWmc31LEsOkrUJbvJZV+xuL3u2phMUK4HIvXlWAakfC8W4nzlK+chPEAkYOYb5ZScIw==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
+ "@types/esrecurse": "^4.3.1",
+ "@types/estree": "^1.0.8",
"esrecurse": "^4.3.0",
"estraverse": "^5.2.0"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
},
"funding": {
"url": "https://opencollective.com/eslint"
@@ -3536,32 +3457,14 @@
"url": "https://github.com/sponsors/epoberezkin"
}
},
- "node_modules/eslint/node_modules/balanced-match": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
- "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/eslint/node_modules/brace-expansion": {
- "version": "1.1.12",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
- "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "balanced-match": "^1.0.0",
- "concat-map": "0.0.1"
- }
- },
"node_modules/eslint/node_modules/eslint-visitor-keys": {
- "version": "4.2.1",
- "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz",
- "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz",
+ "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
},
"funding": {
"url": "https://opencollective.com/eslint"
@@ -3584,45 +3487,32 @@
"dev": true,
"license": "MIT"
},
- "node_modules/eslint/node_modules/minimatch": {
- "version": "3.1.5",
- "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
- "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
- "dev": true,
- "license": "ISC",
- "dependencies": {
- "brace-expansion": "^1.1.7"
- },
- "engines": {
- "node": "*"
- }
- },
"node_modules/espree": {
- "version": "10.4.0",
- "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz",
- "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==",
+ "version": "11.1.1",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-11.1.1.tgz",
+ "integrity": "sha512-AVHPqQoZYc+RUM4/3Ly5udlZY/U4LS8pIG05jEjWM2lQMU/oaZ7qshzAl2YP1tfNmXfftH3ohurfwNAug+MnsQ==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
- "acorn": "^8.15.0",
+ "acorn": "^8.16.0",
"acorn-jsx": "^5.3.2",
- "eslint-visitor-keys": "^4.2.1"
+ "eslint-visitor-keys": "^5.0.1"
},
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
},
"funding": {
"url": "https://opencollective.com/eslint"
}
},
"node_modules/espree/node_modules/eslint-visitor-keys": {
- "version": "4.2.1",
- "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz",
- "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz",
+ "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
- "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+ "node": "^20.19.0 || ^22.13.0 || >=24"
},
"funding": {
"url": "https://opencollective.com/eslint"
@@ -4076,19 +3966,6 @@
"node": ">=10.13.0"
}
},
- "node_modules/globals": {
- "version": "14.0.0",
- "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz",
- "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=18"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -4189,9 +4066,9 @@
}
},
"node_modules/hono": {
- "version": "4.12.3",
- "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.3.tgz",
- "integrity": "sha512-SFsVSjp8sj5UumXOOFlkZOG6XS9SJDKw0TbwFeV+AJ8xlST8kxK5Z/5EYa111UY8732lK2S/xB653ceuaoGwpg==",
+ "version": "4.12.5",
+ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.5.tgz",
+ "integrity": "sha512-3qq+FUBtlTHhtYxbxheZgY8NIFnkkC/MR8u5TTsr7YZ3wixryQ3cCwn3iZbg8p8B88iDBBAYSfZDS75t8MN7Vg==",
"license": "MIT",
"engines": {
"node": ">=16.9.0"
@@ -4311,23 +4188,6 @@
"license": "MIT",
"optional": true
},
- "node_modules/import-fresh": {
- "version": "3.3.1",
- "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
- "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "parent-module": "^1.0.0",
- "resolve-from": "^4.0.0"
- },
- "engines": {
- "node": ">=6"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/imurmurhash": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
@@ -4529,6 +4389,19 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/json-schema-to-ts": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",
+ "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/runtime": "^7.18.3",
+ "ts-algebra": "^2.0.0"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
"node_modules/json-schema-traverse": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
@@ -4629,9 +4502,9 @@
}
},
"node_modules/lint-staged": {
- "version": "16.3.1",
- "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.3.1.tgz",
- "integrity": "sha512-bqvvquXzFBAlSbluugR4KXAe4XnO/QZcKVszpkBtqLWa2KEiVy8n6Xp38OeUbv/gOJOX4Vo9u5pFt/ADvbm42Q==",
+ "version": "16.3.2",
+ "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.3.2.tgz",
+ "integrity": "sha512-xKqhC2AeXLwiAHXguxBjuChoTTWFC6Pees0SHPwOpwlvI3BH7ZADFPddAdN3pgo3aiKgPUx/bxE78JfUnxQnlg==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -4686,11 +4559,10 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
- "node_modules/lodash.merge": {
- "version": "4.6.2",
- "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
- "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
- "dev": true,
+ "node_modules/lodash": {
+ "version": "4.17.23",
+ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
+ "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
"license": "MIT"
},
"node_modules/log-update": {
@@ -5423,19 +5295,6 @@
"license": "(MIT AND Zlib)",
"optional": true
},
- "node_modules/parent-module": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
- "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "callsites": "^3.0.0"
- },
- "engines": {
- "node": ">=6"
- }
- },
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -5589,6 +5448,21 @@
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
"license": "MIT"
},
+ "node_modules/pizzip": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/pizzip/-/pizzip-3.2.0.tgz",
+ "integrity": "sha512-X4NPNICxCfIK8VYhF6wbksn81vTiziyLbvKuORVAmolvnUzl1A1xmz9DAWKxPRq9lZg84pJOOAMq3OE61bD8IQ==",
+ "license": "(MIT OR GPL-3.0)",
+ "dependencies": {
+ "pako": "^2.1.0"
+ }
+ },
+ "node_modules/pizzip/node_modules/pako": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz",
+ "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==",
+ "license": "(MIT AND Zlib)"
+ },
"node_modules/pkce-challenge": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz",
@@ -5918,16 +5792,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/resolve-from": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
- "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=4"
- }
- },
"node_modules/restore-cursor": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz",
@@ -6048,6 +5912,15 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
+ "node_modules/sax": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz",
+ "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==",
+ "license": "BlueOak-1.0.0",
+ "engines": {
+ "node": ">=11.0.0"
+ }
+ },
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
@@ -6611,19 +6484,6 @@
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
}
},
- "node_modules/strip-json-comments": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
- "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=8"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/superjson": {
"version": "2.2.6",
"resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.6.tgz",
@@ -6823,6 +6683,12 @@
"url": "https://github.com/sponsors/wooorm"
}
},
+ "node_modules/ts-algebra": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
+ "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
+ "license": "MIT"
+ },
"node_modules/ts-api-utils": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz",
@@ -6836,6 +6702,33 @@
"typescript": ">=4.8.4"
}
},
+ "node_modules/ts-toolbelt": {
+ "version": "9.6.0",
+ "resolved": "https://registry.npmjs.org/ts-toolbelt/-/ts-toolbelt-9.6.0.tgz",
+ "integrity": "sha512-nsZd8ZeNUzukXPlJmTBwUAuABDe/9qtVDelJeT/qW0ow3ZS3BsQJtNkan1802aM9Uf68/Y8ljw86Hu0h5IUW3w==",
+ "license": "Apache-2.0",
+ "peer": true
+ },
+ "node_modules/ts-type": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/ts-type/-/ts-type-3.0.1.tgz",
+ "integrity": "sha512-cleRydCkBGBFQ4KAvLH0ARIkciduS745prkGVVxPGvcRGhMMoSJUB7gNR1ByKhFTEYrYRg2CsMRGYnqp+6op+g==",
+ "license": "ISC",
+ "dependencies": {
+ "@types/node": "*",
+ "tslib": ">=2",
+ "typedarray-dts": "^1.0.0"
+ },
+ "peerDependencies": {
+ "ts-toolbelt": "^9.6.0"
+ }
+ },
+ "node_modules/tslib": {
+ "version": "2.8.1",
+ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+ "license": "0BSD"
+ },
"node_modules/tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
@@ -6861,6 +6754,15 @@
"node": ">= 0.8.0"
}
},
+ "node_modules/type-detect": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz",
+ "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
"node_modules/type-is": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz",
@@ -6875,6 +6777,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/typedarray-dts": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/typedarray-dts/-/typedarray-dts-1.0.0.tgz",
+ "integrity": "sha512-Ka0DBegjuV9IPYFT1h0Qqk5U4pccebNIJCGl8C5uU7xtOs+jpJvKGAY4fHGK25hTmXZOEUl9Cnsg5cS6K/b5DA==",
+ "license": "MIT"
+ },
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -6896,10 +6804,19 @@
"license": "MIT",
"optional": true
},
+ "node_modules/undici": {
+ "version": "7.22.0",
+ "resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz",
+ "integrity": "sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20.18.1"
+ }
+ },
"node_modules/undici-types": {
- "version": "6.21.0",
- "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
- "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+ "version": "7.18.2",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
+ "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
"license": "MIT"
},
"node_modules/unist-util-is": {
@@ -8439,6 +8356,28 @@
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
},
+ "node_modules/xml2js": {
+ "version": "0.6.2",
+ "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz",
+ "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==",
+ "license": "MIT",
+ "dependencies": {
+ "sax": ">=0.6.0",
+ "xmlbuilder": "~11.0.0"
+ },
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/xml2js/node_modules/xmlbuilder": {
+ "version": "11.0.1",
+ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+ "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4.0"
+ }
+ },
"node_modules/xmlbuilder": {
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
diff --git a/package.json b/package.json
index 0af45d9..ba61b23 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "libscope",
- "version": "1.2.3",
+ "version": "1.3.0",
"description": "AI-powered knowledge base with MCP integration — query library docs, internal wikis, and topics with semantic search",
"homepage": "https://libscope.com",
"type": "module",
@@ -51,17 +51,21 @@
"node": ">=20"
},
"dependencies": {
+ "@anthropic-ai/sdk": "^0.78.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@xenova/transformers": "^2.17.2",
- "better-sqlite3": "^11.0.0",
+ "better-sqlite3": "^12.6.2",
"commander": "^14.0.3",
"csv-parse": "^6.1.0",
+ "epub2": "^3.0.2",
"js-yaml": "^4.1.1",
"node-cron": "^4.2.1",
"node-html-markdown": "^2.0.0",
"openai": "^6.25.0",
"pino": "^10.3.1",
+ "pizzip": "^3.2.0",
"sqlite-vec": "^0.1.0",
+ "undici": "^7.22.0",
"zod": "^4.3.6"
},
"optionalDependencies": {
@@ -71,14 +75,14 @@
"devDependencies": {
"@types/better-sqlite3": "^7.6.0",
"@types/js-yaml": "^4.0.9",
- "@types/node": "^22.0.0",
+ "@types/node": "^25.3.3",
"@types/node-cron": "^3.0.11",
"@types/pdf-parse": "^1.1.5",
"@typescript-eslint/eslint-plugin": "^8.0.0",
"@typescript-eslint/parser": "^8.0.0",
"@vitest/coverage-v8": "^4.0.18",
- "eslint": "^9.0.0",
- "eslint-config-prettier": "^9.0.0",
+ "eslint": "^10.0.2",
+ "eslint-config-prettier": "^10.1.8",
"husky": "^9.0.0",
"lint-staged": "^16.3.1",
"prettier": "^3.0.0",
diff --git a/src/LibScope.ts b/src/LibScope.ts
new file mode 100644
index 0000000..5d88dac
--- /dev/null
+++ b/src/LibScope.ts
@@ -0,0 +1,146 @@
+import type Database from "better-sqlite3";
+import type { LibScopeConfig } from "./config.js";
+import type { EmbeddingProvider } from "./providers/embedding.js";
+import { loadConfig } from "./config.js";
+import { createDatabase } from "./db/connection.js";
+import { runMigrations, createVectorTable } from "./db/schema.js";
+import { createEmbeddingProvider } from "./providers/index.js";
+import { createLlmProvider } from "./core/rag.js";
+import { getWorkspacePath, DEFAULT_WORKSPACE } from "./core/workspace.js";
+import {
+ indexDocument,
+ indexFile,
+ type IndexDocumentInput,
+ type IndexFileOptions,
+ type IndexedDocument,
+} from "./core/indexing.js";
+import { searchDocuments, type SearchOptions, type SearchResponse } from "./core/search.js";
+import {
+ askQuestion,
+ askQuestionStream,
+ type RagOptions,
+ type RagResult,
+ type RagStreamEvent,
+} from "./core/rag.js";
+import { getStats, type OverviewStats } from "./core/analytics.js";
+import { listDocuments, getDocument, deleteDocument, type Document } from "./core/documents.js";
+import {
+ searchBatch,
+ type BatchSearchRequest,
+ type BatchSearchResponse,
+} from "./core/batch-search.js";
+
+export interface LibScopeOptions {
+ /** Workspace name (default: "default"). */
+ workspace?: string;
+ /** Override config values. */
+ config?: Partial;
+}
+
+export class LibScope {
+ private readonly db: Database.Database;
+ private readonly embeddingProvider: EmbeddingProvider;
+ private readonly config: LibScopeConfig;
+
+ private constructor(
+ db: Database.Database,
+ embeddingProvider: EmbeddingProvider,
+ config: LibScopeConfig,
+ ) {
+ this.db = db;
+ this.embeddingProvider = embeddingProvider;
+ this.config = config;
+ }
+
+ /**
+ * Create a new LibScope instance. Initializes DB, runs migrations, and sets up providers.
+ */
+ static create(options?: LibScopeOptions): LibScope {
+ const baseConfig = loadConfig();
+ const config: LibScopeConfig = {
+ embedding: { ...baseConfig.embedding, ...options?.config?.embedding },
+ llm: { ...baseConfig.llm, ...options?.config?.llm },
+ database: { ...baseConfig.database, ...options?.config?.database },
+ indexing: { ...baseConfig.indexing, ...options?.config?.indexing },
+ logging: { ...baseConfig.logging, ...options?.config?.logging },
+ };
+
+ const workspace = options?.workspace ?? DEFAULT_WORKSPACE;
+ const dbPath = config.database.path ?? getWorkspacePath(workspace);
+
+ const db = createDatabase(dbPath);
+ runMigrations(db);
+
+ const embeddingProvider = createEmbeddingProvider(config);
+ createVectorTable(db, embeddingProvider.dimensions);
+
+ return new LibScope(db, embeddingProvider, config);
+ }
+
+ /** Index a document from content. */
+ async index(input: IndexDocumentInput): Promise {
+ return indexDocument(this.db, this.embeddingProvider, input);
+ }
+
+ /** Index a file from disk. */
+ async indexFile(filePath: string, options?: IndexFileOptions): Promise {
+ return indexFile(this.db, this.embeddingProvider, filePath, options);
+ }
+
+ /** Search documents. */
+ async search(query: string, options?: Omit): Promise {
+ return searchDocuments(this.db, this.embeddingProvider, { ...options, query });
+ }
+
+ /** Run multiple searches concurrently. */
+ async searchBatch(requests: BatchSearchRequest[]): Promise {
+ return searchBatch(this.db, this.embeddingProvider, requests);
+ }
+
+ /** Ask a question using RAG. */
+ async ask(question: string, options?: Omit): Promise {
+ const llm = createLlmProvider(this.config);
+ return askQuestion(this.db, this.embeddingProvider, llm, { ...options, question });
+ }
+
+ /** Ask a question with streaming response. */
+ async *askStream(
+ question: string,
+ options?: Omit,
+ ): AsyncGenerator {
+ const llm = createLlmProvider(this.config);
+ yield* askQuestionStream(this.db, this.embeddingProvider, llm, { ...options, question });
+ }
+
+ /** Get overview stats. */
+ stats(): OverviewStats {
+ return getStats(this.db, this.config.database.path);
+ }
+
+ /** List documents. */
+ list(options?: {
+ library?: string;
+ topicId?: string;
+ sourceType?: string;
+ dateFrom?: string;
+ dateTo?: string;
+ limit?: number;
+ }): Document[] {
+ return listDocuments(this.db, options);
+ }
+
+ /** Get a document by ID. */
+ get(id: string): Document {
+ return getDocument(this.db, id);
+ }
+
+ /** Delete a document by ID. */
+ delete(id: string): void {
+ deleteDocument(this.db, id);
+ }
+
+ /** Close the database connection. */
+ close(): void {
+ this.db.close();
+ }
+}
diff --git a/src/api/openapi.ts b/src/api/openapi.ts
index 0264cbe..e61b4f2 100644
--- a/src/api/openapi.ts
+++ b/src/api/openapi.ts
@@ -127,22 +127,44 @@ export const OPENAPI_SPEC = {
},
"/api/v1/documents/url": {
post: {
- summary: "Index document from URL",
+ summary: "Index document from URL (with optional spidering)",
operationId: "indexFromUrl",
requestBody: {
required: true,
content: {
"application/json": {
schema: { $ref: "#/components/schemas/IndexFromUrlRequest" },
- example: { url: "https://example.com/docs", topic: "guides" },
+ examples: {
+ single: {
+ summary: "Single URL",
+ value: { url: "https://example.com/page", topic: "guides" },
+ },
+ spider: {
+ summary: "Spider mode",
+ value: {
+ url: "https://docs.example.com",
+ spider: true,
+ maxPages: 50,
+ maxDepth: 2,
+ },
+ },
+ },
},
},
},
responses: {
"201": {
- description: "Document indexed from URL",
+ description:
+ "Document(s) indexed. Returns DocumentResponse for single-URL mode, SpiderResponse for spider mode.",
content: {
- "application/json": { schema: { $ref: "#/components/schemas/DocumentResponse" } },
+ "application/json": {
+ schema: {
+ oneOf: [
+ { $ref: "#/components/schemas/DocumentResponse" },
+ { $ref: "#/components/schemas/SpiderResponse" },
+ ],
+ },
+ },
},
},
},
@@ -343,6 +365,70 @@ export const OPENAPI_SPEC = {
properties: {
url: { type: "string", format: "uri" },
topic: { type: "string" },
+ spider: {
+ type: "boolean",
+ description: "When true, crawl linked pages starting from the URL (BFS spider mode).",
+ },
+ maxPages: {
+ type: "integer",
+ minimum: 1,
+ description:
+ "Maximum total pages to fetch in spider mode (default: 25, hard cap: 200).",
+ },
+ maxDepth: {
+ type: "integer",
+ minimum: 0,
+ description:
+ "Maximum hop depth from the seed URL in spider mode (default: 2, hard cap: 5).",
+ },
+ sameDomain: {
+ type: "boolean",
+ description: "Only follow links sharing the seed hostname (default: true).",
+ },
+ pathPrefix: {
+ type: "string",
+ description: "Only follow links whose path starts with this prefix (e.g. '/docs/').",
+ },
+ excludePatterns: {
+ type: "array",
+ items: { type: "string" },
+ description: "Glob patterns for URLs to skip during spidering (e.g. ['*/changelog*']).",
+ },
+ },
+ },
+ SpiderResponse: {
+ type: "object",
+ properties: {
+ documents: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ id: { type: "string" },
+ title: { type: "string" },
+ url: { type: "string" },
+ },
+ },
+ },
+ pagesFetched: { type: "integer", description: "Pages successfully fetched and indexed." },
+ pagesCrawled: { type: "integer", description: "Total pages attempted." },
+ pagesSkipped: { type: "integer", description: "Pages skipped by filters or robots.txt." },
+ errors: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ url: { type: "string" },
+ error: { type: "string" },
+ },
+ },
+ },
+ abortReason: {
+ type: "string",
+ nullable: true,
+ enum: ["maxPages", "timeout", null],
+ description: "Set if the crawl was aborted early.",
+ },
},
},
AskRequest: {
diff --git a/src/api/routes.ts b/src/api/routes.ts
index 497d3e5..79fdaa5 100644
--- a/src/api/routes.ts
+++ b/src/api/routes.ts
@@ -32,10 +32,11 @@ import {
bulkDelete,
bulkRetag,
bulkMove,
+ searchBatch,
} from "../core/index.js";
-import type { LinkType, BulkSelector } from "../core/index.js";
+import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js";
import { loadConfig } from "../config.js";
-import { DocumentNotFoundError, LibScopeError } from "../errors.js";
+import { DocumentNotFoundError, FetchError, LibScopeError } from "../errors.js";
import { getLogger } from "../logger.js";
import { parseJsonBody, sendJson, sendError } from "./middleware.js";
import { OPENAPI_SPEC } from "./openapi.js";
@@ -52,6 +53,8 @@ import {
} from "../core/webhooks.js";
import type { WebhookEvent } from "../core/webhooks.js";
import { loadScheduleEntries } from "../core/scheduler.js";
+import { spiderUrl } from "../core/spider.js";
+import type { SpiderOptions, SpiderStats } from "../core/spider.js";
function parseUrl(req: IncomingMessage): URL {
return new URL(req.url ?? "/", `http://${req.headers["host"] ?? "localhost"}`);
@@ -272,6 +275,24 @@ export async function handleRequest(
return;
}
+ // Batch search
+ if (pathname === "/api/v1/batch-search" && method === "POST") {
+ const body = await parseJsonBody(req);
+ if (!body || typeof body !== "object") {
+ sendError(res, 400, "VALIDATION_ERROR", "Request body must be a JSON object");
+ return;
+ }
+ const b = body as Record;
+ if (!Array.isArray(b["requests"])) {
+ sendError(res, 400, "VALIDATION_ERROR", "Field 'requests' must be an array");
+ return;
+ }
+ const result = await searchBatch(db, provider, b["requests"] as BatchSearchRequest[]);
+ const took = Math.round(performance.now() - start);
+ sendJson(res, 200, result, took);
+ return;
+ }
+
// List documents
if (pathname === "/api/v1/documents" && method === "GET") {
const topicId = url.searchParams.get("topic") ?? undefined;
@@ -332,7 +353,7 @@ export async function handleRequest(
return;
}
- // Index from URL
+ // Index from URL (with optional spidering)
if (pathname === "/api/v1/documents/url" && method === "POST") {
const body = await parseJsonBody(req);
if (!body || typeof body !== "object") {
@@ -344,16 +365,100 @@ export async function handleRequest(
sendError(res, 400, "VALIDATION_ERROR", "Field 'url' is required");
return;
}
- const fetched = await fetchAndConvert(b["url"], {
- allowPrivateUrls: loadConfig().indexing.allowPrivateUrls,
- allowSelfSignedCerts: loadConfig().indexing.allowSelfSignedCerts,
- });
+ const url = b["url"];
const topicId = typeof b["topic"] === "string" ? b["topic"] : undefined;
+ const config = loadConfig();
+ const fetchOptions = {
+ allowPrivateUrls: config.indexing.allowPrivateUrls,
+ allowSelfSignedCerts: config.indexing.allowSelfSignedCerts,
+ };
+
+ // Spider mode — crawl linked pages
+ if (b["spider"] === true) {
+ // Validate optional numeric fields — must be finite positive integers
+ if (b["maxPages"] !== undefined) {
+ const v = b["maxPages"];
+ if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 1) {
+ sendError(res, 400, "VALIDATION_ERROR", "maxPages must be a positive integer");
+ return;
+ }
+ }
+ if (b["maxDepth"] !== undefined) {
+ const v = b["maxDepth"];
+ if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 0) {
+ sendError(res, 400, "VALIDATION_ERROR", "maxDepth must be a non-negative integer");
+ return;
+ }
+ }
+
+ const spiderOptions: SpiderOptions = {
+ fetchOptions,
+ ...(typeof b["maxPages"] === "number" ? { maxPages: b["maxPages"] } : {}),
+ ...(typeof b["maxDepth"] === "number" ? { maxDepth: b["maxDepth"] } : {}),
+ ...(typeof b["sameDomain"] === "boolean" ? { sameDomain: b["sameDomain"] } : {}),
+ ...(typeof b["pathPrefix"] === "string" ? { pathPrefix: b["pathPrefix"] } : {}),
+ ...(Array.isArray(b["excludePatterns"])
+ ? {
+ excludePatterns: (b["excludePatterns"] as unknown[]).filter(
+ (p): p is string => typeof p === "string",
+ ),
+ }
+ : {}),
+ };
+
+ const indexedDocs: Array<{ id: string; title: string; url: string }> = [];
+ const errors: Array<{ url: string; error: string }> = [];
+ let stats: SpiderStats = { pagesFetched: 0, pagesCrawled: 0, pagesSkipped: 0, errors };
+
+ const gen = spiderUrl(url, spiderOptions);
+ let result = await gen.next();
+ while (!result.done) {
+ const page = result.value;
+ try {
+ const doc = await indexDocument(db, provider, {
+ content: page.content,
+ title: page.title,
+ sourceType: "manual",
+ url: page.url,
+ topicId,
+ });
+ indexedDocs.push({ id: doc.id, title: page.title, url: page.url });
+ } catch (indexErr) {
+ const msg = indexErr instanceof Error ? indexErr.message : String(indexErr);
+ errors.push({ url: page.url, error: msg });
+ }
+ result = await gen.next();
+ }
+ // result.value is SpiderStats when done (generator is exhausted)
+ if (result.done && result.value) {
+ stats = result.value;
+ stats.errors = errors;
+ }
+
+ const took = Math.round(performance.now() - start);
+ sendJson(
+ res,
+ 201,
+ {
+ documents: indexedDocs,
+ pagesFetched: indexedDocs.length,
+ pagesCrawled: stats.pagesCrawled,
+ pagesSkipped: stats.pagesSkipped,
+ errors,
+ abortReason: stats.abortReason ?? null,
+ },
+ took,
+ );
+ return;
+ }
+
+ // Single-URL mode (default)
+ const fetched = await fetchAndConvert(url, fetchOptions);
const doc = await indexDocument(db, provider, {
content: fetched.content,
title: fetched.title,
sourceType: "manual",
- url: b["url"],
+ url,
topicId,
});
const took = Math.round(performance.now() - start);
@@ -850,6 +955,10 @@ export async function handleRequest(
sendError(res, 404, "NOT_FOUND", err.message);
return;
}
+ if (err instanceof FetchError) {
+ sendError(res, 502, "FETCH_ERROR", err.message);
+ return;
+ }
if (err instanceof LibScopeError && err.code === "VALIDATION_ERROR") {
sendError(res, 400, "VALIDATION_ERROR", err.message);
return;
diff --git a/src/api/server.ts b/src/api/server.ts
index e6a9c7c..7b011ac 100644
--- a/src/api/server.ts
+++ b/src/api/server.ts
@@ -9,6 +9,7 @@ import { ConnectorScheduler, loadScheduleEntries } from "../core/scheduler.js";
export interface ApiServerOptions {
port?: number | undefined;
host?: string | undefined;
+ /** Allowed CORS origins. Defaults to ["http://localhost", "http://localhost:3000"]. */
corsOrigins?: string[] | undefined;
enableScheduler?: boolean | undefined;
}
@@ -25,7 +26,7 @@ export async function startApiServer(
const log = getLogger();
const port = options?.port ?? 3378;
const host = options?.host ?? "localhost";
- const corsOrigins = options?.corsOrigins ?? ["*"];
+ const corsOrigins = options?.corsOrigins ?? ["http://localhost", "http://localhost:3000"];
const server = createServer((req, res) => {
// Rate limiting
diff --git a/src/cli/index.ts b/src/cli/index.ts
index e775e2b..b92bc7f 100644
--- a/src/cli/index.ts
+++ b/src/cli/index.ts
@@ -32,6 +32,7 @@ import {
} from "../core/analytics.js";
import { startRepl } from "./repl.js";
import { confirmAction } from "./confirm.js";
+import { createReporter, isVerbose } from "./reporter.js";
import {
addTagsToDocument,
removeTagFromDocument,
@@ -55,6 +56,7 @@ import {
listInstalledPacks,
listAvailablePacks,
createPack,
+ createPackFromSource,
} from "../core/packs.js";
import { execSync } from "node:child_process";
@@ -119,7 +121,7 @@ program
.name("libscope")
.description("AI-powered knowledge base with MCP integration")
.version(_pkg.version)
- .option("--verbose", "Enable verbose logging")
+ .option("-v, --verbose", "Enable verbose logging")
.option("--log-level ", "Set log level (debug, info, warn, error, silent)")
.option("--workspace ", "Use a specific workspace");
@@ -1085,10 +1087,15 @@ interface ProgramOpts {
}
function setupLogging(opts: ProgramOpts): void {
- const level: LogLevel = opts.verbose
- ? "debug"
- : ((opts.logLevel as LogLevel | undefined) ?? loadConfig().logging.level);
- initLogger(level);
+ if (isVerbose(opts.verbose)) {
+ initLogger("debug");
+ } else if (opts.logLevel) {
+ initLogger(opts.logLevel as LogLevel);
+ } else {
+ // Default to silent in CLI mode — pretty reporter handles user-facing output.
+ // Set LIBSCOPE_VERBOSE=1 or pass --verbose to see structured JSON logs.
+ initLogger("silent");
+ }
}
/** Shared CLI initialization: loadConfig → setupLogging → getDatabase → runMigrations. */
@@ -1607,21 +1614,61 @@ const packCmd = program.command("pack").description("Manage knowledge packs");
packCmd
.command("install ")
- .description("Install a knowledge pack from registry or local .json file")
+ .description("Install a knowledge pack from registry or local .json/.json.gz file")
.option("--registry ", "Custom registry URL")
- .action(async (nameOrPath: string, opts: { registry?: string }) => {
- const { db, provider } = initializeAppWithEmbedding();
- const result = await installPack(db, provider, nameOrPath, {
- registryUrl: opts.registry,
- });
- if (result.alreadyInstalled) {
- console.log(`Pack "${result.packName}" is already installed.`);
- } else {
- console.log(
- `✓ Pack "${result.packName}" installed (${result.documentsInstalled} documents).`,
- );
- }
- });
+ .option("--batch-size ", "Number of documents to embed per batch (default: 10)")
+ .option("--resume-from ", "Skip the first N documents (resume a partial install)")
+ .option("--concurrency ", "Number of batches to embed in parallel (default: 4)")
+ .action(
+ async (
+ nameOrPath: string,
+ opts: { registry?: string; batchSize?: string; resumeFrom?: string; concurrency?: string },
+ ) => {
+ const { db, provider } = initializeAppWithEmbedding();
+ const globalOpts = program.opts();
+ const reporter = createReporter(globalOpts.verbose);
+
+ const batchSize = opts.batchSize ? parseIntOption(opts.batchSize, "--batch-size") : undefined;
+ const resumeFrom = opts.resumeFrom
+ ? parseIntOption(opts.resumeFrom, "--resume-from")
+ : undefined;
+ const concurrency = opts.concurrency
+ ? parseIntOption(opts.concurrency, "--concurrency")
+ : undefined;
+
+ if (concurrency !== undefined && concurrency < 1) {
+ reporter.log('Error: "--concurrency" must be an integer greater than or equal to 1.');
+ closeDatabase();
+ process.exit(1);
+ return;
+ }
+
+ try {
+ const result = await installPack(db, provider, nameOrPath, {
+ registryUrl: opts.registry,
+ batchSize,
+ resumeFrom,
+ concurrency,
+ onProgress: (current, total, docTitle) => {
+ reporter.progress(current, total, docTitle);
+ },
+ });
+
+ reporter.clearProgress();
+
+ if (result.alreadyInstalled) {
+ reporter.log(`Pack "${result.packName}" is already installed.`);
+ } else {
+ const errMsg = result.errors > 0 ? ` (${result.errors} errors)` : "";
+ reporter.success(
+ `Pack "${result.packName}" installed: ${result.documentsInstalled} documents${errMsg}.`,
+ );
+ }
+ } finally {
+ closeDatabase();
+ }
+ },
+ );
packCmd
.command("remove ")
@@ -1682,38 +1729,81 @@ packCmd
packCmd
.command("create")
- .description("Export current documents as a pack file")
+ .description("Create a pack from the database, a local folder, or a URL")
.requiredOption("--name ", "Pack name")
- .option("--topic ", "Filter documents by topic ID")
+ .option("--from ", "Source folder(s), file(s), or URL(s) to build pack from")
+ .option("--topic ", "Filter documents by topic ID (database mode only)")
.option("--version ", "Pack version (default: 1.0.0)")
.option("--description ", "Pack description")
.option("--author ", "Pack author")
.option("--output ", "Output file path")
+ .option(
+ "--extensions ",
+ "Comma-separated file extensions to include (e.g. .md,.html). Default: all supported",
+ )
+ .option("--exclude ", "Glob patterns to exclude (e.g. *.min.js, assets/**)")
+ .option("--no-recursive", "Do not recurse into subdirectories")
.action(
- (opts: {
+ async (opts: {
name: string;
+ from?: string[];
topic?: string;
version?: string;
description?: string;
author?: string;
output?: string;
+ extensions?: string;
+ exclude?: string[];
+ recursive: boolean;
}) => {
- const { db } = initializeApp();
- try {
- const outputPath = opts.output ?? `${opts.name}.json`;
- const pack = createPack(db, {
+ if (opts.from && opts.from.length > 0) {
+ // Source mode: build pack directly from files/URLs (no database needed)
+ // Default to .json.gz for source packs (they can be large)
+ const outputPath = opts.output ?? `${opts.name}.json.gz`;
+ const extensionList = opts.extensions
+ ? opts.extensions.split(",").map((e) => e.trim())
+ : undefined;
+
+ const pack = await createPackFromSource({
name: opts.name,
+ from: opts.from,
version: opts.version,
description: opts.description,
author: opts.author,
- topic: opts.topic,
outputPath,
+ extensions: extensionList,
+ exclude: opts.exclude,
+ recursive: opts.recursive,
+ onProgress: ({ file, index, total }) => {
+ const pct = Math.round(((index + 1) / total) * 100);
+ const short = file.length > 60 ? `...${file.slice(-57)}` : file;
+ process.stdout.write(`\r [${pct}%] ${index + 1}/${total} ${short}`.padEnd(80));
+ },
});
+ // Clear progress line
+ process.stdout.write("\r" + " ".repeat(80) + "\r");
console.log(
`✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`,
);
- } finally {
- closeDatabase();
+ } else {
+ // Database mode: export existing documents
+ const outputPath = opts.output ?? `${opts.name}.json`;
+ const { db } = initializeApp();
+ try {
+ const pack = createPack(db, {
+ name: opts.name,
+ version: opts.version,
+ description: opts.description,
+ author: opts.author,
+ topic: opts.topic,
+ outputPath,
+ });
+ console.log(
+ `✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`,
+ );
+ } finally {
+ closeDatabase();
+ }
}
},
);
@@ -1729,9 +1819,7 @@ connectCmd
.option("--notebook ", "Sync a specific notebook")
.action(async (opts: { token?: string; sync?: boolean; notebook?: string }) => {
const config = loadConfig();
- const logLevel =
- (program.opts().logLevel as LogLevel) ?? (program.opts().verbose ? "debug" : "info");
- initLogger(logLevel);
+ setupLogging(program.opts());
const workspace = program.opts().workspace as string | undefined;
if (workspace) {
@@ -1845,9 +1933,7 @@ disconnectCmd
return;
}
const config = loadConfig();
- const logLevel =
- (program.opts().logLevel as LogLevel) ?? (program.opts().verbose ? "debug" : "info");
- initLogger(logLevel);
+ setupLogging(program.opts());
const workspace2 = program.opts().workspace as string | undefined;
if (workspace2) {
diff --git a/src/cli/reporter.ts b/src/cli/reporter.ts
new file mode 100644
index 0000000..22533cb
--- /dev/null
+++ b/src/cli/reporter.ts
@@ -0,0 +1,87 @@
+/**
+ * CLI output reporter — pretty human-readable output for interactive terminals.
+ * In verbose/JSON mode, a SilentReporter is used so pino JSON logs handle output.
+ */
+
+const RESET = "\x1b[0m";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+const RED = "\x1b[31m";
+const CYAN = "\x1b[36m";
+const DIM = "\x1b[2m";
+
+export interface CliReporter {
+ log(msg: string): void;
+ success(msg: string): void;
+ warn(msg: string): void;
+ error(msg: string): void;
+ progress(current: number, total: number, label: string): void;
+ clearProgress(): void;
+}
+
+function buildBar(pct: number, width = 20): string {
+ const filled = Math.round((pct / 100) * width);
+ return "\u2588".repeat(filled) + "\u2591".repeat(width - filled);
+}
+
+/** Pretty human-readable reporter. Uses ANSI colors and \r progress lines. */
+class PrettyReporter implements CliReporter {
+ private hasProgress = false;
+
+ log(msg: string): void {
+ this.clearProgress();
+ process.stdout.write(`${msg}\n`);
+ }
+
+ success(msg: string): void {
+ this.clearProgress();
+ process.stdout.write(`${GREEN}\u2713${RESET} ${msg}\n`);
+ }
+
+ warn(msg: string): void {
+ this.clearProgress();
+ process.stderr.write(`${YELLOW}\u26a0${RESET} ${msg}\n`);
+ }
+
+ error(msg: string): void {
+ this.clearProgress();
+ process.stderr.write(`${RED}\u2717${RESET} ${msg}\n`);
+ }
+
+ progress(current: number, total: number, label: string): void {
+ const pct = total > 0 ? Math.round((current / total) * 100) : 0;
+ const bar = buildBar(pct);
+ const truncatedLabel = label.length > 40 ? `${label.slice(0, 37)}...` : label;
+ const line = `${CYAN}[${bar}]${RESET} ${pct}% (${current}/${total}) ${DIM}${truncatedLabel}${RESET}`;
+ process.stdout.write(`\r${line}`);
+ this.hasProgress = true;
+ }
+
+ clearProgress(): void {
+ if (this.hasProgress) {
+ const width = process.stdout.columns ?? 80;
+ process.stdout.write(`\r${" ".repeat(width - 1)}\r`);
+ this.hasProgress = false;
+ }
+ }
+}
+
+/** No-op reporter: used in verbose/JSON mode where pino logs handle output. */
+class SilentReporter implements CliReporter {
+ log(_msg: string): void {}
+ success(_msg: string): void {}
+ warn(_msg: string): void {}
+ error(_msg: string): void {}
+ progress(_current: number, _total: number, _label: string): void {}
+ clearProgress(): void {}
+}
+
+/** Returns true if verbose mode is active (flag or env var). */
+export function isVerbose(verbose?: boolean): boolean {
+ return verbose === true || process.env["LIBSCOPE_VERBOSE"] === "1";
+}
+
+/** Create a reporter appropriate for the current mode. */
+export function createReporter(verbose?: boolean): CliReporter {
+ return isVerbose(verbose) ? new SilentReporter() : new PrettyReporter();
+}
diff --git a/src/config.ts b/src/config.ts
index dab9d4c..2815b14 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -13,10 +13,11 @@ export interface LibScopeConfig {
openaiModel?: string;
};
llm?: {
- provider?: "openai" | "ollama";
+ provider?: "openai" | "ollama" | "anthropic" | "passthrough";
model?: string;
ollamaUrl?: string;
openaiApiKey?: string;
+ anthropicApiKey?: string;
};
database: {
path: string;
@@ -112,18 +113,45 @@ function getEnvOverrides(): Partial {
};
}
- if (llmProvider === "openai" || llmProvider === "ollama" || llmModel) {
+ if (
+ llmProvider === "openai" ||
+ llmProvider === "ollama" ||
+ llmProvider === "anthropic" ||
+ llmProvider === "passthrough" ||
+ llmModel
+ ) {
+ const anthropicKey = process.env["LIBSCOPE_ANTHROPIC_API_KEY"];
overrides.llm = {
- ...(llmProvider === "openai" || llmProvider === "ollama" ? { provider: llmProvider } : {}),
+ ...(llmProvider === "openai" ||
+ llmProvider === "ollama" ||
+ llmProvider === "anthropic" ||
+ llmProvider === "passthrough"
+ ? { provider: llmProvider }
+ : {}),
...(llmModel ? { model: llmModel } : {}),
+ ...(anthropicKey ? { anthropicApiKey: anthropicKey } : {}),
};
}
return overrides;
}
-/** Load config with precedence: env > project > user > defaults */
+let _configCache: LibScopeConfig | null = null;
+let _configCacheAt = 0;
+const CONFIG_CACHE_TTL_MS = 30_000;
+
+/** Invalidate the config cache (e.g. after saving new values). */
+export function invalidateConfigCache(): void {
+ _configCache = null;
+ _configCacheAt = 0;
+}
+
+/** Load config with precedence: env > project > user > defaults. Result is cached for 30 s. */
export function loadConfig(): LibScopeConfig {
+ const now = Date.now();
+ if (_configCache && now - _configCacheAt < CONFIG_CACHE_TTL_MS) {
+ return _configCache;
+ }
const userConfig = loadJsonFile(getUserConfigPath());
const projectConfig = loadJsonFile(getProjectConfigPath());
const envOverrides = getEnvOverrides();
@@ -160,6 +188,8 @@ export function loadConfig(): LibScopeConfig {
validateConfig(config);
+ _configCache = config;
+ _configCacheAt = now;
return config;
}
@@ -257,4 +287,5 @@ export function saveUserConfig(config: Partial): void {
},
};
writeFileSync(getUserConfigPath(), JSON.stringify(merged, null, 2), "utf-8");
+ invalidateConfigCache();
}
diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts
index 9068182..38cfb40 100644
--- a/src/connectors/confluence.ts
+++ b/src/connectors/confluence.ts
@@ -115,19 +115,10 @@ async function confluenceFetch(url: string, auth: string): Promise {
async function fetchAllPages(initialUrl: string, baseUrl: string, auth: string): Promise {
const all: T[] = [];
let url: string | undefined = initialUrl;
- const MAX_PAGES = 10_000;
while (url) {
const resp: PaginatedResponse = await confluenceFetch>(url, auth);
all.push(...resp.results);
- if (all.length >= MAX_PAGES) {
- const log = getLogger();
- log.warn(
- { count: all.length, max: MAX_PAGES },
- "Reached max page limit, stopping pagination",
- );
- break;
- }
const next: string | undefined = resp._links?.next;
url = next ? `${baseUrl}${next}` : undefined;
}
@@ -221,6 +212,38 @@ function extractTagContent(html: string, tagName: string): string {
return html.slice(contentStart, end);
}
+/**
+ * Remove self-closing `` tags whose attribute
+ * string matches `nameTest`. Uses indexOf so there is no regex backtracking.
+ */
+function removeSelfClosingMacros(html: string, nameTest: RegExp): string {
+ const OPEN = "", start + OPEN.length);
+ if (tagEnd === -1) {
+ result += html.slice(pos);
+ break;
+ }
+ const isSelfClosing = html[tagEnd - 1] === "/";
+ const attrs = html.slice(start, tagEnd + 1);
+ if (isSelfClosing && nameTest.test(attrs)) {
+ result += html.slice(pos, start); // drop the self-closing tag
+ } else {
+ result += html.slice(pos, tagEnd + 1); // keep it
+ }
+ pos = tagEnd + 1;
+ }
+ return result;
+}
+
export function convertConfluenceStorage(html: string): string {
let processed = html;
@@ -263,13 +286,12 @@ export function convertConfluenceStorage(html: string): string {
return `${title}
${body.trim()}`;
});
- // TOC → strip
+ // TOC → strip (paired tags handled by replaceStructuredMacros, self-closing by indexOf helper)
processed = replaceStructuredMacros(processed, (_inner, attrs) => {
if (!/ac:name="toc"/i.test(attrs)) return undefined;
return "";
});
- // Self-closing TOC
- processed = processed.replace(/]*ac:name="toc"[^>]*\/>/gi, "");
+ processed = removeSelfClosingMacros(processed, /ac:name="toc"/i);
// JIRA macro → [JIRA: KEY-123] as a span to avoid escaping
processed = replaceStructuredMacros(processed, (inner, attrs) => {
@@ -299,7 +321,7 @@ export function convertConfluenceStorage(html: string): string {
// ri:attachment → [attached: filename] as span
processed = processed.replace(
- /]*\/?>/gi,
+ /]{0,500}\/?>/gi,
(_match, filename: string) => `[attached: ${filename}]`,
);
diff --git a/src/connectors/http-utils.ts b/src/connectors/http-utils.ts
index b788367..aeb97bb 100644
--- a/src/connectors/http-utils.ts
+++ b/src/connectors/http-utils.ts
@@ -1,22 +1,13 @@
+import { Agent } from "undici";
import { getLogger } from "../logger.js";
import { FetchError } from "../errors.js";
import { loadConfig } from "../config.js";
-let tlsWarningLogged = false;
-
-/**
- * Log a one-time warning when `allowSelfSignedCerts` is enabled but the
- * user has not set `NODE_TLS_REJECT_UNAUTHORIZED=0` in their environment.
- */
-function warnIfTlsBypassMissing(): void {
- if (tlsWarningLogged) return;
- if (process.env["NODE_TLS_REJECT_UNAUTHORIZED"] === "0") return;
- tlsWarningLogged = true;
- const log = getLogger();
- log.warn(
- "allowSelfSignedCerts is enabled but NODE_TLS_REJECT_UNAUTHORIZED is not set. " +
- "Set NODE_TLS_REJECT_UNAUTHORIZED=0 in your environment to allow self-signed certificates.",
- );
+/** Lazy singleton undici Agent that skips TLS certificate verification. */
+let _insecureAgent: Agent | undefined;
+function getInsecureAgent(): Agent {
+ _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } });
+ return _insecureAgent;
}
export interface RetryConfig {
@@ -39,52 +30,45 @@ export async function fetchWithRetry(
const log = getLogger();
const config = loadConfig();
- if (config.indexing.allowSelfSignedCerts) {
- warnIfTlsBypassMissing();
- }
+ // Use a per-request undici Agent when self-signed certs are allowed.
+ // This is scoped to this fetch chain and does not affect concurrent requests.
+ const dispatcher = config.indexing.allowSelfSignedCerts ? getInsecureAgent() : undefined;
- try {
- for (let attempt = 0; attempt < maxRetries; attempt++) {
- const timeoutSignal = AbortSignal.timeout(30_000);
- const combinedSignal =
- options?.signal != null ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
+ const fetchOptions = {
+ ...(options ?? {}),
+ ...(dispatcher ? { dispatcher: dispatcher as unknown } : {}),
+ } as RequestInit;
+ const response = await fetch(url, fetchOptions);
- const response = await fetch(url, {
- ...options,
- signal: combinedSignal,
- });
-
- if (response.status === 429 || (response.status >= 500 && response.status < 600)) {
- if (attempt >= maxRetries - 1) {
- const body = await response.text().catch(() => "");
- throw new FetchError(`HTTP ${response.status} after ${maxRetries} attempts: ${body}`);
- }
+ if (response.status === 429 || (response.status >= 500 && response.status < 600)) {
+ if (attempt >= maxRetries) {
+ const body = await response.text().catch(() => "");
+ throw new FetchError(`HTTP ${response.status} after ${maxRetries + 1} attempts: ${body}`);
+ }
- let delayMs = baseDelay * 2 ** attempt;
- if (response.status === 429) {
- const retryAfter = response.headers.get("Retry-After");
- if (retryAfter) {
- const parsed = Number(retryAfter);
- if (!Number.isNaN(parsed)) {
- delayMs = parsed * 1000;
- }
+ let delayMs = baseDelay * 2 ** attempt;
+ if (response.status === 429) {
+ const retryAfter = response.headers.get("Retry-After");
+ if (retryAfter) {
+ const parsed = Number(retryAfter);
+ if (!Number.isNaN(parsed)) {
+ delayMs = parsed * 1000;
}
}
-
- log.warn(
- { status: response.status, attempt: attempt + 1, delayMs },
- "Retrying after transient error",
- );
- await new Promise((resolve) => setTimeout(resolve, delayMs));
- continue;
}
- return response;
+ log.warn(
+ { status: response.status, attempt: attempt + 1, delayMs },
+ "Retrying after transient error",
+ );
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
+ continue;
}
- // Unreachable, but satisfies TypeScript
- throw new FetchError("fetchWithRetry: unexpected code path");
- } finally {
- // no-op: TLS state is managed by the user's environment, not this function
+ return response;
}
+
+ // Unreachable, but satisfies TypeScript
+ throw new FetchError("fetchWithRetry: unexpected code path");
}
diff --git a/src/connectors/index.ts b/src/connectors/index.ts
index 4dcded4..09926f5 100644
--- a/src/connectors/index.ts
+++ b/src/connectors/index.ts
@@ -60,7 +60,11 @@ export function loadDbConnectorConfig(
| { config_json: string }
| undefined;
if (!row) return undefined;
- return JSON.parse(row.config_json) as ConnectorConfig;
+ try {
+ return JSON.parse(row.config_json) as ConnectorConfig;
+ } catch (err) {
+ throw new ConfigError(`Corrupted connector config for type "${type}"`, err);
+ }
}
/** Delete connector config from the database. */
@@ -102,7 +106,11 @@ export function loadNamedConnectorConfig(name: string): T {
);
}
const raw = readFileSync(filePath, "utf-8");
- return JSON.parse(raw) as T;
+ try {
+ return JSON.parse(raw) as T;
+ } catch (err) {
+ throw new ConfigError(`Corrupted connector config file for "${name}"`, err);
+ }
}
/** Check if a named connector config exists */
@@ -132,13 +140,19 @@ export function deleteConnectorDocuments(db: Database.Database, sourceType: stri
for (const row of rows) {
try {
deleteChunksFts.run(row.id);
- } catch {
- // FTS table may not exist
+ } catch (err) {
+ getLogger().debug(
+ { err, documentId: row.id },
+ "FTS table cleanup skipped (table may not exist)",
+ );
}
try {
deleteEmbeddings.run(row.id);
- } catch {
- // chunk_embeddings table may not exist
+ } catch (err) {
+ getLogger().debug(
+ { err, documentId: row.id },
+ "chunk_embeddings cleanup skipped (table may not exist)",
+ );
}
deleteChunks.run(row.id);
deleteDoc.run(row.id);
diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts
index 480470b..9739c5e 100644
--- a/src/connectors/obsidian.ts
+++ b/src/connectors/obsidian.ts
@@ -1,4 +1,5 @@
import { readdirSync, readFileSync, statSync } from "node:fs";
+import { load as yamlLoad } from "js-yaml";
import { join, relative, dirname, basename, extname, resolve } from "node:path";
import type Database from "better-sqlite3";
import type { EmbeddingProvider } from "../providers/embedding.js";
@@ -110,7 +111,20 @@ export function parseObsidianMarkdown(
if (fmMatch) {
const fmBlock = fmMatch[1] ?? "";
body = content.slice((fmMatch[0] ?? "").length).trimStart();
- frontmatter = parseSimpleYaml(fmBlock);
+ try {
+ const parsed = yamlLoad(fmBlock);
+ if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+ // js-yaml parses bare YAML date literals (e.g. 2024-01-15) as Date objects per YAML 1.1.
+ // Normalise them to ISO-8601 strings so downstream code always sees strings.
+ const normalised: Record = {};
+ for (const [k, v] of Object.entries(parsed as Record)) {
+ normalised[k] = v instanceof Date ? v.toISOString().slice(0, 10) : v;
+ }
+ frontmatter = normalised;
+ }
+ } catch {
+ // Malformed frontmatter — leave frontmatter as empty object and continue
+ }
}
// Build vault file map for wikilink resolution
@@ -184,61 +198,6 @@ export function parseObsidianMarkdown(
return { frontmatter, body: body.trim(), tags, wikilinks };
}
-function parseSimpleYaml(yaml: string): Record {
- const result: Record = {};
- const lines = yaml.split("\n");
- let currentKey: string | undefined;
- let listValues: string[] | undefined;
-
- for (const line of lines) {
- // List item continuation
- if (listValues !== undefined && /^\s+-\s+(.*)/.test(line)) {
- const itemMatch = /^\s+-\s+(.*)/.exec(line);
- if (itemMatch?.[1] !== undefined) {
- listValues.push(itemMatch[1].trim());
- }
- continue;
- }
-
- // Flush any pending list
- if (currentKey !== undefined && listValues !== undefined) {
- result[currentKey] = listValues;
- listValues = undefined;
- currentKey = undefined;
- }
-
- const colonIdx = line.indexOf(":");
- if (colonIdx < 1) continue;
- const key = line.slice(0, colonIdx);
- if (!/^[a-zA-Z_][a-zA-Z0-9_-]*$/.test(key)) continue;
- const value = line.slice(colonIdx + 1).trim();
-
- if (value === "" || value === "[]") {
- // Could be start of a list
- currentKey = key;
- listValues = value === "[]" ? [] : [];
- continue;
- }
-
- // Inline list: [a, b, c]
- if (value.startsWith("[") && value.endsWith("]")) {
- const inner = value.slice(1, -1);
- result[key] = inner.split(",").map((s) => s.trim().replace(/^['"]|['"]$/g, ""));
- continue;
- }
-
- // Simple scalar
- result[key] = value.replace(/^['"]|['"]$/g, "");
- }
-
- // Flush trailing list
- if (currentKey !== undefined && listValues !== undefined) {
- result[currentKey] = listValues;
- }
-
- return result;
-}
-
function resolveEmbeds(
body: string,
vaultPath: string,
diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts
index 043205b..6035283 100644
--- a/src/connectors/slack.ts
+++ b/src/connectors/slack.ts
@@ -499,13 +499,19 @@ export function disconnectSlack(db: Database.Database): number {
for (const row of rows) {
try {
deleteChunksFts.run(row.id);
- } catch {
- // FTS table may not exist
+ } catch (err) {
+ getLogger().debug(
+ { err, documentId: row.id },
+ "FTS table cleanup skipped (table may not exist)",
+ );
}
try {
deleteEmbeddings.run(row.id);
- } catch {
- // chunk_embeddings table may not exist
+ } catch (err) {
+ getLogger().debug(
+ { err, documentId: row.id },
+ "chunk_embeddings cleanup skipped (table may not exist)",
+ );
}
deleteChunks.run(row.id);
deleteDoc.run(row.id);
diff --git a/src/core/analytics.ts b/src/core/analytics.ts
index a44ef94..1a185ab 100644
--- a/src/core/analytics.ts
+++ b/src/core/analytics.ts
@@ -73,12 +73,23 @@ export function logSearch(db: Database.Database, entry: SearchLogEntry): string
/** Return overview stats for the knowledge base. */
export function getStats(db: Database.Database, dbPath?: string): OverviewStats {
- const docs = db.prepare("SELECT COUNT(*) AS cnt FROM documents").get() as { cnt: number };
- const chunks = db.prepare("SELECT COUNT(*) AS cnt FROM chunks").get() as { cnt: number };
- const topics = db.prepare("SELECT COUNT(*) AS cnt FROM topics").get() as { cnt: number };
- const searches = db.prepare("SELECT COUNT(*) AS cnt FROM search_log").get() as { cnt: number };
- const latency = db.prepare("SELECT AVG(latency_ms) AS avg FROM search_log").get() as {
- avg: number | null;
+ const row = db
+ .prepare(
+ `
+ SELECT
+ (SELECT COUNT(*) FROM documents) AS doc_count,
+ (SELECT COUNT(*) FROM chunks) AS chunk_count,
+ (SELECT COUNT(*) FROM topics) AS topic_count,
+ (SELECT COUNT(*) FROM search_log) AS search_count,
+ (SELECT AVG(latency_ms) FROM search_log) AS avg_latency
+ `,
+ )
+ .get() as {
+ doc_count: number;
+ chunk_count: number;
+ topic_count: number;
+ search_count: number;
+ avg_latency: number | null;
};
let databaseSizeBytes = 0;
@@ -91,12 +102,12 @@ export function getStats(db: Database.Database, dbPath?: string): OverviewStats
}
return {
- totalDocuments: docs.cnt,
- totalChunks: chunks.cnt,
- totalTopics: topics.cnt,
+ totalDocuments: row.doc_count,
+ totalChunks: row.chunk_count,
+ totalTopics: row.topic_count,
databaseSizeBytes,
- totalSearches: searches.cnt,
- avgLatencyMs: Math.round(latency.avg ?? 0),
+ totalSearches: row.search_count,
+ avgLatencyMs: Math.round(row.avg_latency ?? 0),
};
}
@@ -172,8 +183,8 @@ export function recordSearchQuery(db: Database.Database, entry: RecordSearchQuer
db.prepare(
"INSERT INTO search_queries (query, result_count, top_score, search_type) VALUES (?, ?, ?, ?)",
).run(entry.query, entry.resultCount, entry.topScore, entry.searchType);
- } catch {
- // silently ignore if table doesn't exist yet
+ } catch (err) {
+ getLogger().debug({ err }, "search_queries insert skipped (table may not exist yet)");
}
}
diff --git a/src/core/batch-search.ts b/src/core/batch-search.ts
new file mode 100644
index 0000000..7785c18
--- /dev/null
+++ b/src/core/batch-search.ts
@@ -0,0 +1,52 @@
+import type Database from "better-sqlite3";
+import type { EmbeddingProvider } from "../providers/embedding.js";
+import { searchDocuments, type SearchOptions, type SearchResponse } from "./search.js";
+import { ValidationError } from "../errors.js";
+
+export const BATCH_SEARCH_MAX_REQUESTS = 20;
+
+export interface BatchSearchRequest {
+ /** The query string for this search. */
+ query: string;
+ /** Per-request overrides — all SearchOptions except `query` which comes from above. */
+ options?: Omit;
+}
+
+export interface BatchSearchResponse {
+ /** Results keyed by the original query string. */
+ results: Record;
+}
+
+/**
+ * Execute multiple search queries concurrently.
+ * Results are keyed by the query string.
+ */
+export async function searchBatch(
+ db: Database.Database,
+ provider: EmbeddingProvider,
+ requests: BatchSearchRequest[],
+): Promise {
+ if (!Array.isArray(requests) || requests.length === 0) {
+ throw new ValidationError("At least one search request is required");
+ }
+ if (requests.length > BATCH_SEARCH_MAX_REQUESTS) {
+ throw new ValidationError(
+ `Batch size ${requests.length} exceeds maximum of ${BATCH_SEARCH_MAX_REQUESTS}`,
+ );
+ }
+
+ const entries = await Promise.all(
+ requests.map(async (req) => {
+ const searchOpts: SearchOptions = {
+ ...req.options,
+ query: req.query,
+ };
+ const response = await searchDocuments(db, provider, searchOpts);
+ return [req.query, response] as const;
+ }),
+ );
+
+ return {
+ results: Object.fromEntries(entries),
+ };
+}
diff --git a/src/core/bulk.ts b/src/core/bulk.ts
index 5bba9d8..1df1438 100644
--- a/src/core/bulk.ts
+++ b/src/core/bulk.ts
@@ -2,7 +2,12 @@ import type Database from "better-sqlite3";
import { getLogger } from "../logger.js";
import { ValidationError } from "../errors.js";
import { deleteDocument, listDocuments } from "./documents.js";
-import { addTagsToDocument, removeTagFromDocument, getDocumentTags } from "./tags.js";
+import {
+ addTagsToDocument,
+ removeTagFromDocument,
+ getDocumentTags,
+ getDocumentTagsBatch,
+} from "./tags.js";
export interface BulkSelector {
topicId?: string;
@@ -42,71 +47,35 @@ export function resolveSelector(
throw new ValidationError("Bulk selector must specify at least one filter criterion");
}
+ if (limit !== undefined && limit < 0) {
+ throw new ValidationError("limit must be a non-negative integer");
+ }
+
const effectiveLimit = Math.max(0, Math.min(limit ?? MAX_BATCH_SIZE, MAX_BATCH_SIZE));
if (effectiveLimit === 0) {
return [];
}
- // Use listDocuments for basic filters
+ // Push date filters into the SQL query so they apply before LIMIT
const docs = listDocuments(db, {
library: selector.library,
topicId: selector.topicId,
sourceType: selector.sourceType,
+ dateFrom: selector.dateFrom,
+ dateTo: selector.dateTo,
limit: effectiveLimit,
});
let ids = docs.map((d) => d.id);
- // Build a Map for O(1) lookup instead of O(n) .find() per id
- const docMap = new Map(docs.map((d) => [d.id, d]));
-
- // Apply date filters
- if (selector.dateFrom) {
- const from = selector.dateFrom;
- ids = ids.filter((id) => {
- const doc = docMap.get(id);
- return doc != null && doc.createdAt >= from;
- });
- }
- if (selector.dateTo) {
- const to = selector.dateTo;
- ids = ids.filter((id) => {
- const doc = docMap.get(id);
- return doc != null && doc.createdAt <= to;
- });
- }
-
- // Apply tag filter (AND logic — document must have ALL specified tags)
+ // Apply tag filter (AND logic — document must have ALL specified tags).
+ // Fetch all tags in a single query instead of one query per document.
if (selector.tags && selector.tags.length > 0) {
const requiredTags = selector.tags.map((t) => t.trim().toLowerCase());
- // Batch query: fetch tags for all candidate documents, chunked to respect SQLite parameter limits
- const SQLITE_MAX_PARAMS = 999;
- const tagRows: Array<{ document_id: string; name: string }> = [];
- for (let i = 0; i < ids.length; i += SQLITE_MAX_PARAMS) {
- const chunk = ids.slice(i, i + SQLITE_MAX_PARAMS);
- const placeholders = chunk.map(() => "?").join(", ");
- const rows = db
- .prepare(
- `SELECT dt.document_id, t.name
- FROM tags t
- JOIN document_tags dt ON dt.tag_id = t.id
- WHERE dt.document_id IN (${placeholders})`,
- )
- .all(...chunk) as Array<{ document_id: string; name: string }>;
- tagRows.push(...rows);
- }
- const tagsByDoc = new Map();
- for (const row of tagRows) {
- const existing = tagsByDoc.get(row.document_id);
- if (existing) {
- existing.push(row.name);
- } else {
- tagsByDoc.set(row.document_id, [row.name]);
- }
- }
+ const tagsByDoc = getDocumentTagsBatch(db, ids);
ids = ids.filter((id) => {
- const docTags = tagsByDoc.get(id) ?? [];
+ const docTags = (tagsByDoc.get(id) ?? []).map((t) => t.name);
return requiredTags.every((rt) => docTags.includes(rt));
});
}
diff --git a/src/core/documents.ts b/src/core/documents.ts
index 4670890..f6280c9 100644
--- a/src/core/documents.ts
+++ b/src/core/documents.ts
@@ -96,6 +96,8 @@ export function listDocuments(
library?: string | undefined;
topicId?: string | undefined;
sourceType?: string | undefined;
+ dateFrom?: string | undefined;
+ dateTo?: string | undefined;
limit?: number | undefined;
},
): Document[] {
@@ -117,6 +119,14 @@ export function listDocuments(
sql += " AND source_type = ?";
params.push(options.sourceType);
}
+ if (options?.dateFrom) {
+ sql += " AND created_at >= ?";
+ params.push(options.dateFrom);
+ }
+ if (options?.dateTo) {
+ sql += " AND created_at <= ?";
+ params.push(options.dateTo);
+ }
sql += " ORDER BY updated_at DESC LIMIT ?";
params.push(options?.limit ?? 50);
@@ -199,6 +209,10 @@ export async function updateDocument(
? createHash("sha256").update(newContent).digest("hex")
: existing.contentHash;
+ // Use JS Date so test fake-timers (vi.setSystemTime) can control the timestamp.
+ // SQLite's datetime('now') uses the OS clock and cannot be mocked in unit tests.
+ const updatedAt = new Date().toISOString().replace("T", " ").slice(0, 19);
+
if (contentChanged) {
log.info({ docId: documentId }, "Content changed, re-chunking and re-indexing embeddings");
@@ -221,7 +235,7 @@ export async function updateDocument(
db.prepare("DELETE FROM chunks WHERE document_id = ?").run(documentId);
db.prepare(
- `UPDATE documents SET title = ?, content = ?, library = ?, version = ?, url = ?, topic_id = ?, content_hash = ?, updated_at = datetime('now') WHERE id = ?`,
+ `UPDATE documents SET title = ?, content = ?, library = ?, version = ?, url = ?, topic_id = ?, content_hash = ?, updated_at = ? WHERE id = ?`,
).run(
newTitle,
newContent,
@@ -230,6 +244,7 @@ export async function updateDocument(
newUrl,
newTopicId,
contentHash,
+ updatedAt,
documentId,
);
@@ -260,8 +275,8 @@ export async function updateDocument(
saveVersion(db, documentId);
db.prepare(
- `UPDATE documents SET title = ?, library = ?, version = ?, url = ?, topic_id = ?, updated_at = datetime('now') WHERE id = ?`,
- ).run(newTitle, newLibrary, newVersion, newUrl, newTopicId, documentId);
+ `UPDATE documents SET title = ?, library = ?, version = ?, url = ?, topic_id = ?, updated_at = ? WHERE id = ?`,
+ ).run(newTitle, newLibrary, newVersion, newUrl, newTopicId, updatedAt, documentId);
});
transaction();
diff --git a/src/core/export.ts b/src/core/export.ts
index 02b7e35..24c94a5 100644
--- a/src/core/export.ts
+++ b/src/core/export.ts
@@ -39,8 +39,8 @@ export function exportKnowledgeBase(db: Database.Database, outputPath: string):
webhooks = (db.prepare("SELECT * FROM webhooks").all() as Record[]).map(
(w) => ({ ...w, secret: w.secret != null ? "[REDACTED]" : null }),
);
- } catch {
- // webhooks table may not exist
+ } catch (err) {
+ log.debug({ err }, "Webhooks table not present in export (table may not exist)");
}
const data: ExportData = {
diff --git a/src/core/index.ts b/src/core/index.ts
index e95133a..918472e 100644
--- a/src/core/index.ts
+++ b/src/core/index.ts
@@ -1,5 +1,10 @@
export { indexDocument, indexFile, chunkContent } from "./indexing.js";
-export type { IndexDocumentInput, IndexFileOptions, IndexedDocument } from "./indexing.js";
+export type {
+ IndexDocumentInput,
+ IndexFileOptions,
+ IndexedDocument,
+ ChunkOptions,
+} from "./indexing.js";
export { getParserForFile, getSupportedExtensions } from "./parsers/index.js";
export type { DocumentParser } from "./parsers/index.js";
@@ -226,3 +231,12 @@ export type { Webhook, RedactedWebhook, WebhookEvent, WebhookPayload } from "./w
export { ConnectorScheduler, loadScheduleEntries } from "./scheduler.js";
export type { ScheduleConfig, SchedulerStatus } from "./scheduler.js";
+
+export { searchBatch, BATCH_SEARCH_MAX_REQUESTS } from "./batch-search.js";
+export type { BatchSearchRequest, BatchSearchResponse } from "./batch-search.js";
+
+export { pruneExpiredDocuments } from "./ttl.js";
+export type { PruneResult } from "./ttl.js";
+
+export { LibScope } from "../LibScope.js";
+export type { LibScopeOptions } from "../LibScope.js";
diff --git a/src/core/indexing.ts b/src/core/indexing.ts
index 9cfca2d..2a025d5 100644
--- a/src/core/indexing.ts
+++ b/src/core/indexing.ts
@@ -23,6 +23,8 @@ export interface IndexDocumentInput {
dedup?: "skip" | "warn" | "force" | undefined;
/** Options for duplicate detection (threshold, strategy). */
dedupOptions?: DedupOptions | undefined;
+ /** ISO 8601 expiry timestamp. Document will be pruned by pruneExpiredDocuments() after this time. */
+ expiresAt?: string | undefined;
}
export interface IndexedDocument {
@@ -30,26 +32,57 @@ export interface IndexedDocument {
chunkCount: number;
}
+export interface ChunkOptions {
+ /** Maximum characters per chunk (default 1500). */
+ maxChunkSize?: number;
+ /** Fraction of the chunk to overlap with the next (0–0.5, default 0.1). */
+ overlapFraction?: number;
+}
+
/**
- * Split content into chunks by markdown headings.
- * Falls back to paragraph-based splitting for non-markdown content.
+ * Split content into chunks by markdown headings with paragraph-aware
+ * splitting for oversized sections and configurable inter-chunk overlap.
+ * Breadcrumbs use plain text ("Context: …") instead of HTML comments
+ * so the text is meaningful to embedding models.
*/
-export function chunkContent(content: string, maxChunkSize: number = 1500): string[] {
+export function chunkContent(
+ content: string,
+ maxChunkSizeOrOpts: number | ChunkOptions = 1500,
+): string[] {
+ const opts: ChunkOptions =
+ typeof maxChunkSizeOrOpts === "number"
+ ? { maxChunkSize: maxChunkSizeOrOpts }
+ : maxChunkSizeOrOpts;
+ const maxChunkSize = opts.maxChunkSize ?? 1500;
+ const overlapFraction = Math.max(0, Math.min(opts.overlapFraction ?? 0.1, 0.5));
+
const lines = content.split("\n");
- const chunks: string[] = [];
+ const rawChunks: string[] = [];
let currentChunk: string[] = [];
let currentChunkLen = 0; // Running byte length to avoid O(n²) join-per-line
const headingStack: Array<{ level: number; text: string }> = [];
+ /** Flush currentChunk into rawChunks, splitting at paragraph boundaries if oversized. */
+ const flushChunk = (): void => {
+ const text = currentChunk.join("\n").trim();
+ if (text.length === 0) return;
+
+ if (text.length <= maxChunkSize) {
+ rawChunks.push(text);
+ } else {
+ // Paragraph-boundary splitting for oversized chunks
+ splitAtParagraphs(text, maxChunkSize, rawChunks);
+ }
+ currentChunk = [];
+ currentChunkLen = 0;
+ };
+
for (const line of lines) {
const headingMatch = /^(#{1,3}) +(\S.*)$/.exec(line);
// Split on markdown headings (## or higher)
if (headingMatch && currentChunk.length > 0) {
- const text = currentChunk.join("\n").trim();
- if (text.length > 0) {
- chunks.push(text);
- }
+ flushChunk();
// Update heading stack
const level = (headingMatch[1] ?? "").length;
@@ -61,12 +94,12 @@ export function chunkContent(content: string, maxChunkSize: number = 1500): stri
headingStack.pop();
}
- // Build breadcrumb from parent headings
+ // Build breadcrumb from parent headings — plain text for embedding quality
const breadcrumb = headingStack.map((h) => h.text).join(" > ");
headingStack.push({ level, text: (headingMatch[2] ?? "").trim() });
if (breadcrumb) {
- const ctx = ``;
+ const ctx = `Context: ${breadcrumb}`;
currentChunk = [ctx, line];
currentChunkLen = ctx.length + 1 + line.length;
} else {
@@ -85,22 +118,85 @@ export function chunkContent(content: string, maxChunkSize: number = 1500): stri
// Also split if chunk gets too large (use running counter instead of join)
if (currentChunkLen > maxChunkSize) {
- const text = currentChunk.join("\n").trim();
- if (text.length > 0) {
- chunks.push(text);
- }
- currentChunk = [];
- currentChunkLen = 0;
+ flushChunk();
}
}
// Don't forget the last chunk
- const remaining = currentChunk.join("\n").trim();
- if (remaining.length > 0) {
- chunks.push(remaining);
+ flushChunk();
+
+ // Apply inter-chunk overlap
+ if (overlapFraction > 0 && rawChunks.length > 1) {
+ return addChunkOverlap(rawChunks, overlapFraction);
+ }
+
+ return rawChunks;
+}
+
+/**
+ * Split oversized text at paragraph boundaries (double-newline).
+ * Falls back to hard character split when a paragraph exceeds maxSize.
+ */
+function splitAtParagraphs(text: string, maxSize: number, out: string[]): void {
+ const emit = (buf: string): void => {
+ const trimmed = buf.trim();
+ if (trimmed.length === 0) return;
+ if (trimmed.length <= maxSize) {
+ out.push(trimmed);
+ } else {
+ for (let i = 0; i < trimmed.length; i += maxSize) {
+ const slice = trimmed.slice(i, i + maxSize).trim();
+ if (slice.length > 0) out.push(slice);
+ }
+ }
+ };
+
+ const paragraphs = text.split(/\n\n+/);
+ let buffer = "";
+
+ for (const para of paragraphs) {
+ const candidate = buffer.length === 0 ? para : buffer + "\n\n" + para;
+ if (candidate.length > maxSize && buffer.length > 0) {
+ emit(buffer);
+ buffer = para;
+ } else {
+ buffer = candidate;
+ }
}
- return chunks;
+ emit(buffer);
+}
+
+/**
+ * Add overlap between consecutive chunks by appending trailing text from
+ * the previous chunk to the beginning of the next chunk.
+ */
+function addChunkOverlap(chunks: string[], fraction: number): string[] {
+ const result: string[] = [chunks[0]!];
+
+ for (let i = 1; i < chunks.length; i++) {
+ const prev = chunks[i - 1]!;
+ const overlapChars = Math.floor(prev.length * fraction);
+
+ if (overlapChars > 0) {
+ // Take trailing portion of previous chunk, preferring line boundaries
+ let overlapText = prev.slice(-overlapChars);
+ const newlineIdx = overlapText.indexOf("\n");
+ if (newlineIdx > 0) {
+ overlapText = overlapText.slice(newlineIdx + 1);
+ }
+ overlapText = overlapText.trim();
+ if (overlapText.length > 0) {
+ result.push(overlapText + "\n\n" + chunks[i]!);
+ } else {
+ result.push(chunks[i]!);
+ }
+ } else {
+ result.push(chunks[i]!);
+ }
+ }
+
+ return result;
}
/** Size threshold above which streaming chunking is used (1MB). */
@@ -161,7 +257,8 @@ export function chunkContentStreaming(
const window = text.slice(offset, windowEnd);
// Chunk this window using the existing logic
- const windowChunks = chunkContent(window, maxChunkSize);
+ // Disable overlap inside chunkContent — streaming already handles overlap at the window level
+ const windowChunks = chunkContent(window, { maxChunkSize, overlapFraction: 0 });
for (const chunk of windowChunks) {
const normalized = chunk.replace(/\s+/g, " ").trim();
const hash = createHash("sha256").update(normalized).digest("hex");
@@ -253,9 +350,24 @@ export async function indexDocument(
.prepare("SELECT id FROM documents WHERE title = ? AND LENGTH(content) = ?")
.get(input.title, contentLength) as { id: string } | undefined;
if (existingByContent) {
- throw new ValidationError(
- `Document with same title and content length already exists (id: ${existingByContent.id}). Delete it first or modify the content.`,
- );
+ if (input.dedup === "skip") {
+ log.info(
+ { existingDocId: existingByContent.id, title: input.title },
+ "Duplicate by title+length detected, skipping",
+ );
+ return { id: existingByContent.id, chunkCount: 0 };
+ }
+ if (input.dedup === "warn") {
+ log.warn(
+ { existingDocId: existingByContent.id, title: input.title },
+ "Duplicate by title+length detected, indexing anyway",
+ );
+ // Continue indexing with a new ID
+ } else {
+ throw new ValidationError(
+ `Document with same title and content length already exists (id: ${existingByContent.id}). Delete it first or modify the content.`,
+ );
+ }
}
const docId = randomUUID();
@@ -267,13 +379,24 @@ export async function indexDocument(
"Indexing document",
);
- // Generate embeddings for all chunks
- const embeddings = await provider.embedBatch(chunks);
+ // Prepend document metadata to each chunk for richer embeddings.
+ // The stored chunk content stays clean; only the text sent to the embedding
+ // model gets the prefix so that semantic search can leverage title/library/version.
+ const metaParts: string[] = [];
+ if (input.title) metaParts.push(input.title);
+ if (input.library) metaParts.push(`Library: ${input.library}`);
+ if (input.version) metaParts.push(`Version: ${input.version}`);
+ const metaPrefix = metaParts.length > 0 ? metaParts.join(" | ") + "\n\n" : "";
+
+ const textsForEmbedding = chunks.map((c) => metaPrefix + c);
+
+ // Generate embeddings for all chunks (with metadata-enriched text)
+ const embeddings = await provider.embedBatch(textsForEmbedding);
// Store everything in a transaction
const insertDoc = db.prepare(`
- INSERT INTO documents (id, source_type, library, version, topic_id, title, content, url, submitted_by, content_hash)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ INSERT INTO documents (id, source_type, library, version, topic_id, title, content, url, submitted_by, content_hash, expires_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const insertChunk = db.prepare(`
@@ -317,6 +440,7 @@ export async function indexDocument(
input.url ?? null,
input.submittedBy ?? "manual",
contentHash,
+ input.expiresAt ?? null,
);
for (let i = 0; i < chunks.length; i++) {
@@ -333,9 +457,11 @@ export async function indexDocument(
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
if (message.includes("no such table")) {
- log.debug({ chunkId, err }, "Skipped vector insertion (sqlite-vec may not be loaded)");
+ log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)");
} else {
- log.warn({ chunkId, err }, "Failed to insert vector embedding");
+ // Re-throw so the transaction rolls back — don't silently commit
+ // chunks that have no embedding (they would be invisible to semantic search).
+ throw err;
}
}
}
diff --git a/src/core/link-extractor.ts b/src/core/link-extractor.ts
new file mode 100644
index 0000000..5c7c889
--- /dev/null
+++ b/src/core/link-extractor.ts
@@ -0,0 +1,159 @@
+/**
+ * Link extraction from HTML.
+ * Parses tags using fast indexOf-based parsing (no regex catastrophic backtracking).
+ * Resolves relative URLs, strips fragments, deduplicates, and filters to http/https only.
+ */
+
+/**
+ * Extract all unique, normalized http/https links from an HTML string.
+ *
+ * @param html Raw HTML to parse.
+ * @param baseUrl The URL the HTML was fetched from — used to resolve relative hrefs.
+ * @returns Deduplicated array of absolute http/https URLs (no fragments, trailing slashes
+ * on path roots normalized away).
+ */
+export function extractLinks(html: string, baseUrl: string): string[] {
+ const seen = new Set();
+ const links: string[] = [];
+
+ let pos = 0;
+ const lower = html.toLowerCase();
+
+ while (pos < html.length) {
+ // Find the next tag (next char must be space, >, or /)
+ const charAfterA = lower[tagStart + 2];
+ if (
+ charAfterA !== " " &&
+ charAfterA !== "\t" &&
+ charAfterA !== "\n" &&
+ charAfterA !== "\r" &&
+ charAfterA !== ">" &&
+ charAfterA !== "/"
+ ) {
+ pos = tagStart + 2;
+ continue;
+ }
+
+ // Find end of opening tag
+ const tagEnd = html.indexOf(">", tagStart);
+ if (tagEnd === -1) break;
+
+ const tag = html.slice(tagStart, tagEnd + 1);
+ const href = extractHref(tag);
+
+ if (href !== null) {
+ const resolved = resolveUrl(href, baseUrl);
+ if (resolved !== null && !seen.has(resolved)) {
+ seen.add(resolved);
+ links.push(resolved);
+ }
+ }
+
+ pos = tagEnd + 1;
+ }
+
+ return links;
+}
+
+/**
+ * Extract the href attribute value from an tag string.
+ * Returns null if no href found or href is empty.
+ */
+function extractHref(tag: string): string | null {
+ const lowerTag = tag.toLowerCase();
+ let searchPos = 0;
+
+ while (searchPos < lowerTag.length) {
+ const hrefIdx = lowerTag.indexOf("href", searchPos);
+ if (hrefIdx === -1) return null;
+
+ // Require an attribute boundary before "href" to avoid matching data-href, aria-href, etc.
+ // The character immediately preceding "href" must be whitespace (or it's at position 0,
+ // which can't happen in a valid tag and so we skip it).
+ const charBefore = hrefIdx > 0 ? lowerTag[hrefIdx - 1] : "";
+ if (charBefore !== " " && charBefore !== "\t" && charBefore !== "\n" && charBefore !== "\r") {
+ searchPos = hrefIdx + 4;
+ continue;
+ }
+
+ // Skip whitespace before =
+ let eqIdx = hrefIdx + 4;
+ while (eqIdx < tag.length && (tag[eqIdx] === " " || tag[eqIdx] === "\t")) eqIdx++;
+
+ if (tag[eqIdx] !== "=") {
+ searchPos = hrefIdx + 4;
+ continue;
+ }
+
+ // Skip whitespace after =
+ let valStart = eqIdx + 1;
+ while (valStart < tag.length && (tag[valStart] === " " || tag[valStart] === "\t")) valStart++;
+
+ if (valStart >= tag.length) return null;
+
+ let href: string;
+ const quote = tag[valStart];
+ if (quote === '"' || quote === "'") {
+ const closeQuote = tag.indexOf(quote, valStart + 1);
+ if (closeQuote === -1) return null;
+ href = tag.slice(valStart + 1, closeQuote);
+ } else {
+ // Unquoted attribute value — ends at whitespace or >
+ let end = valStart;
+ while (
+ end < tag.length &&
+ tag[end] !== " " &&
+ tag[end] !== "\t" &&
+ tag[end] !== ">" &&
+ tag[end] !== "\n"
+ ) {
+ end++;
+ }
+ href = tag.slice(valStart, end);
+ }
+
+ href = href.trim();
+ return href.length > 0 ? href : null;
+ }
+
+ return null;
+}
+
+/**
+ * Resolve a potentially-relative href against a base URL.
+ * Returns null if the result is not an http/https URL (e.g. mailto:, javascript:, data:, #fragment-only).
+ */
+function resolveUrl(href: string, baseUrl: string): string | null {
+ // Skip fragment-only links immediately — they point to the same page
+ if (href.startsWith("#")) return null;
+
+ let resolved: URL;
+ try {
+ resolved = new URL(href, baseUrl);
+ } catch {
+ return null;
+ }
+
+ // Allowlist: only permit http and https.
+ // This rejects javascript:, vbscript:, data:, mailto:, ftp:, file:, and
+ // any other non-http scheme without needing an enumerated blocklist.
+ if (resolved.protocol !== "http:" && resolved.protocol !== "https:") {
+ return null;
+ }
+
+ // Strip fragment
+ resolved.hash = "";
+
+ // Normalize: remove trailing slash from non-root paths
+ // e.g. https://example.com/docs/ → https://example.com/docs
+ // but https://example.com/ stays as https://example.com/
+ if (resolved.pathname.length > 1 && resolved.pathname.endsWith("/")) {
+ resolved.pathname = resolved.pathname.slice(0, -1);
+ }
+
+ return resolved.href;
+}
diff --git a/src/core/packs.ts b/src/core/packs.ts
index 26642e9..45aeca1 100644
--- a/src/core/packs.ts
+++ b/src/core/packs.ts
@@ -1,10 +1,23 @@
import type Database from "better-sqlite3";
-import { readFileSync, writeFileSync } from "node:fs";
-import { resolve as pathResolve, isAbsolute as pathIsAbsolute } from "node:path";
+import { randomUUID, createHash } from "node:crypto";
+import { readFileSync, writeFileSync, readdirSync, statSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+import {
+ resolve as pathResolve,
+ isAbsolute as pathIsAbsolute,
+ basename,
+ relative,
+ join as pathJoin,
+ extname,
+} from "node:path";
+import { gzipSync, gunzipSync } from "node:zlib";
import type { EmbeddingProvider } from "../providers/embedding.js";
import { ValidationError, FetchError } from "../errors.js";
import { getLogger } from "../logger.js";
-import { indexDocument } from "./indexing.js";
+import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js";
+import { getParserForFile, getSupportedExtensions } from "./parsers/index.js";
+import { suggestTagsFromText } from "./tags.js";
+import { fetchAndConvert } from "./url-fetcher.js";
export interface PackDocument {
title: string;
@@ -45,6 +58,19 @@ export interface InstallResult {
packName: string;
documentsInstalled: number;
alreadyInstalled: boolean;
+ errors: number;
+}
+
+export interface InstallOptions {
+ registryUrl?: string | undefined;
+ /** Number of documents to embed and insert per batch. Default: 10. */
+ batchSize?: number | undefined;
+ /** Skip the first N documents (for resuming a partial install). Default: 0. */
+ resumeFrom?: number | undefined;
+ /** Maximum number of batches to embed concurrently. Default: 4. */
+ concurrency?: number | undefined;
+ /** Called after each batch of documents is processed. */
+ onProgress?: ((current: number, total: number, docTitle: string) => void) | undefined;
}
export interface CreatePackOptions {
@@ -57,8 +83,55 @@ export interface CreatePackOptions {
outputPath?: string | undefined;
}
+export interface CreatePackFromSourceOptions {
+ /** Pack name (required). */
+ name: string;
+ /** One or more source paths (directories or files) or URLs. */
+ from: string[];
+ version?: string | undefined;
+ description?: string | undefined;
+ author?: string | undefined;
+ license?: string | undefined;
+ outputPath?: string | undefined;
+ /** Only include files with these extensions (e.g. [".md", ".html"]). Defaults to all supported. */
+ extensions?: string[] | undefined;
+ /** Glob-style patterns to exclude (matched against the relative path from the source root). */
+ exclude?: string[] | undefined;
+ /** Walk directories recursively (default: true). */
+ recursive?: boolean | undefined;
+ /** Called for each file processed, for progress reporting. */
+ onProgress?: ((info: { file: string; index: number; total: number }) => void) | undefined;
+}
+
const DEFAULT_REGISTRY_URL = "https://raw.githubusercontent.com/libscope/packs/main/registry.json";
+/** Gzip magic number: first two bytes of a gzip stream. */
+const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]);
+
+/** Check if a filename indicates gzip compression (.gz or .json.gz). */
+function isGzipPath(filePath: string): boolean {
+ return filePath.endsWith(".gz");
+}
+
+/** Write a pack to disk, gzip-compressing if the path ends in .gz. */
+function writePackFile(filePath: string, pack: KnowledgePack): void {
+ const json = JSON.stringify(pack, null, 2);
+ if (isGzipPath(filePath)) {
+ writeFileSync(filePath, gzipSync(Buffer.from(json, "utf-8")));
+ } else {
+ writeFileSync(filePath, json, "utf-8");
+ }
+}
+
+/** Read a pack file, auto-detecting gzip by magic bytes or extension. */
+function readPackFile(filePath: string): string {
+ const raw = readFileSync(filePath);
+ if (raw.length >= 2 && raw[0] === GZIP_MAGIC[0] && raw[1] === GZIP_MAGIC[1]) {
+ return gunzipSync(raw).toString("utf-8");
+ }
+ return raw.toString("utf-8");
+}
+
/** Validate that a registry URL uses https and is not a private IP. */
function validateRegistryUrl(url: string): void {
let parsed: URL;
@@ -190,20 +263,20 @@ export async function installPack(
db: Database.Database,
provider: EmbeddingProvider,
packNameOrPath: string,
- options?: { registryUrl?: string | undefined },
+ options?: InstallOptions,
): Promise {
const log = getLogger();
let pack: KnowledgePack;
- // Try loading as a local file first
- if (packNameOrPath.endsWith(".json")) {
+ // Try loading as a local file first (supports .json and .json.gz)
+ if (packNameOrPath.endsWith(".json") || packNameOrPath.endsWith(".json.gz")) {
const resolved = pathResolve(packNameOrPath);
// Prevent path traversal: if a relative path is given, ensure it resolves within CWD
if (!pathIsAbsolute(packNameOrPath) && !resolved.startsWith(process.cwd())) {
throw new ValidationError("Pack file path must be within the current working directory");
}
try {
- const raw = readFileSync(resolved, "utf-8");
+ const raw = readPackFile(resolved);
const parsed: unknown = JSON.parse(raw);
pack = validatePack(parsed);
} catch (err) {
@@ -215,6 +288,7 @@ export async function installPack(
} else {
// Fetch from registry
const registryUrl = options?.registryUrl ?? DEFAULT_REGISTRY_URL;
+
validateRegistryUrl(registryUrl);
const baseUrl = registryUrl.replace(/\/[^/]+$/, "");
const packUrl = `${baseUrl}/${packNameOrPath}.json`;
@@ -241,10 +315,36 @@ export async function installPack(
if (existing) {
log.info({ pack: pack.name }, "Pack already installed");
- return { packName: pack.name, documentsInstalled: 0, alreadyInstalled: true };
+ return { packName: pack.name, documentsInstalled: 0, alreadyInstalled: true, errors: 0 };
}
- log.info({ pack: pack.name, docCount: pack.documents.length }, "Installing pack");
+ const batchSize = options?.batchSize ?? 10;
+ const concurrency = options?.concurrency ?? 4;
+ const resumeFrom = options?.resumeFrom ?? 0;
+ const onProgress = options?.onProgress;
+ const total = pack.documents.length;
+
+ if (!Number.isInteger(batchSize) || batchSize <= 0) {
+ throw new ValidationError("batchSize must be a positive integer");
+ }
+ if (!Number.isInteger(concurrency) || concurrency <= 0) {
+ throw new ValidationError("concurrency must be a positive integer");
+ }
+ if (!Number.isInteger(resumeFrom) || resumeFrom < 0) {
+ throw new ValidationError("resumeFrom must be a non-negative integer");
+ }
+ if (resumeFrom > total) {
+ throw new ValidationError(
+ "resumeFrom cannot be greater than the total number of documents in the pack",
+ );
+ }
+
+ const docs = resumeFrom > 0 ? pack.documents.slice(resumeFrom) : pack.documents;
+
+ log.info(
+ { pack: pack.name, docCount: total, batchSize, concurrency, resumeFrom },
+ "Installing pack",
+ );
// Insert the pack record first (documents.pack_name has FK to packs.name)
db.prepare("INSERT INTO packs (name, version, description, doc_count) VALUES (?, ?, ?, 0)").run(
@@ -253,34 +353,198 @@ export async function installPack(
pack.description,
);
- let installed = 0;
- for (const doc of pack.documents) {
- try {
- const result = await indexDocument(db, provider, {
- title: doc.title,
- content: doc.content,
- sourceType: "library",
- url: doc.source || undefined,
- submittedBy: "manual",
- dedup: "warn",
+ // Prepare statements once (reused across all batches)
+ const insertDoc = db.prepare(`
+ INSERT INTO documents (id, source_type, title, content, url, submitted_by, content_hash, pack_name)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ const insertChunk = db.prepare(`
+ INSERT INTO chunks (id, document_id, content, chunk_index)
+ VALUES (?, ?, ?, ?)
+ `);
+ const insertEmbedding = db.prepare(`
+ INSERT INTO chunk_embeddings (chunk_id, embedding)
+ VALUES (?, ?)
+ `);
+
+ // Slice docs into batches by index only — chunks are computed lazily when each
+ // batch is scheduled, so we never hold all chunks in memory simultaneously.
+ type DocChunkInfo = {
+ doc: PackDocument;
+ docId: string;
+ contentHash: string;
+ chunks: string[];
+ chunkOffset: number; // offset into allChunks for this batch
+ };
+ type BatchData = {
+ batchDocs: PackDocument[];
+ };
+ type ResolvedBatch = {
+ docInfos: DocChunkInfo[];
+ allChunks: string[];
+ };
+
+ const batches: BatchData[] = [];
+ for (let batchStart = 0; batchStart < docs.length; batchStart += batchSize) {
+ batches.push({ batchDocs: docs.slice(batchStart, batchStart + batchSize) });
+ }
+
+ /** Chunk a batch's documents on demand, right before embedding. */
+ function resolveBatch(batch: BatchData): ResolvedBatch {
+ const docInfos: DocChunkInfo[] = [];
+ const allChunks: string[] = [];
+ for (const doc of batch.batchDocs) {
+ const contentHash = createHash("sha256").update(doc.content).digest("hex");
+ const useStreaming = doc.content.length > STREAMING_THRESHOLD;
+ const chunks = useStreaming ? chunkContentStreaming(doc.content) : chunkContent(doc.content);
+ docInfos.push({
+ doc,
+ docId: randomUUID(),
+ contentHash,
+ chunks,
+ chunkOffset: allChunks.length,
});
+ allChunks.push(...chunks);
+ }
+ return { docInfos, allChunks };
+ }
- // Tag the document with the pack name
- db.prepare("UPDATE documents SET pack_name = ? WHERE id = ?").run(pack.name, result.id);
- installed++;
- } catch (err) {
- log.warn(
- { err, title: doc.title, pack: pack.name },
- "Failed to index pack document, skipping",
+ // Phase 2 & 3: Embed batches concurrently (up to `concurrency` at a time),
+ // inserting each batch into the DB in order as embeddings complete.
+ // This maximises provider throughput while keeping inserts serialised (SQLite requirement).
+ let installed = 0;
+ let errors = 0;
+ let processedCount = resumeFrom;
+
+ type EmbedResult = { resolved: ResolvedBatch; embeddings: number[][]; success: boolean };
+ const embedResults: Array = Array.from({
+ length: batches.length,
+ });
+ let nextInsertIdx = 0;
+
+ /** Insert completed batches in index order, advancing nextInsertIdx. */
+ function flushInserts(): void {
+ while (nextInsertIdx < batches.length && embedResults[nextInsertIdx] !== undefined) {
+ const i = nextInsertIdx++;
+ const { resolved: batch, embeddings, success } = embedResults[i]!;
+ const result = { embeddings, success };
+
+ if (!result.success) {
+ errors += batch.docInfos.length;
+ } else {
+ let batchInstalled = 0;
+ const doInsert = db.transaction(() => {
+ batchInstalled = 0;
+ for (const info of batch.docInfos) {
+ insertDoc.run(
+ info.docId,
+ "library",
+ info.doc.title,
+ info.doc.content,
+ info.doc.source || null,
+ "manual",
+ info.contentHash,
+ pack.name,
+ );
+ for (let j = 0; j < info.chunks.length; j++) {
+ const chunkId = randomUUID();
+ const chunkText = info.chunks[j] ?? "";
+ const embedding = result.embeddings[info.chunkOffset + j] ?? [];
+ insertChunk.run(chunkId, info.docId, chunkText, j);
+ try {
+ const vecBuffer = Buffer.from(new Float32Array(embedding).buffer);
+ insertEmbedding.run(chunkId, vecBuffer);
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ if (!message.includes("no such table")) {
+ log.warn({ chunkId, err }, "Failed to insert vector embedding");
+ }
+ }
+ }
+ batchInstalled++;
+ }
+ });
+ try {
+ doInsert();
+ installed += batchInstalled;
+ } catch (err) {
+ log.warn(
+ { err, pack: pack.name, batchIndex: i },
+ "Transaction failed for batch, skipping these documents",
+ );
+ errors += batch.docInfos.length;
+ }
+ }
+
+ processedCount += batch.docInfos.length;
+ onProgress?.(
+ processedCount,
+ total,
+ batch.docInfos[batch.docInfos.length - 1]?.doc.title ?? "",
);
}
}
+ // Semaphore-based concurrent embedding: up to `concurrency` embedBatch calls in flight at once.
+ await new Promise((resolve) => {
+ if (batches.length === 0) {
+ resolve();
+ return;
+ }
+
+ let activeCount = 0;
+ let scheduleIdx = 0;
+
+ function scheduleNext(): void {
+ while (activeCount < concurrency && scheduleIdx < batches.length) {
+ const i = scheduleIdx++;
+ const resolved = resolveBatch(batches[i]!);
+ activeCount++;
+
+ // Wrap in try/catch so synchronous throws from embedBatch don't leave
+ // the surrounding Promise permanently pending.
+ let embedPromise: Promise;
+ if (resolved.allChunks.length > 0) {
+ try {
+ embedPromise = provider.embedBatch(resolved.allChunks);
+ } catch (err) {
+ embedPromise = Promise.reject(err instanceof Error ? err : new Error(String(err)));
+ }
+ } else {
+ embedPromise = Promise.resolve([] as number[][]);
+ }
+
+ embedPromise
+ .then((embeddings) => {
+ embedResults[i] = { resolved, embeddings, success: true };
+ })
+ .catch((err) => {
+ log.warn(
+ { err, pack: pack.name, batchIndex: i },
+ "Failed to embed batch, skipping these documents",
+ );
+ embedResults[i] = { resolved, embeddings: [], success: false };
+ })
+ .finally(() => {
+ activeCount--;
+ flushInserts();
+ if (scheduleIdx < batches.length) {
+ scheduleNext();
+ } else if (activeCount === 0) {
+ resolve();
+ }
+ });
+ }
+ }
+
+ scheduleNext();
+ });
+
// Update doc count
db.prepare("UPDATE packs SET doc_count = ? WHERE name = ?").run(installed, pack.name);
- log.info({ pack: pack.name, installed }, "Pack installed");
- return { packName: pack.name, documentsInstalled: installed, alreadyInstalled: false };
+ log.info({ pack: pack.name, installed, errors }, "Pack installed");
+ return { packName: pack.name, documentsInstalled: installed, alreadyInstalled: false, errors };
}
/** Remove a pack and all its associated documents. */
@@ -305,8 +569,11 @@ export function removePack(db: Database.Database, packName: string): void {
db.prepare(
"DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)",
).run(id);
- } catch {
- // chunk_embeddings table may not exist
+ } catch (err) {
+ log.debug(
+ { err, documentId: id },
+ "chunk_embeddings cleanup skipped (table may not exist)",
+ );
}
db.prepare("DELETE FROM documents WHERE id = ?").run(id);
}
@@ -387,9 +654,235 @@ export function createPack(db: Database.Database, options: CreatePackOptions): K
};
if (options.outputPath) {
- writeFileSync(options.outputPath, JSON.stringify(pack, null, 2), "utf-8");
+ writePackFile(options.outputPath, pack);
log.info({ outputPath: options.outputPath, docCount: documents.length }, "Pack file created");
}
return pack;
}
+
+// ---------------------------------------------------------------------------
+// Create pack from filesystem / URL sources (no database required)
+// ---------------------------------------------------------------------------
+
+/** Simple glob-style pattern matching (supports * and ** wildcards). */
+function matchesExcludePattern(relativePath: string, pattern: string): boolean {
+ // Escape regex special chars except * and **
+ const escaped = pattern
+ .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+ .replace(/\*\*/g, "\0")
+ .replace(/\*/g, "[^/]*")
+ .replace(/\0/g, ".*");
+ return new RegExp(`^${escaped}$`).test(relativePath);
+}
+
+/** Recursively collect files from a directory. */
+function collectFiles(
+ dir: string,
+ rootDir: string,
+ recursive: boolean,
+ extensions: Set,
+ excludePatterns: string[],
+): string[] {
+ const results: string[] = [];
+ let entries: string[];
+ try {
+ entries = readdirSync(dir);
+ } catch (err) {
+ throw new ValidationError(
+ `Cannot read directory "${dir}": ${err instanceof Error ? err.message : String(err)}`,
+ );
+ }
+
+ for (const entry of entries) {
+ const fullPath = pathJoin(dir, entry);
+ const rel = relative(rootDir, fullPath);
+
+ // Check exclude patterns
+ if (excludePatterns.some((p) => matchesExcludePattern(rel, p))) {
+ continue;
+ }
+
+ let stat;
+ try {
+ stat = statSync(fullPath);
+ } catch {
+ continue; // Skip unreadable entries
+ }
+
+ if (stat.isDirectory()) {
+ if (recursive) {
+ results.push(...collectFiles(fullPath, rootDir, recursive, extensions, excludePatterns));
+ }
+ } else if (stat.isFile()) {
+ const ext = extname(fullPath).toLowerCase();
+ if (extensions.has(ext)) {
+ results.push(fullPath);
+ }
+ }
+ }
+
+ return results;
+}
+
+function isUrl(value: string): boolean {
+ return value.startsWith("http://") || value.startsWith("https://");
+}
+
+/** Create a pack directly from filesystem paths and/or URLs (no database needed). */
+export async function createPackFromSource(
+ options: CreatePackFromSourceOptions,
+): Promise {
+ const log = getLogger();
+
+ if (!options.name.trim()) {
+ throw new ValidationError("Pack name is required");
+ }
+ if (options.from.length === 0) {
+ throw new ValidationError("At least one --from source is required");
+ }
+
+ const allSupported = getSupportedExtensions();
+ const extensions = new Set(
+ options.extensions?.map((e) => (e.startsWith(".") ? e.toLowerCase() : `.${e.toLowerCase()}`)) ??
+ allSupported,
+ );
+ const excludePatterns = options.exclude ?? [];
+ const recursive = options.recursive ?? true;
+
+ const documents: PackDocument[] = [];
+ const errors: Array<{ source: string; error: string }> = [];
+
+ // Separate URLs from file paths
+ const urls: string[] = [];
+ const fileSources: string[] = [];
+ for (const src of options.from) {
+ if (isUrl(src)) {
+ urls.push(src);
+ } else {
+ fileSources.push(src);
+ }
+ }
+
+ // Collect all files from filesystem sources
+ const allFiles: string[] = [];
+ for (const src of fileSources) {
+ const resolved = pathResolve(src);
+ let stat;
+ try {
+ stat = statSync(resolved);
+ } catch (err) {
+ throw new ValidationError(
+ `Source path "${src}" does not exist or is not accessible: ${err instanceof Error ? err.message : String(err)}`,
+ );
+ }
+
+ if (stat.isDirectory()) {
+ allFiles.push(...collectFiles(resolved, resolved, recursive, extensions, excludePatterns));
+ } else if (stat.isFile()) {
+ allFiles.push(resolved);
+ } else {
+ throw new ValidationError(`Source path "${src}" is not a file or directory`);
+ }
+ }
+
+ // Parse filesystem files
+ const totalCount = allFiles.length + urls.length;
+ for (let i = 0; i < allFiles.length; i++) {
+ const filePath = allFiles[i]!;
+ options.onProgress?.({ file: filePath, index: i, total: totalCount });
+
+ const parser = getParserForFile(filePath);
+ if (!parser) {
+ log.debug({ file: filePath }, "No parser for file, skipping");
+ continue;
+ }
+
+ try {
+ const buffer = readFileSync(filePath);
+ const content = await parser.parse(buffer);
+ const trimmed = content.trimEnd();
+ if (trimmed.length === 0) {
+ log.debug({ file: filePath }, "Empty content after parsing, skipping");
+ continue;
+ }
+
+ const title = basename(filePath).replace(/\.[^.]+$/, "");
+ const tags = suggestTagsFromText(title, trimmed);
+ documents.push({
+ title,
+ content: trimmed,
+ source: pathToFileURL(filePath).href,
+ tags,
+ });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ log.warn({ file: filePath, err: msg }, "Failed to parse file, skipping");
+ errors.push({ source: filePath, error: msg });
+ }
+ }
+
+ // Fetch URLs
+ for (let i = 0; i < urls.length; i++) {
+ const url = urls[i]!;
+ options.onProgress?.({ file: url, index: allFiles.length + i, total: totalCount });
+
+ try {
+ const fetched = await fetchAndConvert(url);
+ if (!fetched.content.trim()) {
+ log.debug({ url }, "Empty content from URL, skipping");
+ continue;
+ }
+
+ const tags = suggestTagsFromText(fetched.title, fetched.content.trimEnd());
+ documents.push({
+ title: fetched.title,
+ content: fetched.content.trimEnd(),
+ source: url,
+ tags,
+ });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ log.warn({ url, err: msg }, "Failed to fetch URL, skipping");
+ errors.push({ source: url, error: msg });
+ }
+ }
+
+ if (documents.length === 0) {
+ const detail =
+ errors.length > 0
+ ? ` (${errors.length} source(s) failed: ${errors.map((e) => e.source).join(", ")})`
+ : "";
+ throw new ValidationError(`No documents could be created from the provided sources${detail}`);
+ }
+
+ if (errors.length > 0) {
+ log.warn({ errorCount: errors.length, errors }, "Some sources failed during pack creation");
+ }
+
+ const pack: KnowledgePack = {
+ name: options.name,
+ version: options.version ?? "1.0.0",
+ description: options.description ?? `Knowledge pack: ${options.name}`,
+ documents,
+ metadata: {
+ author: options.author ?? "libscope",
+ license: options.license ?? "MIT",
+ createdAt: new Date().toISOString(),
+ },
+ };
+
+ if (options.outputPath) {
+ writePackFile(options.outputPath, pack);
+ log.info(
+ { outputPath: options.outputPath, docCount: documents.length },
+ "Pack file created from source",
+ );
+ }
+
+ log.info(
+ { name: pack.name, docCount: documents.length, errorCount: errors.length },
+ "Pack created from source",
+ );
+ return pack;
+}
diff --git a/src/core/parsers/epub.ts b/src/core/parsers/epub.ts
new file mode 100644
index 0000000..b19a4c0
--- /dev/null
+++ b/src/core/parsers/epub.ts
@@ -0,0 +1,65 @@
+import { writeFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { randomUUID } from "node:crypto";
+import type { DocumentParser } from "./index.js";
+import { ValidationError } from "../../errors.js";
+
+/** Parses EPUB files using epub2. */
+export class EpubParser implements DocumentParser {
+ readonly extensions = [".epub"];
+
+ async parse(content: Buffer): Promise {
+ let EPub: typeof import("epub2").EPub;
+ try {
+ const mod = await import("epub2");
+ EPub = mod.EPub;
+ } catch (err) {
+ throw new ValidationError(
+ 'EPUB parsing requires the "epub2" package. Install it with: npm install epub2',
+ err,
+ );
+ }
+
+ // epub2 needs a file path, so write buffer to a temp file
+ const tmpPath = join(tmpdir(), `libscope-epub-${randomUUID()}.epub`);
+ try {
+ writeFileSync(tmpPath, content);
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+ const epub = await EPub.createAsync(tmpPath);
+
+ const chapters: string[] = [];
+ for (const item of (epub as { flow: Array<{ id?: string }> }).flow) {
+ if (!item.id) continue;
+ try {
+ const getChapter = (epub as Record Promise>)
+ .getChapterAsync;
+ if (!getChapter) continue;
+ const html: string = await getChapter.call(epub, item.id);
+ // Strip HTML tags to get plain text
+ const text = html
+ .replace(/<[^>]+>/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+ if (text.length > 0) {
+ chapters.push(text);
+ }
+ } catch {
+ // Skip unreadable chapters
+ }
+ }
+
+ if (chapters.length === 0) {
+ throw new ValidationError("EPUB file contains no readable chapters");
+ }
+
+ return chapters.join("\n\n");
+ } finally {
+ try {
+ unlinkSync(tmpPath);
+ } catch {
+ /* ignore cleanup errors */
+ }
+ }
+ }
+}
diff --git a/src/core/parsers/html.ts b/src/core/parsers/html.ts
new file mode 100644
index 0000000..1e9d4a1
--- /dev/null
+++ b/src/core/parsers/html.ts
@@ -0,0 +1,23 @@
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import { ValidationError } from "../../errors.js";
+import type { DocumentParser } from "./index.js";
+
+const nhm = new NodeHtmlMarkdown({ ignore: ["script", "style", "nav"] });
+
+/** Parser for HTML files — converts to Markdown via node-html-markdown. */
+export class HtmlParser implements DocumentParser {
+ readonly extensions = [".html", ".htm"];
+
+ parse(content: Buffer): Promise {
+ try {
+ const html = content.toString("utf-8");
+ const markdown = nhm.translate(html);
+
+ // Collapse excessive blank lines left by ignored elements
+ return Promise.resolve(markdown.replace(/\n{3,}/g, "\n\n").trimEnd());
+ } catch (err: unknown) {
+ const message = err instanceof Error ? err.message : "Unknown HTML parsing error";
+ throw new ValidationError(`Failed to parse HTML: ${message}`);
+ }
+ }
+}
diff --git a/src/core/parsers/index.ts b/src/core/parsers/index.ts
index 35e43ff..3f47b99 100644
--- a/src/core/parsers/index.ts
+++ b/src/core/parsers/index.ts
@@ -6,6 +6,9 @@ import { YamlParser } from "./yaml.js";
import { CsvParser } from "./csv.js";
import { PdfParser } from "./pdf.js";
import { WordParser } from "./word.js";
+import { HtmlParser } from "./html.js";
+import { EpubParser } from "./epub.js";
+import { PptxParser } from "./pptx.js";
/** Interface for document format parsers. */
export interface DocumentParser {
@@ -23,6 +26,9 @@ const parsers: DocumentParser[] = [
new CsvParser(),
new PdfParser(),
new WordParser(),
+ new HtmlParser(),
+ new EpubParser(),
+ new PptxParser(),
];
const extensionMap = new Map();
diff --git a/src/core/parsers/pptx.ts b/src/core/parsers/pptx.ts
new file mode 100644
index 0000000..0daa6b7
--- /dev/null
+++ b/src/core/parsers/pptx.ts
@@ -0,0 +1,58 @@
+import type { DocumentParser } from "./index.js";
+import { ValidationError } from "../../errors.js";
+
+/** Parses PPTX files using pizzip. */
+export class PptxParser implements DocumentParser {
+ readonly extensions = [".pptx", ".ppt"];
+
+ async parse(content: Buffer): Promise {
+ let PizZip: typeof import("pizzip").default;
+ try {
+ const mod = await import("pizzip");
+ PizZip = mod.default;
+ } catch (err) {
+ throw new ValidationError(
+ 'PPTX parsing requires the "pizzip" package. Install it with: npm install pizzip',
+ err,
+ );
+ }
+
+ let zip: InstanceType;
+ try {
+ zip = new PizZip(content);
+ } catch {
+ return ""; // binary .ppt format not supported
+ }
+
+ const slides: string[] = [];
+ let slideNum = 1;
+
+ // PPTX slides are at ppt/slides/slide1.xml, slide2.xml, etc.
+ while (true) {
+ const slideFile = zip.file(`ppt/slides/slide${slideNum}.xml`);
+ if (!slideFile) break;
+
+ const xml = slideFile.asText();
+ // Extract text from elements
+ const texts: string[] = [];
+ const regex = /([\s\S]*?)<\/a:t>/g;
+ let match: RegExpExecArray | null;
+ while ((match = regex.exec(xml)) !== null) {
+ const text = match[1]?.trim();
+ if (text) texts.push(text);
+ }
+
+ if (texts.length > 0) {
+ slides.push(`--- Slide ${slideNum} ---\n${texts.join(" ")}`);
+ }
+
+ slideNum++;
+ }
+
+ if (slides.length === 0) {
+ throw new ValidationError("PPTX file contains no readable slides");
+ }
+
+ return slides.join("\n\n");
+ }
+}
diff --git a/src/core/rag.ts b/src/core/rag.ts
index ecc7ed7..935f8d8 100644
--- a/src/core/rag.ts
+++ b/src/core/rag.ts
@@ -67,10 +67,16 @@ export function extractSources(results: SearchResult[]): RagSource[] {
}
interface LlmConfig {
- provider?: "openai" | "ollama";
+ provider?: "openai" | "ollama" | "anthropic" | "passthrough";
model?: string;
ollamaUrl?: string;
openaiApiKey?: string;
+ anthropicApiKey?: string;
+}
+
+/** Returns true if the config is set to passthrough mode (delegate synthesis to the calling LLM). */
+export function isPassthroughMode(config: LibScopeConfig): boolean {
+ return config.llm?.provider === "passthrough";
}
/** Create an LLM provider from config. */
@@ -84,13 +90,45 @@ export function createLlmProvider(config: LibScopeConfig): LlmProvider {
if (providerType === "ollama") {
return createOllamaProvider(config.embedding, llmConfig);
}
+ if (providerType === "anthropic") {
+ return createAnthropicProvider(llmConfig);
+ }
throw new ConfigError(
- "No LLM provider configured. Set llm.provider to 'openai' or 'ollama' in your config, " +
+ "No LLM provider configured. Set llm.provider to 'openai', 'ollama', 'anthropic', or 'passthrough' in your config, " +
"or set LIBSCOPE_LLM_PROVIDER environment variable.",
);
}
+export interface PassthroughResult {
+ contextPrompt: string;
+ sources: RagSource[];
+}
+
+/**
+ * Retrieve relevant chunks and return the formatted context prompt without calling an LLM.
+ * Used in passthrough mode so the calling LLM can synthesize the answer itself.
+ */
+export async function getContextForQuestion(
+ db: Database.Database,
+ embeddingProvider: EmbeddingProvider,
+ options: RagOptions,
+): Promise {
+ const topK = options.topK ?? 5;
+
+ const { results } = await searchDocuments(db, embeddingProvider, {
+ query: options.question,
+ topic: options.topic,
+ library: options.library,
+ limit: topK,
+ });
+
+ return {
+ contextPrompt: buildContextPrompt(options.question, results),
+ sources: extractSources(results),
+ };
+}
+
function createOpenAiProvider(
embedding: LibScopeConfig["embedding"],
llmConfig: LlmConfig | undefined,
@@ -236,6 +274,44 @@ function createOllamaProvider(
};
}
+function createAnthropicProvider(llmConfig: LlmConfig | undefined): LlmProvider {
+ const apiKey = llmConfig?.anthropicApiKey ?? process.env["ANTHROPIC_API_KEY"];
+ if (!apiKey) {
+ throw new ConfigError(
+ "Anthropic API key is required. Set llm.anthropicApiKey in config or ANTHROPIC_API_KEY env var.",
+ );
+ }
+
+ const model = llmConfig?.model ?? "claude-3-5-haiku-20241022";
+
+ return {
+ model,
+ async complete(
+ prompt: string,
+ systemPrompt?: string,
+ ): Promise<{ text: string; tokensUsed?: number | undefined }> {
+ const { default: Anthropic } = await import("@anthropic-ai/sdk");
+ const client = new Anthropic({ apiKey });
+
+ const message = await client.messages.create({
+ model,
+ max_tokens: 4096,
+ ...(systemPrompt ? { system: systemPrompt } : {}),
+ messages: [{ role: "user", content: prompt }],
+ });
+
+ const text = message.content
+ .filter((block) => block.type === "text")
+ .map((block) => ("text" in block ? block.text : ""))
+ .join("");
+
+ const tokensUsed = message.usage.input_tokens + message.usage.output_tokens;
+
+ return { text, tokensUsed };
+ },
+ };
+}
+
/** SSE event for streaming RAG responses. */
export type RagStreamEvent =
| { token: string }
diff --git a/src/core/reindex.ts b/src/core/reindex.ts
index 5ffc55a..08a9ee7 100644
--- a/src/core/reindex.ts
+++ b/src/core/reindex.ts
@@ -2,6 +2,7 @@ import type Database from "better-sqlite3";
import type { EmbeddingProvider } from "../providers/embedding.js";
import { DatabaseError } from "../errors.js";
import { getLogger } from "../logger.js";
+import { createVectorTable } from "../db/schema.js";
export interface ReindexOptions {
/** Only reindex chunks belonging to these document IDs. */
@@ -59,16 +60,12 @@ export async function reindex(
log.info({ total }, "Chunks to reindex");
- // Ensure the vector table exists for the current provider dimensions
+ // Ensure the vector table exists with the correct dimensions for this provider.
+ // Delegates to schema.createVectorTable() — single source of truth for the DDL.
try {
- db.exec(`
- CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0(
- chunk_id TEXT PRIMARY KEY,
- embedding float[${provider.dimensions}]
- );
- `);
- } catch (err: unknown) {
- log.warn({ err }, "Could not ensure vector table — continuing anyway");
+ createVectorTable(db, provider.dimensions);
+ } catch {
+ log.warn("Could not ensure vector table — continuing anyway");
}
const deleteStmt = db.prepare("DELETE FROM chunk_embeddings WHERE chunk_id = ?");
diff --git a/src/core/saved-searches.ts b/src/core/saved-searches.ts
index 8a10dd3..42b8ddf 100644
--- a/src/core/saved-searches.ts
+++ b/src/core/saved-searches.ts
@@ -4,6 +4,7 @@ import type { EmbeddingProvider } from "../providers/embedding.js";
import { searchDocuments } from "./search.js";
import type { SearchOptions, SearchResult } from "./search.js";
import { ValidationError, DocumentNotFoundError } from "../errors.js";
+import { getLogger } from "../logger.js";
export interface SavedSearch {
id: string;
@@ -28,7 +29,11 @@ interface SavedSearchRow {
function rowToSavedSearch(row: SavedSearchRow): SavedSearch {
let filters: Omit | null = null;
if (row.filters) {
- filters = JSON.parse(row.filters) as Omit;
+ try {
+ filters = JSON.parse(row.filters) as Omit;
+ } catch {
+ getLogger().warn({ id: row.id }, "Failed to parse saved search filters JSON; using null");
+ }
}
return {
id: row.id,
diff --git a/src/core/search.ts b/src/core/search.ts
index 4acb086..70d1edd 100644
--- a/src/core/search.ts
+++ b/src/core/search.ts
@@ -59,6 +59,8 @@ export interface SearchOptions {
maxChunksPerDocument?: number | undefined;
contextChunks?: number | undefined;
analyticsEnabled?: boolean | undefined;
+ /** MMR diversity factor 0–1. 0 = pure relevance, 1 = maximum diversity. */
+ diversity?: number | undefined;
}
export interface SearchResponse {
@@ -66,7 +68,7 @@ export interface SearchResponse {
totalCount: number;
}
-export type SearchMethod = "vector" | "fts5" | "keyword";
+export type SearchMethod = "vector" | "fts5" | "keyword" | "hybrid";
export interface ScoreExplanation {
method: SearchMethod;
@@ -104,6 +106,95 @@ interface ChunkRow {
chunk_index: number;
}
+// ---------------------------------------------------------------------------
+// Title boost multiplier: chunks whose document title contains any query word
+// receive this multiplicative boost to their final score.
+// ---------------------------------------------------------------------------
+const TITLE_BOOST_MULTIPLIER = 1.5;
+
+/**
+ * Check whether any query word appears in the document title (case-insensitive).
+ */
+function titleMatchesQuery(title: string, query: string): boolean {
+ const words = query.split(/\s+/).filter((w) => w.length > 0);
+ const lowerTitle = title.toLowerCase();
+ return words.some((w) => lowerTitle.includes(w.toLowerCase()));
+}
+
+// ---------------------------------------------------------------------------
+// Reciprocal Rank Fusion (RRF)
+// ---------------------------------------------------------------------------
+/** Constant k for RRF scoring – standard value from the literature. */
+const RRF_K = 60;
+
+interface RankedItem {
+ result: SearchResult;
+ /** Ranks across contributing lists (1-indexed). */
+ ranks: number[];
+}
+
+/**
+ * Merge two ranked result lists via Reciprocal Rank Fusion.
+ * Returns results sorted by fused score in descending order.
+ */
+function reciprocalRankFusion(listA: SearchResult[], listB: SearchResult[]): SearchResult[] {
+ const map = new Map();
+
+ for (let i = 0; i < listA.length; i++) {
+ const r = listA[i]!;
+ const key = r.chunkId;
+ const existing = map.get(key);
+ if (existing) {
+ existing.ranks.push(i + 1);
+ } else {
+ map.set(key, { result: r, ranks: [i + 1] });
+ }
+ }
+
+ for (let i = 0; i < listB.length; i++) {
+ const r = listB[i]!;
+ const key = r.chunkId;
+ const existing = map.get(key);
+ if (existing) {
+ existing.ranks.push(i + 1);
+ // Prefer result with richer explanation (vector > fts5 > keyword)
+ if (
+ r.scoreExplanation.method === "vector" &&
+ existing.result.scoreExplanation.method !== "vector"
+ ) {
+ existing.result = r;
+ }
+ } else {
+ map.set(key, { result: r, ranks: [i + 1] });
+ }
+ }
+
+ const fused: Array<{ result: SearchResult; score: number }> = [];
+ for (const item of map.values()) {
+ let rrfScore = 0;
+ for (const rank of item.ranks) {
+ rrfScore += 1.0 / (RRF_K + rank);
+ }
+ const boostFactors = [...item.result.scoreExplanation.boostFactors];
+ fused.push({
+ result: {
+ ...item.result,
+ score: rrfScore,
+ scoreExplanation: {
+ method: "hybrid" as SearchMethod,
+ rawScore: rrfScore,
+ boostFactors,
+ details: `Hybrid RRF: ranks=[${item.ranks.join(",")}], score=${rrfScore.toFixed(6)}`,
+ },
+ },
+ score: rrfScore,
+ });
+ }
+
+ fused.sort((a, b) => b.score - a.score);
+ return fused.map((f) => f.result);
+}
+
/** Fetch neighboring chunks for a given chunk within its document. */
function fetchContextChunks(
db: Database.Database,
@@ -160,6 +251,143 @@ function attachContext(
});
}
+/** Apply title boost to search results whose document title matches the query. */
+function applyTitleBoost(results: SearchResult[], query: string): SearchResult[] {
+ return results.map((r) => {
+ if (titleMatchesQuery(r.title, query)) {
+ const boosted = r.score * TITLE_BOOST_MULTIPLIER;
+ return {
+ ...r,
+ score: boosted,
+ scoreExplanation: {
+ ...r.scoreExplanation,
+ boostFactors: [
+ ...r.scoreExplanation.boostFactors,
+ `title_match:x${TITLE_BOOST_MULTIPLIER}`,
+ ],
+ details: r.scoreExplanation.details + ` (title boost x${TITLE_BOOST_MULTIPLIER})`,
+ },
+ };
+ }
+ return r;
+ });
+}
+
+/**
+ * Rerank results using Maximal Marginal Relevance for diversity.
+ * @param results - Pre-sorted results (highest score first)
+ * @param diversity - 0–1 where 1 = maximum diversity
+ */
+function applyMMR(results: SearchResult[], diversity: number): SearchResult[] {
+ if (results.length <= 1) return results;
+ const lambda = 1 - Math.max(0, Math.min(diversity, 1));
+ const selected: SearchResult[] = [];
+ const remaining = [...results];
+
+ selected.push(remaining.shift()!);
+
+ while (remaining.length > 0) {
+ let bestIdx = 0;
+ let bestMmrScore = -Infinity;
+
+ for (let i = 0; i < remaining.length; i++) {
+ const candidate = remaining[i]!;
+ let maxSim = 0;
+ for (const sel of selected) {
+ const sim = 1 - Math.abs(candidate.score - sel.score);
+ if (sim > maxSim) maxSim = sim;
+ }
+ const mmrScore = lambda * candidate.score - (1 - lambda) * maxSim;
+ if (mmrScore > bestMmrScore) {
+ bestMmrScore = mmrScore;
+ bestIdx = i;
+ }
+ }
+
+ selected.push(remaining.splice(bestIdx, 1)[0]!);
+ }
+
+ return selected;
+}
+
+/** Deduplicate results by document, keeping at most N chunks per document. */
+function deduplicateByDocument(results: SearchResult[], maxPerDoc: number): SearchResult[] {
+ const countByDoc = new Map();
+ return results.filter((r) => {
+ const count = countByDoc.get(r.documentId) ?? 0;
+ if (count >= maxPerDoc) return false;
+ countByDoc.set(r.documentId, count + 1);
+ return true;
+ });
+}
+
+/** Append standard filter clauses and params to a SQL query. */
+function appendFilters(
+ sql: string,
+ params: unknown[],
+ options: SearchOptions,
+ docAlias: string,
+): string {
+ if (options.library) {
+ sql += ` AND ${docAlias}.library = ?`;
+ params.push(options.library);
+ }
+ if (options.topic) {
+ sql += ` AND ${docAlias}.topic_id = ?`;
+ params.push(options.topic);
+ }
+ if (options.version) {
+ sql += ` AND ${docAlias}.version = ?`;
+ params.push(options.version);
+ }
+ if (options.minRating !== undefined) {
+ sql += " AND avg_r.avg_rating >= ?";
+ params.push(options.minRating);
+ }
+ if (options.dateFrom) {
+ sql += ` AND ${docAlias}.created_at >= ?`;
+ params.push(options.dateFrom);
+ }
+ if (options.dateTo) {
+ sql += ` AND ${docAlias}.created_at <= ?`;
+ params.push(options.dateTo);
+ }
+ if (options.source) {
+ sql += ` AND ${docAlias}.source_type = ?`;
+ params.push(options.source);
+ }
+
+ const tagFilter = buildTagFilter(options.tags, docAlias);
+ sql += tagFilter.clause;
+ params.push(...tagFilter.params);
+
+ return sql;
+}
+
+/**
+ * Lazy totalCount: skip the expensive COUNT query when we can infer the total
+ * from the result set (offset === 0 and fewer results than the limit).
+ * Always returns a non-negative count.
+ */
+function lazyCount(
+ db: Database.Database,
+ baseSql: string,
+ baseParams: unknown[],
+ offset: number,
+ resultLen: number,
+ limit: number,
+ label: string,
+): number {
+ // If offset is 0 and we got fewer results than the limit, we know the total
+ if (offset === 0 && resultLen < limit) {
+ return resultLen;
+ }
+ return validateCountRow(
+ db.prepare(`SELECT COUNT(*) AS cnt FROM (${baseSql})`).get(...baseParams),
+ label,
+ );
+}
+
/** Perform semantic search across indexed documents. */
export async function searchDocuments(
db: Database.Database,
@@ -179,129 +407,81 @@ export async function searchDocuments(
const queryEmbedding = await provider.embed(options.query);
const vecBuffer = Buffer.from(new Float32Array(queryEmbedding).buffer);
- // Try vector search first
+ // Try hybrid search: vector + FTS5 merged via RRF
+ // Over-fetch from each source by 3x (capped) to compensate for overlap
+ // between vector and FTS5 lists — RRF deduplicates shared chunks, so the
+ // fused unique set is often smaller than the sum of both lists.
+ const overfetchFactor = 3;
+ const maxCandidateLimit = 5000;
+ const candidateLimit = Math.min((offset + limit) * overfetchFactor, maxCandidateLimit);
try {
- let sql = `
- SELECT
- candidates.chunk_id,
- candidates.distance,
- c.document_id,
- c.content AS chunk_content,
- d.title,
- d.source_type,
- d.library,
- d.version,
- d.topic_id,
- d.url,
- avg_r.avg_rating
- FROM (
- SELECT chunk_id, distance
- FROM chunk_embeddings
- WHERE embedding MATCH ?
- ORDER BY distance
- LIMIT ?
- ) candidates
- JOIN chunks c ON c.id = candidates.chunk_id
- JOIN documents d ON d.id = c.document_id
- LEFT JOIN (
- SELECT document_id, AVG(rating) AS avg_rating
- FROM ratings
- GROUP BY document_id
- ) avg_r ON avg_r.document_id = d.id
- WHERE 1=1
- `;
+ const vectorResults = vectorSearch(db, options, vecBuffer, candidateLimit, 0);
+ let ftsResults: SearchResult[] | null = null;
+ let ftsTotalCount = 0;
+
+ try {
+ const ftsResponse = fts5Search(db, options, candidateLimit, 0);
+ ftsResults = ftsResponse.results;
+ ftsTotalCount = ftsResponse.totalCount;
+ } catch {
+ // FTS5 not available — proceed with vector-only results
+ }
- const params: unknown[] = [vecBuffer, limit * 3];
+ let mergedResults: SearchResult[];
+ let searchMethod: SearchMethod;
- if (options.library) {
- sql += " AND d.library = ?";
- params.push(options.library);
+ if (ftsResults && ftsResults.length > 0) {
+ // Hybrid: merge vector + FTS5 via RRF
+ mergedResults = reciprocalRankFusion(vectorResults.results, ftsResults);
+ searchMethod = "hybrid";
+ } else {
+ mergedResults = vectorResults.results;
+ searchMethod = "vector";
}
- if (options.topic) {
- sql += " AND d.topic_id = ?";
- params.push(options.topic);
- }
- if (options.version) {
- sql += " AND d.version = ?";
- params.push(options.version);
- }
- if (options.minRating) {
- sql += " AND avg_r.avg_rating >= ?";
- params.push(options.minRating);
- }
- if (options.dateFrom) {
- sql += " AND d.created_at >= ?";
- params.push(options.dateFrom);
+
+ // Apply title boost
+ mergedResults = applyTitleBoost(mergedResults, options.query);
+ // Re-sort after boost
+ mergedResults.sort((a, b) => b.score - a.score);
+
+ // Apply MMR diversity reranking if requested
+ if (options.diversity !== undefined && options.diversity > 0) {
+ mergedResults = applyMMR(mergedResults, options.diversity);
}
- if (options.dateTo) {
- sql += " AND d.created_at <= ?";
- params.push(options.dateTo);
+
+ // Apply per-document deduplication on the full ranked list BEFORE
+ // pagination so that page sizes stay stable and later pages aren't
+ // short-changed by dedup removing items from within the page.
+ if (options.maxChunksPerDocument !== undefined && options.maxChunksPerDocument > 0) {
+ mergedResults = deduplicateByDocument(mergedResults, options.maxChunksPerDocument);
}
- if (options.source) {
- sql += " AND d.source_type = ?";
- params.push(options.source);
+
+ // Apply pagination to merged (and possibly deduped) results
+ let paginatedResults = mergedResults.slice(offset, offset + limit);
+
+ // Attach ratings post-pagination (deferred from search queries for perf)
+ if (options.minRating === undefined) {
+ paginatedResults = attachRatings(db, paginatedResults);
}
- const tagFilterVec = buildTagFilter(options.tags, "d");
- sql += tagFilterVec.clause;
- params.push(...tagFilterVec.params);
-
- // Build count query from base SQL (before adding ORDER BY/LIMIT/OFFSET)
- const baseSql = sql;
- const baseParams = [...params];
- const totalCount = validateCountRow(
- db.prepare(`SELECT COUNT(*) AS cnt FROM (${baseSql})`).get(...baseParams),
- "vector search count",
- );
-
- sql += ` ORDER BY candidates.distance LIMIT ? OFFSET ?`;
- params.push(limit);
- params.push(offset);
-
- const rows = db.prepare(sql).all(...params) as Array<{
- chunk_id: string;
- distance: number;
- document_id: string;
- chunk_content: string;
- title: string;
- source_type: string;
- library: string | null;
- version: string | null;
- topic_id: string | null;
- url: string | null;
- avg_rating: number | null;
- }>;
+ // totalCount: the fused unique set is the best lower bound we have.
+ // For hybrid searches, the true match count is at least as large as
+ // the unique items after fusion; use the max of that and each source's
+ // reported count so we never under-report.
+ const totalCount =
+ searchMethod === "hybrid"
+ ? Math.max(mergedResults.length, vectorResults.totalCount, ftsTotalCount)
+ : vectorResults.totalCount;
const response: SearchResponse = {
totalCount,
- results: rows.map((row) => {
- const similarity = 1 - row.distance;
- return {
- documentId: row.document_id,
- chunkId: row.chunk_id,
- title: row.title,
- content: row.chunk_content,
- sourceType: row.source_type,
- library: row.library,
- version: row.version,
- topicId: row.topic_id,
- url: row.url,
- score: similarity,
- avgRating: row.avg_rating,
- scoreExplanation: {
- method: "vector" as SearchMethod,
- rawScore: row.distance,
- boostFactors: [],
- details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`,
- },
- };
- }),
+ results: paginatedResults,
};
if (analyticsEnabled) {
logSearch(db, {
query: options.query,
- searchMethod: "vector",
+ searchMethod,
resultCount: response.totalCount,
latencyMs: performance.now() - startTime,
documentIds: response.results.map((r) => r.documentId),
@@ -310,18 +490,7 @@ export async function searchDocuments(
query: options.query,
resultCount: response.results.length,
topScore: response.results[0]?.score ?? null,
- searchType: "vector",
- });
- }
-
- // Deduplicate by document — keep top N chunks per document
- if (options.maxChunksPerDocument !== undefined && options.maxChunksPerDocument > 0) {
- const countByDoc = new Map();
- response.results = response.results.filter((r) => {
- const count = countByDoc.get(r.documentId) ?? 0;
- if (count >= options.maxChunksPerDocument!) return false;
- countByDoc.set(r.documentId, count + 1);
- return true;
+ searchType: searchMethod,
});
}
@@ -337,6 +506,20 @@ export async function searchDocuments(
log.warn({ err }, "Vector table missing, falling back to keyword search");
const response = keywordSearch(db, options, limit, offset);
+ // Attach ratings post-query when minRating filter is not active
+ if (options.minRating === undefined) {
+ response.results = attachRatings(db, response.results);
+ }
+
+ // Apply title boost to fallback results
+ response.results = applyTitleBoost(response.results, options.query);
+ response.results.sort((a, b) => b.score - a.score);
+
+ // Apply MMR diversity reranking if requested
+ if (options.diversity !== undefined && options.diversity > 0) {
+ response.results = applyMMR(response.results, options.diversity);
+ }
+
if (analyticsEnabled) {
const method = response.results[0]?.scoreExplanation.method ?? "keyword";
logSearch(db, {
@@ -356,13 +539,7 @@ export async function searchDocuments(
// Deduplicate by document — keep top N chunks per document
if (options.maxChunksPerDocument !== undefined && options.maxChunksPerDocument > 0) {
- const countByDoc = new Map();
- response.results = response.results.filter((r) => {
- const count = countByDoc.get(r.documentId) ?? 0;
- if (count >= options.maxChunksPerDocument!) return false;
- countByDoc.set(r.documentId, count + 1);
- return true;
- });
+ response.results = deduplicateByDocument(response.results, options.maxChunksPerDocument);
}
if (options.contextChunks) {
@@ -373,6 +550,105 @@ export async function searchDocuments(
}
}
+/** Pure vector search — returns candidates for fusion/pagination.
+ * `limit` should already account for offset (i.e. caller passes offset+limit). */
+function vectorSearch(
+ db: Database.Database,
+ options: SearchOptions,
+ vecBuffer: Buffer,
+ limit: number,
+ _offset: number,
+): SearchResponse {
+ // The caller already over-fetches; use the limit directly.
+ const annCandidateLimit = limit;
+
+ const needsRatingJoin = options.minRating !== undefined;
+
+ let sql = `
+ SELECT
+ candidates.chunk_id,
+ candidates.distance,
+ c.document_id,
+ c.content AS chunk_content,
+ d.title,
+ d.source_type,
+ d.library,
+ d.version,
+ d.topic_id,
+ d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""}
+ FROM (
+ SELECT chunk_id, distance
+ FROM chunk_embeddings
+ WHERE embedding MATCH ?
+ ORDER BY distance
+ LIMIT ?
+ ) candidates
+ JOIN chunks c ON c.id = candidates.chunk_id
+ JOIN documents d ON d.id = c.document_id${
+ needsRatingJoin
+ ? `
+ LEFT JOIN (
+ SELECT document_id, AVG(rating) AS avg_rating
+ FROM ratings
+ GROUP BY document_id
+ ) avg_r ON avg_r.document_id = d.id`
+ : ""
+ }
+ WHERE 1=1
+ `;
+
+ const params: unknown[] = [vecBuffer, annCandidateLimit];
+ sql = appendFilters(sql, params, options, "d");
+
+ sql += ` ORDER BY candidates.distance`;
+
+ const rows = db.prepare(sql).all(...params) as Array<{
+ chunk_id: string;
+ distance: number;
+ document_id: string;
+ chunk_content: string;
+ title: string;
+ source_type: string;
+ library: string | null;
+ version: string | null;
+ topic_id: string | null;
+ url: string | null;
+ avg_rating: number | null;
+ }>;
+
+ // totalCount: if we got fewer rows than the ANN candidate limit, we know
+ // the true total (all candidates survived filtering). Otherwise the real
+ // total may be larger than what we fetched — report the row count as a
+ // lower bound (exact COUNT is not feasible over an ANN index).
+ const totalCount = rows.length;
+
+ return {
+ totalCount,
+ results: rows.map((row) => {
+ const similarity = 1 - row.distance;
+ return {
+ documentId: row.document_id,
+ chunkId: row.chunk_id,
+ title: row.title,
+ content: row.chunk_content,
+ sourceType: row.source_type,
+ library: row.library,
+ version: row.version,
+ topicId: row.topic_id,
+ url: row.url,
+ score: similarity,
+ avgRating: needsRatingJoin ? row.avg_rating : null,
+ scoreExplanation: {
+ method: "vector" as SearchMethod,
+ rawScore: row.distance,
+ boostFactors: [],
+ details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`,
+ },
+ };
+ }),
+ };
+}
+
/** Fallback keyword search when vector search is unavailable.
* Tries FTS5 first, falls back to LIKE search. */
function keywordSearch(
@@ -393,6 +669,8 @@ function keywordSearch(
const words = options.query.split(/\s+/).filter((w) => w.length > 0);
if (words.length === 0) return { results: [], totalCount: 0 };
+ const needsRatingJoin = options.minRating !== undefined;
+
const likeConditions = words.map(() => "c.content LIKE ? ESCAPE '\\'").join(" OR ");
const params: unknown[] = words.map((w) => `%${escapeLikePattern(w)}%`);
@@ -406,58 +684,26 @@ function keywordSearch(
d.library,
d.version,
d.topic_id,
- d.url,
- avg_r.avg_rating
+ d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""}
FROM chunks c
- JOIN documents d ON d.id = c.document_id
+ JOIN documents d ON d.id = c.document_id${
+ needsRatingJoin
+ ? `
LEFT JOIN (
SELECT document_id, AVG(rating) AS avg_rating
FROM ratings
GROUP BY document_id
- ) avg_r ON avg_r.document_id = d.id
+ ) avg_r ON avg_r.document_id = d.id`
+ : ""
+ }
WHERE (${likeConditions})
`;
- if (options.library) {
- sql += " AND d.library = ?";
- params.push(options.library);
- }
- if (options.topic) {
- sql += " AND d.topic_id = ?";
- params.push(options.topic);
- }
- if (options.version) {
- sql += " AND d.version = ?";
- params.push(options.version);
- }
- if (options.minRating) {
- sql += " AND (SELECT AVG(r.rating) FROM ratings r WHERE r.document_id = d.id) >= ?";
- params.push(options.minRating);
- }
- if (options.dateFrom) {
- sql += " AND d.created_at >= ?";
- params.push(options.dateFrom);
- }
- if (options.dateTo) {
- sql += " AND d.created_at <= ?";
- params.push(options.dateTo);
- }
- if (options.source) {
- sql += " AND d.source_type = ?";
- params.push(options.source);
- }
+ sql = appendFilters(sql, params, options, "d");
- const tagFilterKw = buildTagFilter(options.tags, "d");
- sql += tagFilterKw.clause;
- params.push(...tagFilterKw.params);
-
- // Build count query from base SQL (before adding LIMIT/OFFSET)
+ // Lazy count: avoid expensive COUNT when not needed
const baseSql = sql;
const baseParams = [...params];
- const totalCount = validateCountRow(
- db.prepare(`SELECT COUNT(*) AS cnt FROM (${baseSql})`).get(...baseParams),
- "keyword search count",
- );
sql += " LIMIT ? OFFSET ?";
params.push(limit);
@@ -476,6 +722,16 @@ function keywordSearch(
avg_rating: number | null;
}>;
+ const totalCount = lazyCount(
+ db,
+ baseSql,
+ baseParams,
+ offset,
+ rows.length,
+ limit,
+ "keyword search count",
+ );
+
return {
totalCount,
results: rows.map((row, index) => {
@@ -491,7 +747,7 @@ function keywordSearch(
topicId: row.topic_id,
url: row.url,
score: rankScore,
- avgRating: row.avg_rating,
+ avgRating: needsRatingJoin ? row.avg_rating : null,
scoreExplanation: {
method: "keyword" as SearchMethod,
rawScore: rankScore,
@@ -503,18 +759,61 @@ function keywordSearch(
};
}
-/** FTS5-based full-text search with BM25 ranking. */
+/** Strip FTS5 special syntax from a single query word before quoting. */
+function sanitizeFtsWord(word: string): string {
+ // Strip column-filter syntax (e.g. "chunk_id:foo" → "foo")
+ const colonIdx = word.indexOf(":");
+ if (colonIdx !== -1) {
+ word = word.slice(colonIdx + 1);
+ }
+ // Strip prefix/suffix wildcards using index scan to avoid ReDoS
+ let start = 0;
+ while (start < word.length && word[start] === "*") start++;
+ let end = word.length;
+ while (end > start && word[end - 1] === "*") end--;
+ word = word.slice(start, end);
+ // If the remaining word is a standalone FTS5 operator, return empty
+ if (/^(NEAR|AND|OR|NOT)$/i.test(word)) {
+ return "";
+ }
+ return word;
+}
+
+/** Fetch avg ratings for a small set of documents and attach to results. */
+function attachRatings(db: Database.Database, results: SearchResult[]): SearchResult[] {
+ if (results.length === 0) return results;
+ const ids = [...new Set(results.map((r) => r.documentId))];
+ const placeholders = ids.map(() => "?").join(", ");
+ const rows = db
+ .prepare(
+ `SELECT document_id, AVG(rating) AS avg_rating
+ FROM ratings
+ WHERE document_id IN (${placeholders})
+ GROUP BY document_id`,
+ )
+ .all(...ids) as Array<{ document_id: string; avg_rating: number | null }>;
+ const ratingMap = new Map(rows.map((r) => [r.document_id, r.avg_rating]));
+ return results.map((r) => ({ ...r, avgRating: ratingMap.get(r.documentId) ?? null }));
+}
+
+/** FTS5-based full-text search with BM25 ranking. Uses AND logic by default. */
function fts5Search(
db: Database.Database,
options: SearchOptions,
limit: number,
offset: number,
): SearchResponse {
- // Escape FTS5 query: wrap each word in quotes for safety
- const words = options.query.split(/\s+/).filter((w) => w.length > 0);
+ // Sanitize and escape FTS5 query: strip dangerous syntax, wrap each word in quotes.
+ // AND-by-default: require all terms to match for better precision.
+ const words = options.query
+ .split(/\s+/)
+ .map((w) => sanitizeFtsWord(w))
+ .filter((w) => w.length > 0);
if (words.length === 0) return { results: [], totalCount: 0 };
- const ftsQuery = words.map((w) => `"${w.replace(/"/g, '""')}"`).join(" OR ");
+ const needsRatingJoin = options.minRating !== undefined;
+
+ const ftsQuery = words.map((w) => `"${w.replace(/"/g, '""')}"`).join(" AND ");
const params: unknown[] = [ftsQuery];
let sql = `
@@ -528,64 +827,32 @@ function fts5Search(
d.version,
d.topic_id,
d.url,
- rank AS fts_rank,
- avg_r.avg_rating
+ rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""}
FROM chunks_fts f
- JOIN documents d ON d.id = f.document_id
+ JOIN documents d ON d.id = f.document_id${
+ needsRatingJoin
+ ? `
LEFT JOIN (
SELECT document_id, AVG(rating) AS avg_rating
FROM ratings
GROUP BY document_id
- ) avg_r ON avg_r.document_id = d.id
+ ) avg_r ON avg_r.document_id = d.id`
+ : ""
+ }
WHERE chunks_fts MATCH ?
`;
- if (options.library) {
- sql += " AND d.library = ?";
- params.push(options.library);
- }
- if (options.topic) {
- sql += " AND d.topic_id = ?";
- params.push(options.topic);
- }
- if (options.version) {
- sql += " AND d.version = ?";
- params.push(options.version);
- }
- if (options.minRating) {
- sql += " AND (SELECT AVG(r.rating) FROM ratings r WHERE r.document_id = d.id) >= ?";
- params.push(options.minRating);
- }
- if (options.dateFrom) {
- sql += " AND d.created_at >= ?";
- params.push(options.dateFrom);
- }
- if (options.dateTo) {
- sql += " AND d.created_at <= ?";
- params.push(options.dateTo);
- }
- if (options.source) {
- sql += " AND d.source_type = ?";
- params.push(options.source);
- }
-
- const tagFilterFts = buildTagFilter(options.tags, "d");
- sql += tagFilterFts.clause;
- params.push(...tagFilterFts.params);
+ sql = appendFilters(sql, params, options, "d");
- // Build count query from base SQL (before adding ORDER BY/LIMIT/OFFSET)
- const baseSql = sql;
- const baseParams = [...params];
- const totalCount = validateCountRow(
- db.prepare(`SELECT COUNT(*) AS cnt FROM (${baseSql})`).get(...baseParams),
- "FTS5 search count",
- );
+ // Lazy count – may be updated if OR fallback is used
+ let baseSql = sql;
+ let baseParams = [...params];
sql += " ORDER BY rank LIMIT ? OFFSET ?";
params.push(limit);
params.push(offset);
- const rows = db.prepare(sql).all(...params) as Array<{
+ let rows = db.prepare(sql).all(...params) as Array<{
chunk_id: string;
document_id: string;
chunk_content: string;
@@ -599,6 +866,58 @@ function fts5Search(
avg_rating: number | null;
}>;
+ // If AND returned nothing, retry with OR for recall
+ if (rows.length === 0 && words.length > 1) {
+ const orQuery = words.map((w) => `"${w.replace(/"/g, '""')}"`).join(" OR ");
+ const orParams: unknown[] = [orQuery];
+ let orSql = `
+ SELECT
+ f.chunk_id,
+ f.document_id,
+ f.content AS chunk_content,
+ d.title,
+ d.source_type,
+ d.library,
+ d.version,
+ d.topic_id,
+ d.url,
+ rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""}
+ FROM chunks_fts f
+ JOIN documents d ON d.id = f.document_id${
+ needsRatingJoin
+ ? `
+ LEFT JOIN (
+ SELECT document_id, AVG(rating) AS avg_rating
+ FROM ratings
+ GROUP BY document_id
+ ) avg_r ON avg_r.document_id = d.id`
+ : ""
+ }
+ WHERE chunks_fts MATCH ?
+ `;
+ orSql = appendFilters(orSql, orParams, options, "d");
+
+ // Update count base to use OR query
+ baseSql = orSql;
+ baseParams = [...orParams];
+
+ orSql += " ORDER BY rank LIMIT ? OFFSET ?";
+ orParams.push(limit);
+ orParams.push(offset);
+
+ rows = db.prepare(orSql).all(...orParams) as typeof rows;
+ }
+
+ const totalCount = lazyCount(
+ db,
+ baseSql,
+ baseParams,
+ offset,
+ rows.length,
+ limit,
+ "FTS5 search count",
+ );
+
return {
totalCount,
results: rows.map((row) => {
@@ -614,7 +933,7 @@ function fts5Search(
topicId: row.topic_id,
url: row.url,
score: bm25Score,
- avgRating: row.avg_rating,
+ avgRating: needsRatingJoin ? row.avg_rating : null,
scoreExplanation: {
method: "fts5" as SearchMethod,
rawScore: row.fts_rank,
diff --git a/src/core/spider.ts b/src/core/spider.ts
new file mode 100644
index 0000000..44dd6a1
--- /dev/null
+++ b/src/core/spider.ts
@@ -0,0 +1,478 @@
+/**
+ * URL spider engine — BFS crawl from a seed URL with configurable depth, page, domain, and path limits.
+ *
+ * Safety guarantees:
+ * - All URLs are SSRF-validated via fetchRaw() before fetching
+ * - Hard caps on pages (200) and depth (5) that cannot be overridden by callers
+ * - Total wall-clock timeout of 10 minutes aborts the crawl
+ * - robots.txt is fetched once per origin and its Disallow rules are honoured
+ * - Private/internal IPs are blocked by the underlying url-fetcher
+ */
+
+import { getLogger } from "../logger.js";
+import { FetchError } from "../errors.js";
+import { fetchRaw, type FetchOptions } from "./url-fetcher.js";
+import { extractLinks } from "./link-extractor.js";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+
+// ── Hard limits that callers cannot override ────────────────────────────────
+
+const HARD_MAX_PAGES = 200;
+const HARD_MAX_DEPTH = 5;
+/** Total spider wall-clock timeout in ms (10 minutes). */
+const HARD_TOTAL_TIMEOUT_MS = 10 * 60 * 1000;
+/** Default delay between requests in ms (1 second). */
+const DEFAULT_REQUEST_DELAY_MS = 1_000;
+
+// ── Public types ─────────────────────────────────────────────────────────────
+
+export interface SpiderOptions {
+ /** Maximum total pages to index (default: 25, hard cap: 200). */
+ maxPages?: number;
+ /** Maximum hop depth from the seed URL (default: 2, hard cap: 5). 0 = seed only. */
+ maxDepth?: number;
+ /** Only follow links that share the same hostname as the seed (default: true). */
+ sameDomain?: boolean;
+ /** Only follow links whose path starts with this prefix (e.g. "/docs/"). */
+ pathPrefix?: string;
+ /** Glob-style patterns for URLs to skip (matched against full URL string). */
+ excludePatterns?: string[];
+ /** Milliseconds to wait between requests (default: 1000). */
+ requestDelay?: number;
+ /** Passed through to fetchRaw for each page request. */
+ fetchOptions?: Pick<
+ FetchOptions,
+ "allowPrivateUrls" | "allowSelfSignedCerts" | "timeout" | "maxBodySize"
+ >;
+}
+
+export interface SpiderResult {
+ url: string;
+ title: string;
+ content: string;
+ depth: number;
+}
+
+export interface SpiderStats {
+ /** Pages successfully fetched and yielded to the caller (caller decides whether to index). */
+ pagesFetched: number;
+ pagesCrawled: number;
+ pagesSkipped: number;
+ errors: Array<{ url: string; error: string }>;
+ abortReason?: "maxPages" | "timeout";
+}
+
+// ── robots.txt parsing ───────────────────────────────────────────────────────
+
+/** Fetch robots.txt for an origin, capping the timeout at 10 s regardless of caller options. */
+async function fetchRobotsTxt(
+ origin: string,
+ fetchOptions?: SpiderOptions["fetchOptions"],
+): Promise> {
+ const robotsUrl = origin + "/robots.txt";
+ // Cap robots.txt timeout: use caller's timeout only if shorter than our hard cap.
+ const effectiveTimeout =
+ fetchOptions?.timeout !== undefined && Number.isFinite(fetchOptions.timeout)
+ ? Math.min(fetchOptions.timeout, 10_000)
+ : 10_000;
+ try {
+ const raw = await fetchRaw(robotsUrl, { ...fetchOptions, timeout: effectiveTimeout });
+ return parseRobotsTxt(raw.body);
+ } catch {
+ // robots.txt missing or inaccessible — no restrictions
+ return new Set();
+ }
+}
+
+/**
+ * Parse robots.txt and return Disallow path prefixes that apply to our agent.
+ *
+ * Implements proper UA precedence: if any group explicitly names "libscope",
+ * only those groups apply (ignoring wildcard). Otherwise wildcard groups apply.
+ * This matches the robots.txt spec — a specific UA rule overrides the wildcard.
+ */
+function parseRobotsTxt(text: string): Set {
+ type RobotsGroup = { agents: string[]; disallows: string[] };
+ const groups: RobotsGroup[] = [];
+ let current: RobotsGroup | null = null;
+
+ for (const raw of text.split(/\r?\n/)) {
+ const line = raw.trim();
+ if (line.startsWith("#") || line.length === 0) continue;
+
+ const lower = line.toLowerCase();
+ if (lower.startsWith("user-agent:")) {
+ const agent = line.slice("user-agent:".length).trim();
+ // Start a new group only if current has already collected Disallow lines
+ if (current === null || current.disallows.length > 0) {
+ current = { agents: [], disallows: [] };
+ groups.push(current);
+ }
+ current.agents.push(agent.toLowerCase());
+ } else if (lower.startsWith("disallow:") && current !== null) {
+ const path = line.slice("disallow:".length).trim();
+ if (path.length > 0) current.disallows.push(path);
+ }
+ }
+
+ // Prefer explicit "libscope" group over the wildcard group
+ const libscopeGroups = groups.filter((g) => g.agents.includes("libscope"));
+ const selected =
+ libscopeGroups.length > 0 ? libscopeGroups : groups.filter((g) => g.agents.includes("*"));
+
+ const disallowed = new Set();
+ for (const group of selected) {
+ for (const path of group.disallows) disallowed.add(path);
+ }
+ return disallowed;
+}
+
+function isDisallowedByRobots(url: string, disallowed: Set): boolean {
+ if (disallowed.size === 0) return false;
+ let pathname: string;
+ try {
+ pathname = new URL(url).pathname;
+ } catch {
+ return false;
+ }
+ for (const prefix of disallowed) {
+ if (pathname.startsWith(prefix)) return true;
+ }
+ return false;
+}
+
+// ── Wildcard/glob pattern matching ──────────────────────────────────────────
+
+const REGEX_SPECIAL = new Set([".", "+", "^", "$", "{", "}", "(", ")", "|", "[", "]", "\\"]);
+
+// Match a URL against a simple glob pattern.
+// Both * and ** match any sequence of characters including path separators.
+// Matching is case-insensitive and applied to the full URL string.
+function matchesGlob(url: string, pattern: string): boolean {
+ let regexStr = "^";
+ let i = 0;
+ while (i < pattern.length) {
+ if (pattern[i] === "*" && pattern[i + 1] === "*") {
+ regexStr += ".*";
+ i += 2;
+ if (pattern[i] === "/") i++; // skip optional trailing slash after **
+ } else if (pattern[i] === "*") {
+ regexStr += ".*"; // * also matches / in URL context
+ i++;
+ } else {
+ const ch = pattern[i]!;
+ // Escape chars that are special in regex
+ if (REGEX_SPECIAL.has(ch)) {
+ regexStr += "\\" + ch;
+ } else {
+ regexStr += ch;
+ }
+ i++;
+ }
+ }
+ regexStr += "$";
+ try {
+ return new RegExp(regexStr, "i").test(url);
+ } catch {
+ return false;
+ }
+}
+
+function isExcluded(url: string, patterns: string[]): boolean {
+ return patterns.some((p) => matchesGlob(url, p));
+}
+
+// ── Domain / path filtering ──────────────────────────────────────────────────
+
+function isSameDomain(url: string, seedHostname: string): boolean {
+ try {
+ const parsed = new URL(url);
+ const host = parsed.hostname.toLowerCase();
+ const seed = seedHostname.toLowerCase();
+ // Allow exact match or subdomain match (e.g. docs.example.com vs example.com)
+ return host === seed || host.endsWith("." + seed);
+ } catch {
+ return false;
+ }
+}
+
+function hasPathPrefix(url: string, prefix: string): boolean {
+ if (!prefix) return true;
+ try {
+ return new URL(url).pathname.startsWith(prefix);
+ } catch {
+ return false;
+ }
+}
+
+// ── HTML → markdown (reuse url-fetcher's approach) ──────────────────────────
+
+function htmlToMarkdown(html: string): string {
+ return NodeHtmlMarkdown.translate(html);
+}
+
+/**
+ * Remove all HTML tags from a string using indexOf-based scanning.
+ * Handles tags that span multiple lines and tags with > inside attribute values.
+ * This avoids regex-based tag stripping which can be bypassed by newlines in tags.
+ */
+function stripTags(input: string): string {
+ let result = "";
+ let pos = 0;
+ while (pos < input.length) {
+ const open = input.indexOf("<", pos);
+ if (open === -1) {
+ result += input.slice(pos);
+ break;
+ }
+ result += input.slice(pos, open);
+ // Scan for the closing > of this tag, respecting quoted attribute values
+ let i = open + 1;
+ while (i < input.length) {
+ const ch = input[i];
+ if (ch === ">") {
+ i++;
+ break;
+ }
+ // Skip quoted attribute values so > inside them doesn't end the tag early
+ if (ch === '"' || ch === "'") {
+ const close = input.indexOf(ch, i + 1);
+ i = close === -1 ? input.length : close + 1;
+ } else {
+ i++;
+ }
+ }
+ pos = i;
+ }
+ // Collapse whitespace left behind by removed tags
+ return result.replace(/\s+/g, " ");
+}
+
+function extractTitle(html: string, url: string): string {
+ // Try tag
+ const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html);
+ if (match?.[1]) return match[1].trim();
+ // Try first <h1>
+ const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html);
+ if (h1?.[1]) {
+ return stripTags(h1[1]).trim();
+ }
+ // Fall back to URL path
+ try {
+ const parsed = new URL(url);
+ const path = parsed.pathname.replace(/\/$/, "");
+ const last = path.split("/").pop();
+ if (last) return last.replace(/[-_]/g, " ").replace(/\.\w+$/, "");
+ return parsed.hostname;
+ } catch {
+ return url;
+ }
+}
+
+// ── Spider engine ────────────────────────────────────────────────────────────
+
+/**
+ * Spider a seed URL, yielding each successfully fetched page as a SpiderResult.
+ * Performs BFS up to maxDepth hops and maxPages total.
+ *
+ * @example
+ * for await (const page of spiderUrl("https://docs.example.com", { maxPages: 50, maxDepth: 2 })) {
+ * await indexDocument(db, provider, { title: page.title, content: page.content, url: page.url });
+ * }
+ */
+export async function* spiderUrl(
+ seedUrl: string,
+ options: SpiderOptions = {},
+): AsyncGenerator<SpiderResult, SpiderStats, unknown> {
+ const log = getLogger();
+
+ // Resolve effective limits
+ const maxPages = Math.min(options.maxPages ?? 25, HARD_MAX_PAGES);
+ const maxDepth = Math.min(options.maxDepth ?? 2, HARD_MAX_DEPTH);
+ const sameDomain = options.sameDomain ?? true;
+ const pathPrefix = options.pathPrefix ?? "";
+ const excludePatterns = options.excludePatterns ?? [];
+ const requestDelay = options.requestDelay ?? DEFAULT_REQUEST_DELAY_MS;
+ const fetchOptions = options.fetchOptions;
+
+ // Parse seed URL for domain filtering
+ let seedHostname: string;
+ let seedOrigin: string;
+ try {
+ const parsed = new URL(seedUrl);
+ seedHostname = parsed.hostname;
+ seedOrigin = parsed.origin;
+ } catch {
+ throw new FetchError("Invalid seed URL: " + seedUrl);
+ }
+
+ const stats: SpiderStats = {
+ pagesFetched: 0,
+ pagesCrawled: 0,
+ pagesSkipped: 0,
+ errors: [],
+ };
+
+ // Per-origin robots.txt cache — fetched lazily as new origins are encountered.
+ // Pre-populate with the seed origin so we don't re-fetch it on the first page.
+ const robotsCache = new Map<string, Set<string>>();
+ const seedRobots = await fetchRobotsTxt(seedOrigin, fetchOptions);
+ robotsCache.set(seedOrigin, seedRobots);
+ log.debug({ origin: seedOrigin, rules: seedRobots.size }, "Loaded robots.txt rules");
+
+ const visited = new Set<string>();
+ // BFS queue entries
+ type QueueEntry = { url: string; depth: number };
+ const queue: QueueEntry[] = [{ url: seedUrl, depth: 0 }];
+
+ const deadline = Date.now() + HARD_TOTAL_TIMEOUT_MS;
+
+ while (queue.length > 0 && stats.pagesFetched < maxPages) {
+ // Check total timeout
+ if (Date.now() > deadline) {
+ log.warn({ pagesFetched: stats.pagesFetched }, "Spider total timeout reached");
+ stats.abortReason = "timeout";
+ break;
+ }
+
+ const entry = queue.shift()!;
+ const { url, depth } = entry;
+
+ // Skip already-visited
+ if (visited.has(url)) continue;
+ visited.add(url);
+
+ // Apply filters (except for seed URL at depth 0 — always fetch it)
+ if (depth > 0) {
+ if (sameDomain && !isSameDomain(url, seedHostname)) {
+ log.debug({ url }, "Spider: skipping cross-domain link");
+ stats.pagesSkipped++;
+ continue;
+ }
+ if (pathPrefix && !hasPathPrefix(url, pathPrefix)) {
+ log.debug({ url, pathPrefix }, "Spider: skipping link outside path prefix");
+ stats.pagesSkipped++;
+ continue;
+ }
+ if (excludePatterns.length > 0 && isExcluded(url, excludePatterns)) {
+ log.debug({ url }, "Spider: skipping excluded URL");
+ stats.pagesSkipped++;
+ continue;
+ }
+ // Fetch robots.txt for new origins (cross-domain crawl when sameDomain is false)
+ let urlOrigin: string;
+ try {
+ urlOrigin = new URL(url).origin;
+ } catch {
+ urlOrigin = seedOrigin;
+ }
+ if (!robotsCache.has(urlOrigin)) {
+ const rules = await fetchRobotsTxt(urlOrigin, fetchOptions);
+ robotsCache.set(urlOrigin, rules);
+ log.debug(
+ { origin: urlOrigin, rules: rules.size },
+ "Loaded robots.txt rules for new origin",
+ );
+ }
+ if (isDisallowedByRobots(url, robotsCache.get(urlOrigin)!)) {
+ log.debug({ url }, "Spider: skipping URL disallowed by robots.txt");
+ stats.pagesSkipped++;
+ continue;
+ }
+ }
+
+ // Check maxPages before fetching
+ if (stats.pagesFetched >= maxPages) {
+ stats.abortReason = "maxPages";
+ break;
+ }
+
+ // Delay between requests (skip delay before first request)
+ if (stats.pagesCrawled > 0 && requestDelay > 0) {
+ await sleep(requestDelay);
+ }
+
+ // Fetch the page
+ log.info({ url, depth }, "Spider: fetching page");
+ stats.pagesCrawled++;
+
+ let raw: Awaited<ReturnType<typeof fetchRaw>>;
+ try {
+ raw = await fetchRaw(url, fetchOptions);
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ log.warn({ url, err: msg }, "Spider: fetch failed, skipping");
+ stats.errors.push({ url, error: msg });
+ continue;
+ }
+
+ // Normalize to the final URL after any redirects.
+ // This ensures the visited set, yielded URL, and link-extraction base are all consistent.
+ const canonicalUrl = raw.finalUrl || url;
+ if (canonicalUrl !== url) {
+ visited.add(canonicalUrl);
+ }
+
+ // Convert to markdown
+ const isHtml = raw.contentType.includes("text/html");
+ const content = isHtml ? htmlToMarkdown(raw.body) : raw.body;
+ const title = isHtml
+ ? extractTitle(raw.body, canonicalUrl)
+ : extractTextTitle(raw.body, canonicalUrl);
+
+ stats.pagesFetched++;
+ yield { url: canonicalUrl, title, content, depth };
+
+ // Extract and enqueue child links if we haven't hit maxDepth
+ if (depth < maxDepth) {
+ if (isHtml) {
+ const links = extractLinks(raw.body, canonicalUrl);
+ for (const link of links) {
+ if (!visited.has(link)) {
+ queue.push({ url: link, depth: depth + 1 });
+ }
+ }
+ log.debug({ url, linksFound: links.length }, "Spider: extracted links");
+ }
+ }
+ }
+
+ // If the loop exited via the outer while condition hitting maxPages (not via
+ // an explicit break with abortReason already set), record the reason now.
+ if (!stats.abortReason && queue.length > 0 && stats.pagesFetched >= maxPages) {
+ stats.abortReason = "maxPages";
+ }
+
+ log.info(
+ {
+ pagesFetched: stats.pagesFetched,
+ pagesCrawled: stats.pagesCrawled,
+ pagesSkipped: stats.pagesSkipped,
+ errors: stats.errors.length,
+ abortReason: stats.abortReason,
+ },
+ "Spider: crawl complete",
+ );
+
+ return stats;
+}
+
+function extractTextTitle(text: string, url: string): string {
+ // For plain text/markdown, try first # heading
+ const match = /^#\s+(.+)$/m.exec(text);
+ if (match?.[1]) return match[1].trim();
+ // Fall back to URL
+ try {
+ const parsed = new URL(url);
+ const path = parsed.pathname.replace(/\/$/, "");
+ const last = path.split("/").pop();
+ if (last) return last.replace(/[-_]/g, " ").replace(/\.\w+$/, "");
+ return parsed.hostname;
+ } catch {
+ return url;
+ }
+}
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
diff --git a/src/core/tags.ts b/src/core/tags.ts
index 1fe4bfe..f37180d 100644
--- a/src/core/tags.ts
+++ b/src/core/tags.ts
@@ -222,6 +222,37 @@ export function removeTagFromDocument(
log.info({ documentId, tagId }, "Tag removed from document");
}
+/** Get all tags for multiple documents in a single query. Returns a Map of documentId → tags. */
+export function getDocumentTagsBatch(
+ db: Database.Database,
+ documentIds: string[],
+): Map<string, Tag[]> {
+ if (documentIds.length === 0) return new Map();
+ const placeholders = documentIds.map(() => "?").join(", ");
+ const rows = db
+ .prepare(
+ `SELECT dt.document_id, t.id, t.name, t.created_at
+ FROM tags t
+ JOIN document_tags dt ON dt.tag_id = t.id
+ WHERE dt.document_id IN (${placeholders})
+ ORDER BY t.name`,
+ )
+ .all(...documentIds) as Array<{
+ document_id: string;
+ id: string;
+ name: string;
+ created_at: string;
+ }>;
+
+ const result = new Map<string, Tag[]>();
+ for (const row of rows) {
+ const entry = result.get(row.document_id) ?? [];
+ entry.push({ id: row.id, name: row.name, createdAt: row.created_at });
+ result.set(row.document_id, entry);
+ }
+ return result;
+}
+
/** Get all tags for a specific document. */
export function getDocumentTags(db: Database.Database, documentId: string): Tag[] {
const rows = db
@@ -313,14 +344,39 @@ export function getDocumentsByTag(
}
/** Tokenize text into lowercase words, filtering stopwords and short words. */
-function tokenize(text: string): string[] {
+export function tokenize(text: string): string[] {
return text
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((w) => w.length >= 3 && !STOPWORDS.has(w));
}
-/** Suggest tags for a document based on content analysis (TF-IDF-like keyword extraction). */
+/** Suggest tags from raw text without requiring a database (for pack creation). */
+export function suggestTagsFromText(
+ title: string,
+ content: string,
+ maxSuggestions?: number,
+): string[] {
+ const limit = maxSuggestions ?? 5;
+ const fullText = `${title} ${content}`;
+ const tokens = tokenize(fullText);
+ if (tokens.length === 0) return [];
+
+ const tf = new Map<string, number>();
+ for (const token of tokens) {
+ tf.set(token, (tf.get(token) ?? 0) + 1);
+ }
+
+ const maxTf = Math.max(...tf.values());
+ const scored: Array<{ term: string; score: number }> = [];
+
+ for (const [term, count] of tf) {
+ scored.push({ term, score: count / maxTf });
+ }
+
+ scored.sort((a, b) => b.score - a.score);
+ return scored.slice(0, limit).map((s) => s.term);
+}
export function suggestTags(
db: Database.Database,
documentId: string,
diff --git a/src/core/ttl.ts b/src/core/ttl.ts
new file mode 100644
index 0000000..b6d5ee8
--- /dev/null
+++ b/src/core/ttl.ts
@@ -0,0 +1,59 @@
+import type Database from "better-sqlite3";
+import { getLogger } from "../logger.js";
+
+export interface PruneResult {
+ pruned: number;
+}
+
+/**
+ * Delete all documents whose `expires_at` timestamp is in the past.
+ * Also removes associated chunks and embeddings.
+ */
+export function pruneExpiredDocuments(db: Database.Database): PruneResult {
+ const log = getLogger();
+
+ // Find expired document IDs first
+ const expired = db
+ .prepare(
+ `SELECT id FROM documents
+ WHERE expires_at IS NOT NULL AND expires_at <= datetime('now')`,
+ )
+ .all() as Array<{ id: string }>;
+
+ if (expired.length === 0) return { pruned: 0 };
+
+ const ids = expired.map((r) => r.id);
+ const placeholders = ids.map(() => "?").join(", ");
+
+ const pruneTransaction = db.transaction(() => {
+ // Remove embeddings first (foreign key dependency)
+ try {
+ db.prepare(
+ `DELETE FROM chunk_embeddings
+ WHERE chunk_id IN (
+ SELECT id FROM chunks WHERE document_id IN (${placeholders})
+ )`,
+ ).run(...ids);
+ } catch {
+ // chunk_embeddings may not exist (sqlite-vec not loaded)
+ }
+
+ // Remove chunks
+ db.prepare(`DELETE FROM chunks WHERE document_id IN (${placeholders})`).run(...ids);
+
+ // Remove document tags
+ try {
+ db.prepare(`DELETE FROM document_tags WHERE document_id IN (${placeholders})`).run(...ids);
+ } catch {
+ // document_tags may not exist in all schema versions
+ }
+
+ // Remove documents
+ db.prepare(`DELETE FROM documents WHERE id IN (${placeholders})`).run(...ids);
+ });
+
+ pruneTransaction();
+
+ log.info({ pruned: ids.length }, "Pruned expired documents");
+ return { pruned: ids.length };
+}
diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts
index 6953bac..74407fc 100644
--- a/src/core/url-fetcher.ts
+++ b/src/core/url-fetcher.ts
@@ -1,30 +1,19 @@
import { promises as dns, lookup as dnsLookup } from "node:dns";
import { promisify } from "node:util";
+import { Agent } from "undici";
import { NodeHtmlMarkdown } from "node-html-markdown";
import { FetchError } from "../errors.js";
import { getLogger } from "../logger.js";
-const lookupAsync = promisify(dnsLookup);
-
-let tlsWarningLogged = false;
-
-/**
- * Log a one-time warning when `allowSelfSignedCerts` is enabled but the
- * user has not set `NODE_TLS_REJECT_UNAUTHORIZED=0` in their environment.
- * Setting the env var programmatically is a security anti-pattern flagged
- * by static analysis tools — the user must opt in at the process level.
- */
-function warnIfTlsBypassMissing(): void {
- if (tlsWarningLogged) return;
- if (process.env["NODE_TLS_REJECT_UNAUTHORIZED"] === "0") return;
- tlsWarningLogged = true;
- const log = getLogger();
- log.warn(
- "allowSelfSignedCerts is enabled but NODE_TLS_REJECT_UNAUTHORIZED is not set. " +
- "Set NODE_TLS_REJECT_UNAUTHORIZED=0 in your environment to allow self-signed certificates.",
- );
+/** Lazy singleton undici Agent that skips TLS certificate verification. */
+let _insecureAgent: Agent | undefined;
+function getInsecureAgent(): Agent {
+ _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } });
+ return _insecureAgent;
}
+const lookupAsync = promisify(dnsLookup);
+
export interface FetchedDocument {
title: string;
content: string;
@@ -178,10 +167,11 @@ async function fetchWithRedirects(
allowPrivateUrls: boolean,
allowSelfSignedCerts: boolean,
): Promise<Response> {
- if (allowSelfSignedCerts) {
- warnIfTlsBypassMissing();
- }
- return _fetchWithRedirects(url, timeout, maxRedirects, allowPrivateUrls);
+ // Pass a per-request undici Agent when self-signed certs are allowed.
+ // This is scoped to this specific request chain and does not affect other
+ // concurrent requests (unlike mutating process.env["NODE_TLS_REJECT_UNAUTHORIZED"]).
+ const dispatcher = allowSelfSignedCerts ? getInsecureAgent() : undefined;
+ return _fetchWithRedirects(url, timeout, maxRedirects, allowPrivateUrls, dispatcher);
}
async function _fetchWithRedirects(
@@ -189,6 +179,7 @@ async function _fetchWithRedirects(
timeout: number,
maxRedirects: number,
allowPrivateUrls: boolean,
+ dispatcher: Agent | undefined,
): Promise<Response> {
let current = url;
for (let i = 0; i < maxRedirects; i++) {
@@ -197,7 +188,7 @@ async function _fetchWithRedirects(
// SSRF protection: validateUrl() above resolves DNS and blocks private/internal IPs.
// Redirect following is manual with per-hop validation. DNS rebinding is checked post-fetch.
- // codeql[js/request-forgery] — URL scheme and destination validated above
+ // codeql[js/request-forgery] -- URL scheme and destination validated by validateUrl() above
const response = await fetch(current, {
headers: {
"User-Agent": "LibScope/0.1.0 (documentation indexer)",
@@ -205,7 +196,8 @@ async function _fetchWithRedirects(
},
signal: AbortSignal.timeout(timeout),
redirect: "manual",
- });
+ ...(dispatcher ? { dispatcher: dispatcher as unknown } : {}),
+ } as RequestInit);
// Re-validate the connected IP hasn't changed (DNS rebinding defense)
// Re-resolve and confirm it still matches the pinned set
@@ -246,6 +238,57 @@ async function _fetchWithRedirects(
throw new FetchError(`Too many redirects (max ${maxRedirects})`);
}
+/** Result of a raw fetch — HTML/text body before any conversion, plus resolved final URL. */
+export interface FetchedRaw {
+ body: string;
+ contentType: string;
+ finalUrl: string;
+}
+
+/**
+ * Fetch a URL and return the raw body without converting to markdown.
+ * Useful for callers (e.g. the spider) that need access to raw HTML for link extraction.
+ * Applies all the same SSRF protection, redirect following, and body-size limits as fetchAndConvert.
+ */
+export async function fetchRaw(url: string, options?: FetchOptions): Promise<FetchedRaw> {
+ const log = getLogger();
+ log.debug({ url }, "Fetching raw URL");
+
+ const { timeout, maxRedirects, maxBodySize, allowPrivateUrls, allowSelfSignedCerts } = {
+ ...DEFAULT_FETCH_OPTIONS,
+ ...options,
+ };
+
+ try {
+ await validateUrl(url, allowPrivateUrls);
+
+ const response = await fetchWithRedirects(
+ url,
+ timeout,
+ maxRedirects,
+ allowPrivateUrls,
+ allowSelfSignedCerts,
+ );
+
+ if (!response.ok) {
+ throw new FetchError(`HTTP ${response.status}: ${response.statusText}`);
+ }
+
+ const contentType = response.headers.get("content-type") ?? "";
+ const body = await readBodyWithLimit(response, maxBodySize);
+ // Derive final URL from redirect chain (fetchWithRedirects resolves relative locations)
+ const finalUrl = response.url ?? url;
+
+ return { body, contentType, finalUrl };
+ } catch (err) {
+ if (err instanceof FetchError) throw err;
+ throw new FetchError(
+ `Failed to fetch URL: ${url} — ${err instanceof Error ? err.message : String(err)}`,
+ err,
+ );
+ }
+}
+
/**
* Fetch a URL and convert its HTML content to clean markdown-like text.
* Strips tags, preserves code blocks and headings.
diff --git a/src/core/versioning.ts b/src/core/versioning.ts
index 0bcee13..756fbf5 100644
--- a/src/core/versioning.ts
+++ b/src/core/versioning.ts
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
import type { EmbeddingProvider } from "../providers/embedding.js";
import { DocumentNotFoundError } from "../errors.js";
import { getDocument, updateDocument } from "./documents.js";
+import { getLogger } from "../logger.js";
export const MAX_VERSIONS_DEFAULT = 10;
@@ -174,7 +175,11 @@ function mapRow(row: {
if (row.metadata) {
try {
metadata = JSON.parse(row.metadata) as Record<string, unknown>;
- } catch {
+ } catch (err) {
+ getLogger().warn(
+ { err, versionId: row.id },
+ "Failed to parse version metadata JSON; using null",
+ );
metadata = null;
}
}
diff --git a/src/core/webhooks.ts b/src/core/webhooks.ts
index 84c5b2b..c6c7e77 100644
--- a/src/core/webhooks.ts
+++ b/src/core/webhooks.ts
@@ -1,4 +1,11 @@
-import { randomUUID, createHmac } from "node:crypto";
+import {
+ randomUUID,
+ createHmac,
+ createCipheriv,
+ createDecipheriv,
+ randomBytes,
+ scryptSync,
+} from "node:crypto";
import { promises as dns, lookup as dnsLookup } from "node:dns";
import { promisify } from "node:util";
import type Database from "better-sqlite3";
@@ -8,6 +15,34 @@ import { isPrivateIP } from "./url-fetcher.js";
const lookupAsync = promisify(dnsLookup);
+const SECRET_KEY_ENV = "LIBSCOPE_SECRET_KEY";
+
+/** Encrypt a webhook secret using AES-256-GCM if LIBSCOPE_SECRET_KEY is set. */
+function encryptSecret(plaintext: string): string {
+ const key = process.env[SECRET_KEY_ENV];
+ if (!key) return plaintext;
+ const keyBuf = scryptSync(key, "libscope-webhook-salt", 32);
+ const iv = randomBytes(12);
+ const cipher = createCipheriv("aes-256-gcm", keyBuf, iv);
+ const encrypted = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
+ const tag = cipher.getAuthTag();
+ return `enc:${iv.toString("hex")}:${tag.toString("hex")}:${encrypted.toString("hex")}`;
+}
+
+/** Decrypt a webhook secret encrypted by encryptSecret. */
+function decryptSecret(stored: string): string {
+ if (!stored.startsWith("enc:")) return stored;
+ const key = process.env[SECRET_KEY_ENV];
+ if (!key) return stored;
+ const parts = stored.split(":");
+ if (parts.length !== 4) return stored;
+ const [, ivHex, tagHex, encHex] = parts;
+ const keyBuf = scryptSync(key, "libscope-webhook-salt", 32);
+ const decipher = createDecipheriv("aes-256-gcm", keyBuf, Buffer.from(ivHex!, "hex"));
+ decipher.setAuthTag(Buffer.from(tagHex!, "hex"));
+ return decipher.update(Buffer.from(encHex!, "hex")).toString("utf8") + decipher.final("utf8");
+}
+
export const WEBHOOK_EVENTS = [
"document.created",
"document.updated",
@@ -79,10 +114,19 @@ function recordFailure(
}
function rowToWebhook(row: WebhookRow): Webhook {
+ let events: WebhookEvent[] = [];
+ try {
+ events = JSON.parse(row.events) as WebhookEvent[];
+ } catch {
+ getLogger().warn(
+ { webhookId: row.id },
+ "Failed to parse webhook events JSON; defaulting to []",
+ );
+ }
return {
id: row.id,
url: row.url,
- events: JSON.parse(row.events) as WebhookEvent[],
+ events,
secret: row.secret,
active: row.active === 1,
createdAt: row.created_at,
@@ -157,14 +201,21 @@ export async function createWebhook(
validateEvents(events);
await validateWebhookUrlSsrf(url);
+ if (!process.env[SECRET_KEY_ENV]) {
+ getLogger().warn(
+ "LIBSCOPE_SECRET_KEY is not set — webhook secrets stored in plaintext. Set this env var to enable at-rest encryption.",
+ );
+ }
+
const id = randomUUID();
const eventsJson = JSON.stringify(events);
+ const encryptedSecret = secret ? encryptSecret(secret) : null;
db.prepare("INSERT INTO webhooks (id, url, events, secret) VALUES (?, ?, ?, ?)").run(
id,
url,
eventsJson,
- secret ?? null,
+ encryptedSecret,
);
const row = db.prepare("SELECT * FROM webhooks WHERE id = ?").get(id) as WebhookRow;
@@ -212,7 +263,12 @@ export async function updateWebhook(
const url = updates.url ?? existing.url;
const events = updates.events ?? existing.events;
- const secret = "secret" in updates ? updates.secret : existing.secret;
+ const secret =
+ "secret" in updates
+ ? updates.secret
+ ? encryptSecret(updates.secret)
+ : updates.secret
+ : existing.secret;
const active = "active" in updates ? (updates.active ? 1 : 0) : existing.active ? 1 : 0;
db.prepare("UPDATE webhooks SET url = ?, events = ?, secret = ?, active = ? WHERE id = ?").run(
@@ -251,7 +307,11 @@ export function fireWebhooks(
data: Record<string, unknown>,
): void {
const log = getLogger();
- const rows = db.prepare("SELECT * FROM webhooks WHERE active = 1").all() as WebhookRow[];
+ const rows = db
+ .prepare(
+ "SELECT id, url, events, secret, active, created_at, last_triggered_at, failure_count FROM webhooks WHERE active = 1",
+ )
+ .all() as WebhookRow[];
const body = buildPayload(event, data);
@@ -265,7 +325,7 @@ export function fireWebhooks(
"Content-Type": "application/json",
};
if (webhook.secret) {
- headers["X-LibScope-Signature"] = signPayload(body, webhook.secret);
+ headers["X-LibScope-Signature"] = signPayload(body, decryptSecret(webhook.secret));
}
// SSRF check before firing
diff --git a/src/db/connection.ts b/src/db/connection.ts
index 8d11591..43a3853 100644
--- a/src/db/connection.ts
+++ b/src/db/connection.ts
@@ -42,6 +42,9 @@ export function getDatabase(dbPath?: string): Database.Database {
db = new Database(resolvedPath);
cachedPath = resolvedPath;
db.pragma("journal_mode = WAL");
+ db.pragma("synchronous = NORMAL"); // safe with WAL, 2-3x faster writes
+ db.pragma("cache_size = -32000"); // 32 MB page cache
+ db.pragma("temp_store = MEMORY"); // keep temp tables in memory
db.pragma("foreign_keys = ON");
// Load sqlite-vec extension
@@ -90,6 +93,9 @@ export function createDatabase(dbPath: string): Database.Database {
}
const newDb = new Database(dbPath);
newDb.pragma("journal_mode = WAL");
+ newDb.pragma("synchronous = NORMAL");
+ newDb.pragma("cache_size = -32000");
+ newDb.pragma("temp_store = MEMORY");
newDb.pragma("foreign_keys = ON");
try {
const sqliteVec = require("sqlite-vec") as { load: (db: Database.Database) => void };
diff --git a/src/db/schema.ts b/src/db/schema.ts
index 74cfc6f..5b0fb61 100644
--- a/src/db/schema.ts
+++ b/src/db/schema.ts
@@ -2,7 +2,7 @@ import type Database from "better-sqlite3";
import { DatabaseError } from "../errors.js";
import { getLogger } from "../logger.js";
-const SCHEMA_VERSION = 15;
+const SCHEMA_VERSION = 17;
const MIGRATIONS: Record<number, string> = {
1: `
@@ -257,6 +257,18 @@ const MIGRATIONS: Record<number, string> = {
INSERT INTO schema_version (version) VALUES (15);
`,
+ 16: `
+ CREATE INDEX IF NOT EXISTS idx_documents_content_hash ON documents(content_hash);
+ CREATE INDEX IF NOT EXISTS idx_chunks_doc_idx ON chunks(document_id, chunk_index);
+
+ INSERT INTO schema_version (version) VALUES (16);
+ `,
+ 17: `
+ ALTER TABLE documents ADD COLUMN expires_at TEXT;
+ CREATE INDEX IF NOT EXISTS idx_documents_expires_at ON documents(expires_at) WHERE expires_at IS NOT NULL;
+
+ INSERT INTO schema_version (version) VALUES (17);
+ `,
};
const FTS_BACKFILL_SQL = `
diff --git a/src/mcp/errors.ts b/src/mcp/errors.ts
new file mode 100644
index 0000000..38f07b8
--- /dev/null
+++ b/src/mcp/errors.ts
@@ -0,0 +1,40 @@
+import { LibScopeError } from "../errors.js";
+import { getLogger } from "../logger.js";
+
+/** Convert a thrown error into an MCP error response object. */
+export function errorResponse(err: unknown): {
+ content: Array<{ type: "text"; text: string }>;
+ isError: true;
+} {
+ let message: string;
+ if (err instanceof LibScopeError) {
+ message = err.message;
+ } else if (err instanceof Error) {
+ message = `${err.name}: ${err.message}`;
+ } else {
+ message = `An unexpected error occurred: ${String(err)}`;
+ }
+
+ const log = getLogger();
+ log.error({ err }, "MCP tool error");
+
+ return {
+ content: [{ type: "text" as const, text: `Error: ${message}` }],
+ isError: true,
+ };
+}
+
+export type ToolResult = { content: Array<{ type: "text"; text: string }>; isError?: boolean };
+
+/** Wraps a tool handler so that thrown errors are converted to MCP error responses. */
+export function withErrorHandling<P>(
+ handler: (params: P) => ToolResult | Promise<ToolResult>,
+): (params: P) => Promise<ToolResult> {
+ return async (params: P) => {
+ try {
+ return await handler(params);
+ } catch (err) {
+ return errorResponse(err);
+ }
+ };
+}
diff --git a/src/mcp/server.ts b/src/mcp/server.ts
index 8d62806..cd44e5c 100644
--- a/src/mcp/server.ts
+++ b/src/mcp/server.ts
@@ -6,7 +6,13 @@ import { getDatabase, runMigrations, createVectorTable } from "../db/index.js";
import { getActiveWorkspace, getWorkspacePath } from "../core/workspace.js";
import { createEmbeddingProvider } from "../providers/index.js";
import { searchDocuments } from "../core/search.js";
-import { askQuestion, createLlmProvider, type LlmProvider } from "../core/rag.js";
+import {
+ askQuestion,
+ createLlmProvider,
+ getContextForQuestion,
+ isPassthroughMode,
+ type LlmProvider,
+} from "../core/rag.js";
import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js";
import { rateDocument, getDocumentRatings } from "../core/ratings.js";
import { indexDocument } from "../core/indexing.js";
@@ -23,45 +29,12 @@ import { createWebhook, listWebhooks, deleteWebhook, redactWebhook } from "../co
import type { WebhookEvent } from "../core/webhooks.js";
import { suggestTags } from "../core/tags.js";
import { fetchAndConvert } from "../core/url-fetcher.js";
+import { spiderUrl } from "../core/spider.js";
+import type { SpiderOptions } from "../core/spider.js";
import { initLogger, getLogger } from "../logger.js";
-import { ConfigError, LibScopeError, ValidationError } from "../errors.js";
-
-function errorResponse(err: unknown): {
- content: Array<{ type: "text"; text: string }>;
- isError: true;
-} {
- let message: string;
- if (err instanceof LibScopeError) {
- message = err.message;
- } else if (err instanceof Error) {
- message = `${err.name}: ${err.message}`;
- } else {
- message = `An unexpected error occurred: ${String(err)}`;
- }
-
- const log = getLogger();
- log.error({ err }, "MCP tool error");
-
- return {
- content: [{ type: "text" as const, text: `Error: ${message}` }],
- isError: true,
- };
-}
-
-type ToolResult = { content: Array<{ type: "text"; text: string }>; isError?: boolean };
-
-/** Wraps a tool handler so that thrown errors are converted to MCP error responses. */
-function withErrorHandling<P>(
- handler: (params: P) => ToolResult | Promise<ToolResult>,
-): (params: P) => Promise<ToolResult> {
- return async (params: P) => {
- try {
- return await handler(params);
- } catch (err) {
- return errorResponse(err);
- }
- };
-}
+import { ConfigError, ValidationError } from "../errors.js";
+import { errorResponse, withErrorHandling } from "./errors.js";
+export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js";
// Start the server
async function main(): Promise<void> {
@@ -330,7 +303,7 @@ async function main(): Promise<void> {
// Tool: submit-document
server.tool(
"submit-document",
- "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically.",
+ "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically. Set spider=true to crawl linked pages from the URL.",
{
title: z
.string()
@@ -353,17 +326,104 @@ async function main(): Promise<void> {
topic: z.string().optional().describe("Topic ID to categorize under"),
library: z.string().optional().describe("Library name (for library docs)"),
version: z.string().optional().describe("Library version"),
+ spider: z
+ .boolean()
+ .optional()
+ .describe("When true, crawl pages linked from the URL. Requires 'url'. Default: false."),
+ maxPages: z
+ .number()
+ .int()
+ .positive()
+ .optional()
+ .describe("Maximum pages to index during a spider run (default: 25, hard cap: 200)."),
+ maxDepth: z
+ .number()
+ .int()
+ .min(0)
+ .optional()
+ .describe(
+ "Maximum link-hop depth from the seed URL (default: 2, hard cap: 5). 0 = seed only.",
+ ),
+ sameDomain: z
+ .boolean()
+ .optional()
+ .describe("Only follow links on the same domain as the seed URL (default: true)."),
+ pathPrefix: z
+ .string()
+ .optional()
+ .describe("Only follow links whose path starts with this prefix (e.g. '/docs/')."),
+ excludePatterns: z
+ .array(z.string())
+ .optional()
+ .describe("Glob patterns for URLs to skip (e.g. ['*/changelog*', '*/api/v1/*'])."),
},
withErrorHandling(async (params) => {
let { title, content } = params;
const { url, library, version, topic } = params;
+ const fetchOptions = {
+ allowPrivateUrls: config.indexing.allowPrivateUrls,
+ allowSelfSignedCerts: config.indexing.allowSelfSignedCerts,
+ };
+
+ // Spider mode — crawl linked pages from the URL
+ if (params.spider && !url) {
+ throw new ValidationError("Field 'url' is required when spider is true");
+ }
+ if (params.spider && url) {
+ const spiderOptions: SpiderOptions = { fetchOptions };
+ if (params.maxPages !== undefined) spiderOptions.maxPages = params.maxPages;
+ if (params.maxDepth !== undefined) spiderOptions.maxDepth = params.maxDepth;
+ if (params.sameDomain !== undefined) spiderOptions.sameDomain = params.sameDomain;
+ if (params.pathPrefix !== undefined) spiderOptions.pathPrefix = params.pathPrefix;
+ if (params.excludePatterns !== undefined)
+ spiderOptions.excludePatterns = params.excludePatterns;
+
+ const indexed: Array<{ id: string; title: string }> = [];
+ const errors: Array<{ url: string; error: string }> = [];
+ const sourceType = params.sourceType ?? (library ? "library" : "manual");
+
+ const gen = spiderUrl(url, spiderOptions);
+ let result = await gen.next();
+ while (!result.done) {
+ const page = result.value;
+ try {
+ const doc = await indexDocument(db, provider, {
+ title: page.title,
+ content: page.content,
+ sourceType,
+ library,
+ version,
+ topicId: topic,
+ url: page.url,
+ submittedBy: "model",
+ });
+ indexed.push({ id: doc.id, title: page.title });
+ } catch (err) {
+ errors.push({ url: page.url, error: err instanceof Error ? err.message : String(err) });
+ }
+ result = await gen.next();
+ }
+ const stats = result.value;
+
+ const summary = [
+ `Spider complete.`,
+ `Pages indexed: ${indexed.length}`,
+ `Pages crawled: ${stats?.pagesCrawled ?? indexed.length}`,
+ `Pages skipped: ${stats?.pagesSkipped ?? 0}`,
+ errors.length > 0 ? `Errors: ${errors.length}` : null,
+ stats?.abortReason ? `Stopped early: ${stats.abortReason}` : null,
+ ]
+ .filter(Boolean)
+ .join("\n");
+
+ return {
+ content: [{ type: "text" as const, text: summary }],
+ };
+ }
// If URL is provided and no content, fetch it
if (url && !content) {
- const fetched = await fetchAndConvert(url, {
- allowPrivateUrls: config.indexing.allowPrivateUrls,
- allowSelfSignedCerts: config.indexing.allowSelfSignedCerts,
- });
+ const fetched = await fetchAndConvert(url, fetchOptions);
content = fetched.content;
title ??= fetched.title;
}
@@ -543,9 +603,30 @@ async function main(): Promise<void> {
library: z.string().optional().describe("Filter by library name"),
},
withErrorHandling(async (params) => {
+ if (isPassthroughMode(config)) {
+ const { contextPrompt, sources } = await getContextForQuestion(db, provider, {
+ question: params.question,
+ topK: params.topK,
+ topic: params.topic,
+ library: params.library,
+ });
+
+ const sourcesText =
+ sources.length > 0
+ ? "\n\n**Sources:**\n" +
+ sources
+ .map((s) => `- ${s.title} (score: ${s.score.toFixed(2)}) [${s.documentId}]`)
+ .join("\n")
+ : "";
+
+ return {
+ content: [{ type: "text" as const, text: contextPrompt + sourcesText }],
+ };
+ }
+
if (!llmProvider) {
throw new ConfigError(
- "No LLM provider configured. Set llm.provider to 'openai' or 'ollama' in your config.",
+ "No LLM provider configured. Set llm.provider to 'openai', 'ollama', or 'passthrough' in your config.",
);
}
diff --git a/src/providers/local.ts b/src/providers/local.ts
index 10e4a7b..37391b3 100644
--- a/src/providers/local.ts
+++ b/src/providers/local.ts
@@ -2,6 +2,17 @@ import { EmbeddingError } from "../errors.js";
import type { EmbeddingProvider } from "./embedding.js";
import { getLogger } from "../logger.js";
+/** Minimal typed interface for the @xenova/transformers feature-extraction pipeline output. */
+interface TransformersOutput {
+ data: Float32Array;
+}
+
+/** Minimal typed interface for the @xenova/transformers feature-extraction pipeline function. */
+type FeatureExtractionPipeline = (
+ input: string,
+ options: { pooling: string; normalize: boolean },
+) => Promise<TransformersOutput>;
+
/**
* Local embedding provider using @xenova/transformers (all-MiniLM-L6-v2).
* Downloads the model on first use (~80MB). Runs entirely in-process.
@@ -10,7 +21,7 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
readonly name = "local";
readonly dimensions = 384;
- private pipeline: unknown = null;
+ private pipeline: FeatureExtractionPipeline | null = null;
private initPromise: Promise<void> | null = null;
private async ensureInitialized(): Promise<void> {
@@ -24,7 +35,11 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
try {
// Dynamic import to avoid loading transformers until needed
const { pipeline } = await import("@xenova/transformers");
- this.pipeline = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+ // Cast to the typed interface; @xenova/transformers lacks precise TS generics for pipeline output
+ this.pipeline = (await pipeline(
+ "feature-extraction",
+ "Xenova/all-MiniLM-L6-v2",
+ )) as unknown as FeatureExtractionPipeline;
log.info("Local embedding model loaded successfully");
} catch (err) {
this.initPromise = null;
@@ -38,10 +53,8 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
}
await this.ensureInitialized();
try {
- // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
- const output = await (this.pipeline as any)(text, { pooling: "mean", normalize: true });
- // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
- const embedding = Array.from(output.data as Float32Array);
+ const output = await this.pipeline!(text, { pooling: "mean", normalize: true });
+ const embedding = Array.from(output.data);
if (embedding.length !== this.dimensions) {
throw new EmbeddingError(
`Expected embedding dimension ${this.dimensions}, got ${embedding.length}`,
diff --git a/tests/integration/retrieval-quality.test.ts b/tests/integration/retrieval-quality.test.ts
new file mode 100644
index 0000000..cd73f78
--- /dev/null
+++ b/tests/integration/retrieval-quality.test.ts
@@ -0,0 +1,371 @@
+/**
+ * Integration test: End-to-end retrieval quality benchmark.
+ *
+ * Uses sqlite-vec for real vector search with a lightweight TF-IDF–style
+ * embedding provider that produces deterministic, semantically meaningful
+ * vectors without needing a neural model or network access.
+ *
+ * This proves the full pipeline: index → embed → store → search → rank,
+ * including metadata enrichment, title boosting, hybrid RRF, and AND logic.
+ *
+ * If the local neural model (all-MiniLM-L6-v2) is available, a second
+ * describe block runs the same queries with real embeddings.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import Database from "better-sqlite3";
+import type { EmbeddingProvider } from "../../src/providers/embedding.js";
+import { chunkContent } from "../../src/core/indexing.js";
+import { searchDocuments } from "../../src/core/search.js";
+import { createRequire } from "node:module";
+import { runMigrations, createVectorTable } from "../../src/db/schema.js";
+
+const TIMEOUT = 60_000;
+
+// ---------------------------------------------------------------------------
+// TF-IDF–style embedding provider: deterministic, no network, semantically
+// meaningful (documents sharing words will have closer vectors).
+// ---------------------------------------------------------------------------
+class TfIdfEmbeddingProvider implements EmbeddingProvider {
+ readonly name = "tfidf-test";
+ readonly dimensions: number;
+ private readonly vocab: Map<string, number>;
+
+ constructor(corpusTexts: string[]) {
+ // Build vocabulary from corpus
+ const wordSet = new Set<string>();
+ for (const text of corpusTexts) {
+ for (const w of this.tokenize(text)) wordSet.add(w);
+ }
+ const sorted = [...wordSet].sort();
+ this.vocab = new Map(sorted.map((w, i) => [w, i]));
+ this.dimensions = sorted.length;
+ }
+
+ private tokenize(text: string): string[] {
+ return text
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .split(/\s+/)
+ .filter((w) => w.length > 1);
+ }
+
+ // eslint-disable-next-line @typescript-eslint/require-await
+ async embed(text: string): Promise<number[]> {
+ const vec = new Float64Array(this.dimensions);
+ const words = this.tokenize(text);
+ for (const w of words) {
+ const idx = this.vocab.get(w);
+ if (idx !== undefined) vec[idx] += 1;
+ }
+ // L2 normalize
+ let mag = 0;
+ for (const v of vec) mag += v * v;
+ mag = Math.sqrt(mag);
+ if (mag > 0) {
+ for (let i = 0; i < vec.length; i++) vec[i] /= mag;
+ }
+ return Array.from(vec);
+ }
+
+ async embedBatch(texts: string[]): Promise<number[][]> {
+ return Promise.all(texts.map((t) => this.embed(t)));
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Corpus
+// ---------------------------------------------------------------------------
+interface CorpusDoc {
+ id: string;
+ title: string;
+ content: string;
+ library?: string;
+}
+
+const CORPUS: CorpusDoc[] = [
+ {
+ id: "react-hooks",
+ title: "React Hooks",
+ content:
+ "React hooks like useState and useEffect allow state management in functional components. " +
+ "useState returns a stateful value and a function to update it. " +
+ "useEffect performs side effects after render.",
+ },
+ {
+ id: "react-router",
+ title: "React Router",
+ content:
+ "React Router provides declarative routing for React applications with dynamic route matching. " +
+ "Use BrowserRouter and Route components to define navigation paths.",
+ },
+ {
+ id: "ts-generics",
+ title: "TypeScript Generics",
+ library: "typescript",
+ content:
+ "TypeScript generics enable writing reusable type-safe functions and classes. " +
+ "Use angle brackets to define type parameters like function identity<T>(arg: T): T.",
+ },
+ {
+ id: "ts-types",
+ title: "TypeScript Type System",
+ library: "typescript",
+ content:
+ "TypeScript type system includes union types intersection types and conditional types. " +
+ "Mapped types transform existing types property by property.",
+ },
+ {
+ id: "node-streams",
+ title: "Node.js Streams",
+ content:
+ "Node.js streams provide an interface for reading and writing data in chunks efficiently. " +
+ "Readable streams emit data events, writable streams accept data via write method.",
+ },
+ {
+ id: "node-http",
+ title: "Node.js HTTP",
+ content:
+ "Node.js HTTP module allows creating web servers and handling HTTP requests and responses. " +
+ "Use http.createServer to start a server listening on a port.",
+ },
+ {
+ id: "sql-joins",
+ title: "SQL Joins",
+ content:
+ "SQL joins combine rows from two or more tables based on related columns between them. " +
+ "INNER JOIN returns matching rows, LEFT JOIN includes all rows from the left table.",
+ },
+ {
+ id: "sql-index",
+ title: "SQL Indexing",
+ content:
+ "SQL indexes improve query performance by creating efficient data structures for lookups. " +
+ "B-tree indexes are the most common type supporting equality and range queries.",
+ },
+];
+
+const QUERIES: Array<{ query: string; expectedTopId: string; label: string }> = [
+ {
+ query: "React state management hooks useState",
+ expectedTopId: "react-hooks",
+ label: "React hooks",
+ },
+ {
+ query: "TypeScript generic type parameters reusable",
+ expectedTopId: "ts-generics",
+ label: "TS generics",
+ },
+ {
+ query: "Node.js streaming data reading writing chunks",
+ expectedTopId: "node-streams",
+ label: "Node streams",
+ },
+ {
+ query: "SQL join tables combine rows matching",
+ expectedTopId: "sql-joins",
+ label: "SQL joins",
+ },
+ {
+ query: "React routing pages navigation BrowserRouter",
+ expectedTopId: "react-router",
+ label: "React routing",
+ },
+ {
+ query: "SQL database index query performance",
+ expectedTopId: "sql-index",
+ label: "SQL indexing",
+ },
+];
+
+// ---------------------------------------------------------------------------
+// Helper: load sqlite-vec (returns false when the extension isn't available)
+// ---------------------------------------------------------------------------
+let vecAvailable: boolean | undefined;
+
+function isVecAvailable(): boolean {
+ if (vecAvailable !== undefined) return vecAvailable;
+ try {
+ const require = createRequire(import.meta.url);
+ require.resolve("sqlite-vec");
+ vecAvailable = true;
+ } catch {
+ vecAvailable = false;
+ }
+ return vecAvailable;
+}
+
+function loadVec(db: Database.Database): void {
+ const require = createRequire(import.meta.url);
+ const sqliteVec = require("sqlite-vec") as { load: (db: Database.Database) => void };
+ sqliteVec.load(db);
+}
+
+// ---------------------------------------------------------------------------
+// Helper: index full corpus into a DB
+// ---------------------------------------------------------------------------
+async function indexCorpus(db: Database.Database, provider: EmbeddingProvider): Promise<void> {
+ const insertDoc = db.prepare(
+ `INSERT INTO documents (id, title, content, source_type, library) VALUES (?, ?, ?, 'manual', ?)`,
+ );
+ const insertChunkStmt = db.prepare(
+ `INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)`,
+ );
+ const insertEmbedding = db.prepare(
+ `INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)`,
+ );
+
+ for (const doc of CORPUS) {
+ insertDoc.run(doc.id, doc.title, doc.content, doc.library ?? null);
+
+ const chunks = chunkContent(doc.content);
+ for (let i = 0; i < chunks.length; i++) {
+ const chunkId = `${doc.id}-c${i}`;
+ insertChunkStmt.run(chunkId, doc.id, chunks[i], i);
+
+ // Metadata-enriched embedding (same as production indexDocument)
+ const metaParts: string[] = [];
+ if (doc.title) metaParts.push(doc.title);
+ if (doc.library) metaParts.push(`Library: ${doc.library}`);
+ const metaPrefix = metaParts.length > 0 ? metaParts.join(" | ") + "\n\n" : "";
+ const enrichedText = metaPrefix + chunks[i]!;
+
+ const embedding = await provider.embed(enrichedText);
+ const vecBuffer = Buffer.from(new Float32Array(embedding).buffer);
+ insertEmbedding.run(chunkId, vecBuffer);
+ }
+ }
+}
+
+// =========================================================================
+// Test suite with TF-IDF provider (always runs, no network needed)
+// =========================================================================
+describe.runIf(isVecAvailable())("retrieval quality: TF-IDF embeddings + sqlite-vec", () => {
+ let db: Database.Database;
+ let provider: TfIdfEmbeddingProvider;
+
+ beforeAll(async () => {
+ // Build vocabulary from all corpus text + queries
+ const allTexts = [
+ ...CORPUS.map((d) => `${d.title} ${d.content}`),
+ ...QUERIES.map((q) => q.query),
+ ];
+ provider = new TfIdfEmbeddingProvider(allTexts);
+
+ db = new Database(":memory:");
+ db.pragma("foreign_keys = ON");
+ loadVec(db);
+ runMigrations(db);
+ createVectorTable(db, provider.dimensions);
+
+ await indexCorpus(db, provider);
+ }, TIMEOUT);
+
+ afterAll(() => {
+ db?.close();
+ });
+
+ for (const { query, expectedTopId, label } of QUERIES) {
+ it(`ranks "${label}" in top 3 for: "${query}"`, async () => {
+ const { results } = await searchDocuments(db, provider, {
+ query,
+ limit: 8,
+ analyticsEnabled: false,
+ });
+
+ expect(results.length).toBeGreaterThan(0);
+
+ const rank = results.findIndex((r) => r.documentId === expectedTopId);
+ const topResult = results[0]!;
+
+ if (process.env.DEBUG) {
+ console.log(
+ ` [${label}] top=${topResult.documentId} (${topResult.score.toFixed(4)}), ` +
+ `expected=${expectedTopId} at rank ${rank + 1}, method=${topResult.scoreExplanation.method}`,
+ );
+ }
+
+ expect(rank).toBeGreaterThanOrEqual(0);
+ expect(rank).toBeLessThan(3);
+ });
+ }
+
+ it("uses hybrid search method (RRF fusion)", async () => {
+ const { results } = await searchDocuments(db, provider, {
+ query: "React hooks state management",
+ limit: 8,
+ analyticsEnabled: false,
+ });
+
+ expect(results.length).toBeGreaterThan(0);
+ const methods = new Set(results.map((r) => r.scoreExplanation.method));
+
+ if (process.env.DEBUG) console.log(` search methods used: ${[...methods].join(", ")}`);
+
+ // With both vector + FTS5, we should get hybrid results
+ expect(methods.has("hybrid")).toBe(true);
+ });
+
+ it("title boost lifts title-matching documents", async () => {
+ const { results } = await searchDocuments(db, provider, {
+ query: "TypeScript Generics",
+ limit: 8,
+ analyticsEnabled: false,
+ });
+
+ const tsGenerics = results.find((r) => r.documentId === "ts-generics");
+ expect(tsGenerics).toBeDefined();
+ expect(tsGenerics!.scoreExplanation.boostFactors.some((f) => f.includes("title_match"))).toBe(
+ true,
+ );
+
+ if (process.env.DEBUG) {
+ console.log(
+ ` title boost: ts-generics rank=${results.findIndex((r) => r.documentId === "ts-generics") + 1}, ` +
+ `score=${tsGenerics!.score.toFixed(4)}, factors=${tsGenerics!.scoreExplanation.boostFactors.join(",")}`,
+ );
+ }
+
+ // Should be rank 1
+ expect(results[0]!.documentId).toBe("ts-generics");
+ });
+
+ it("AND logic prefers chunks containing all query terms", async () => {
+ const { results } = await searchDocuments(db, provider, {
+ query: "TypeScript generics reusable",
+ limit: 8,
+ analyticsEnabled: false,
+ });
+
+ expect(results.length).toBeGreaterThan(0);
+
+ // The ts-generics doc contains all three terms
+ const tsGenerics = results.find((r) => r.documentId === "ts-generics");
+ expect(tsGenerics).toBeDefined();
+
+ // It should be ranked high
+ const rank = results.findIndex((r) => r.documentId === "ts-generics");
+ if (process.env.DEBUG)
+ console.log(` AND logic: ts-generics rank=${rank + 1} for "TypeScript generics reusable"`);
+ expect(rank).toBeLessThan(3);
+ });
+
+ it("overall precision: at least 5/6 queries rank expected doc in top 3", async () => {
+ let hits = 0;
+
+ for (const { query, expectedTopId, label } of QUERIES) {
+ const { results } = await searchDocuments(db, provider, {
+ query,
+ limit: 8,
+ analyticsEnabled: false,
+ });
+ const rank = results.findIndex((r) => r.documentId === expectedTopId);
+ if (rank >= 0 && rank < 3) hits++;
+ else if (process.env.DEBUG)
+ console.log(` miss: "${label}" expected=${expectedTopId} actual rank=${rank + 1}`);
+ }
+
+ if (process.env.DEBUG)
+ console.log(`\n ★ Overall precision: ${hits}/${QUERIES.length} in top-3\n`);
+ expect(hits).toBeGreaterThanOrEqual(5);
+ });
+});
diff --git a/tests/unit/bulk.test.ts b/tests/unit/bulk.test.ts
index e9239fc..37094b8 100644
--- a/tests/unit/bulk.test.ts
+++ b/tests/unit/bulk.test.ts
@@ -76,6 +76,25 @@ describe("bulk operations", () => {
expect(ids).toContain("doc-b");
});
+ it("filters by dateFrom", () => {
+ insertDoc(db, "doc-old", "Old Doc", {
+ library: "react",
+ createdAt: "2020-01-01T00:00:00.000Z",
+ });
+ const ids = resolveSelector(db, { library: "react", dateFrom: "2024-01-01T00:00:00.000Z" });
+ expect(ids).not.toContain("doc-old");
+ expect(ids).toContain("doc-a");
+ });
+
+ it("filters by dateTo", () => {
+ insertDoc(db, "doc-future", "Future Doc", {
+ library: "react",
+ createdAt: "2099-01-01T00:00:00.000Z",
+ });
+ const ids = resolveSelector(db, { library: "react", dateTo: "2025-01-01T00:00:00.000Z" });
+ expect(ids).not.toContain("doc-future");
+ });
+
it("throws on empty selector", () => {
expect(() => resolveSelector(db, {})).toThrow(ValidationError);
});
@@ -90,9 +109,64 @@ describe("bulk operations", () => {
expect(ids.length).toBeLessThanOrEqual(10);
});
- it("returns empty array for negative limit", () => {
- const ids = resolveSelector(db, { library: "react" }, -5);
- expect(ids).toHaveLength(0);
+ it("throws ValidationError for negative limit", () => {
+ expect(() => resolveSelector(db, { library: "react" }, -5)).toThrow(ValidationError);
+ expect(() => resolveSelector(db, { library: "react" }, -1)).toThrow(
+ "limit must be a non-negative integer",
+ );
+ });
+
+ it("applies dateFrom filter at SQL level before LIMIT", () => {
+ // Insert enough docs to exceed a small limit, with varying dates
+ for (let i = 0; i < 20; i++) {
+ insertDoc(db, `old-${i}`, `Old Doc ${i}`, {
+ library: "test-lib",
+ createdAt: "2020-01-01T00:00:00.000Z",
+ });
+ }
+ for (let i = 0; i < 5; i++) {
+ insertDoc(db, `new-${i}`, `New Doc ${i}`, {
+ library: "test-lib",
+ createdAt: "2025-06-01T00:00:00.000Z",
+ });
+ }
+
+ // With a limit of 10, date filter must happen in SQL before LIMIT,
+ // otherwise old docs could fill the limit and exclude new ones
+ const ids = resolveSelector(
+ db,
+ { library: "test-lib", dateFrom: "2025-01-01T00:00:00.000Z" },
+ 10,
+ );
+ expect(ids).toHaveLength(5);
+ for (const id of ids) {
+ expect(id).toMatch(/^new-/);
+ }
+ });
+
+ it("applies dateTo filter at SQL level before LIMIT", () => {
+ for (let i = 0; i < 20; i++) {
+ insertDoc(db, `future-${i}`, `Future Doc ${i}`, {
+ library: "test-lib",
+ createdAt: "2099-01-01T00:00:00.000Z",
+ });
+ }
+ for (let i = 0; i < 5; i++) {
+ insertDoc(db, `past-${i}`, `Past Doc ${i}`, {
+ library: "test-lib",
+ createdAt: "2020-06-01T00:00:00.000Z",
+ });
+ }
+
+ const ids = resolveSelector(
+ db,
+ { library: "test-lib", dateTo: "2025-01-01T00:00:00.000Z" },
+ 10,
+ );
+ expect(ids).toHaveLength(5);
+ for (const id of ids) {
+ expect(id).toMatch(/^past-/);
+ }
});
});
diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts
index 9d380c8..a84d0be 100644
--- a/tests/unit/config.test.ts
+++ b/tests/unit/config.test.ts
@@ -1,10 +1,11 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { loadConfig, validateConfig } from "../../src/config.js";
+import { loadConfig, validateConfig, invalidateConfigCache } from "../../src/config.js";
import type { LibScopeConfig } from "../../src/config.js";
import * as loggerModule from "../../src/logger.js";
describe("config", () => {
it("should return default config when no files exist", () => {
+ invalidateConfigCache();
const config = loadConfig();
expect(config.embedding.provider).toBe("local");
@@ -12,10 +13,18 @@ describe("config", () => {
expect(config.database.path).toContain("libscope.db");
});
+ it("should return cached config on repeated calls", () => {
+ invalidateConfigCache();
+ const first = loadConfig();
+ const second = loadConfig(); // cache hit
+ expect(second).toBe(first); // same object reference
+ });
+
it("should respect LIBSCOPE_EMBEDDING_PROVIDER env var", () => {
const original = process.env["LIBSCOPE_EMBEDDING_PROVIDER"];
try {
process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = "ollama";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.embedding.provider).toBe("ollama");
} finally {
@@ -31,6 +40,7 @@ describe("config", () => {
const original = process.env["LIBSCOPE_EMBEDDING_PROVIDER"];
try {
process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = "invalid";
+ invalidateConfigCache();
const config = loadConfig();
// Should fall through to default since "invalid" doesn't match the switch
expect(config.embedding.provider).toBe("local");
@@ -47,6 +57,7 @@ describe("config", () => {
const original = process.env["LIBSCOPE_OPENAI_API_KEY"];
try {
process.env["LIBSCOPE_OPENAI_API_KEY"] = "sk-test123";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.embedding.openaiApiKey).toBe("sk-test123");
} finally {
@@ -62,6 +73,7 @@ describe("config", () => {
const original = process.env["LIBSCOPE_OLLAMA_URL"];
try {
process.env["LIBSCOPE_OLLAMA_URL"] = "http://custom:11434";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.embedding.ollamaUrl).toBe("http://custom:11434");
} finally {
@@ -77,6 +89,7 @@ describe("config", () => {
const original = process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"];
try {
process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"] = "true";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.indexing.allowPrivateUrls).toBe(true);
} finally {
@@ -92,6 +105,7 @@ describe("config", () => {
const original = process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"];
try {
process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"] = "1";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.indexing.allowSelfSignedCerts).toBe(true);
} finally {
@@ -109,6 +123,7 @@ describe("config", () => {
try {
process.env["LIBSCOPE_LLM_PROVIDER"] = "ollama";
process.env["LIBSCOPE_LLM_MODEL"] = "llama3";
+ invalidateConfigCache();
const config = loadConfig();
expect(config.llm?.provider).toBe("ollama");
expect(config.llm?.model).toBe("llama3");
diff --git a/tests/unit/connectors-config.test.ts b/tests/unit/connectors-config.test.ts
index 28d15dc..3bfa6ac 100644
--- a/tests/unit/connectors-config.test.ts
+++ b/tests/unit/connectors-config.test.ts
@@ -144,6 +144,17 @@ describe("connectors config", () => {
expect(result).toBe(true);
expect(loadDbConnectorConfig(db, "notion")).toBeUndefined();
});
+
+ it("loadDbConnectorConfig throws ConfigError when config_json is corrupted", () => {
+ // Directly insert corrupted JSON into the database
+ db.prepare(
+ "INSERT INTO connector_configs (type, config_json, updated_at) VALUES (?, ?, datetime('now'))",
+ ).run("corrupted", "not valid json{{{");
+
+ expect(() => loadDbConnectorConfig(db, "corrupted")).toThrow(
+ /Corrupted connector config for type "corrupted"/,
+ );
+ });
});
describe("sync tracker", () => {
diff --git a/tests/unit/http-utils.test.ts b/tests/unit/http-utils.test.ts
index fa8d745..7658343 100644
--- a/tests/unit/http-utils.test.ts
+++ b/tests/unit/http-utils.test.ts
@@ -108,7 +108,7 @@ describe("fetchWithRetry", () => {
baseDelay: 10,
}),
).rejects.toThrow(FetchError);
- expect(mockFetch).toHaveBeenCalledTimes(2);
+ expect(mockFetch).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
vi.useFakeTimers();
});
diff --git a/tests/unit/indexing.test.ts b/tests/unit/indexing.test.ts
index 70dbc14..f00a652 100644
--- a/tests/unit/indexing.test.ts
+++ b/tests/unit/indexing.test.ts
@@ -21,9 +21,9 @@ Content of section two.`;
expect(chunks.length).toBe(3);
expect(chunks[0]).toContain("Introduction");
expect(chunks[1]).toContain("Section One");
- expect(chunks[1]).toContain("<!-- context: Introduction -->");
+ expect(chunks[1]).toContain("Context: Introduction");
expect(chunks[2]).toContain("Section Two");
- expect(chunks[2]).toContain("<!-- context: Introduction -->");
+ expect(chunks[2]).toContain("Context: Introduction");
});
it("should handle content without headings", () => {
@@ -105,7 +105,7 @@ Fourth level stays with H3.`;
expect(chunks.length).toBe(3);
expect(chunks[2]).toContain("H3");
expect(chunks[2]).toContain("H4");
- expect(chunks[2]).toContain("<!-- context: H1 > H2 -->");
+ expect(chunks[2]).toContain("Context: H1 > H2");
});
});
@@ -210,7 +210,109 @@ More content here.`;
const smallWindow = chunkContentStreaming(content, { windowSize: 1024 });
const largeWindow = chunkContentStreaming(content, { windowSize: 8192 });
- expect(smallWindow.length).toBeGreaterThanOrEqual(largeWindow.length);
+ // Both window sizes should produce chunks covering the content
+ expect(smallWindow.length).toBeGreaterThan(0);
+ expect(largeWindow.length).toBeGreaterThan(0);
+ // Smaller window produces different chunk boundaries but should still cover the content
+ const smallJoined = smallWindow.join("\n");
+ const largeJoined = largeWindow.join("\n");
+ expect(smallJoined).toContain("Line 0");
+ expect(largeJoined).toContain("Line 0");
+ expect(smallJoined).toContain("Line 499");
+ expect(largeJoined).toContain("Line 499");
+ });
+});
+
+describe("chunkContent with overlap", () => {
+ it("should add overlap between consecutive chunks", () => {
+ const content = `# Section A
+Content of section A with enough text to form a meaningful chunk.
+
+## Section B
+Content of section B with different information.
+
+## Section C
+Content of section C wraps up the document.`;
+
+ const chunks = chunkContent(content, { maxChunkSize: 1500, overlapFraction: 0.1 });
+
+ // With overlap, later chunks should contain trailing text from previous chunks
+ expect(chunks.length).toBeGreaterThanOrEqual(3);
+ // First chunk should not have overlap prefix
+ expect(chunks[0]).toContain("Section A");
+
+ // Verify overlap: chunk[1] should begin with text from the end of chunk[0]
+ const noOverlapChunks = chunkContent(content, { maxChunkSize: 1500, overlapFraction: 0 });
+ // With overlap enabled, chunk[1] should be longer than the no-overlap version
+ // because it includes a prefix from the previous chunk
+ expect(chunks[1]!.length).toBeGreaterThan(noOverlapChunks[1]!.length);
+ // The overlap text should come from the end of the first chunk's content
+ const overlapPrefix = chunks[1]!.split("\n\n")[0]!;
+ expect(chunks[0]).toContain(overlapPrefix);
+ });
+
+ it("should produce no overlap when overlapFraction is 0", () => {
+ const content = `# Part 1
+First part content.
+
+## Part 2
+Second part content.`;
+
+ const withOverlap = chunkContent(content, { maxChunkSize: 1500, overlapFraction: 0 });
+ const withoutOverlap = chunkContent(content, 1500);
+
+ // With 0 overlap, results should match the no-overlap behavior
+ expect(withOverlap.length).toBe(withoutOverlap.length);
+ });
+
+ it("should clamp overlapFraction to valid range", () => {
+ const content = `# Title
+Some content here.
+
+## Section
+More content here.`;
+
+ // Should not throw with out-of-range values
+ const chunksNeg = chunkContent(content, { overlapFraction: -0.5 });
+ const chunksHigh = chunkContent(content, { overlapFraction: 0.9 });
+
+ expect(chunksNeg.length).toBeGreaterThan(0);
+ expect(chunksHigh.length).toBeGreaterThan(0);
+ });
+});
+
+describe("chunkContent paragraph-boundary splitting", () => {
+ it("should split oversized sections at paragraph boundaries", () => {
+ // Create content with paragraphs that exceeds maxChunkSize within one section
+ const paragraphs = Array.from(
+ { length: 10 },
+ (_, i) => `Paragraph ${i}: ${"word ".repeat(40)}`,
+ );
+ const content = paragraphs.join("\n\n");
+
+ const chunks = chunkContent(content, 300);
+
+ expect(chunks.length).toBeGreaterThan(1);
+ // Each chunk should be under the max size
+ for (const chunk of chunks) {
+ expect(chunk.length).toBeLessThanOrEqual(300 + 50); // small tolerance for overlap
+ }
+ });
+});
+
+describe("plain-text breadcrumbs", () => {
+ it("should use 'Context:' prefix instead of HTML comments", () => {
+ const content = `# Parent
+Intro.
+
+## Child
+Detail.`;
+
+ const chunks = chunkContent(content);
+ const childChunk = chunks.find((c) => c.includes("Child"));
+ expect(childChunk).toBeDefined();
+ expect(childChunk).toContain("Context: Parent");
+ expect(childChunk).not.toContain("<!--");
});
});
diff --git a/tests/unit/link-extractor.test.ts b/tests/unit/link-extractor.test.ts
new file mode 100644
index 0000000..f623eba
--- /dev/null
+++ b/tests/unit/link-extractor.test.ts
@@ -0,0 +1,155 @@
+import { describe, it, expect } from "vitest";
+import { extractLinks } from "../../src/core/link-extractor.js";
+
+const BASE = "https://example.com/docs/intro";
+
+describe("extractLinks", () => {
+ it("extracts absolute http links", () => {
+ const html = `<a href="https://example.com/page">link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("resolves relative links against base URL", () => {
+ const html = `<a href="../guide">guide</a>`;
+ const links = extractLinks(html, BASE);
+ expect(links).toEqual(["https://example.com/guide"]);
+ });
+
+ it("resolves root-relative links", () => {
+ const html = `<a href="/about">about</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/about"]);
+ });
+
+ it("strips fragment-only links", () => {
+ const html = `<a href="#section">jump</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("strips fragments from full URLs", () => {
+ const html = `<a href="https://example.com/page#section">link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("deduplicates links", () => {
+ const html = `
+ <a href="https://example.com/page">first</a>
+ <a href="https://example.com/page">second</a>
+ `;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("deduplicates after fragment stripping", () => {
+ const html = `
+ <a href="https://example.com/page#a">a</a>
+ <a href="https://example.com/page#b">b</a>
+ `;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("filters out mailto: links", () => {
+ const html = `<a href="mailto:user@example.com">email</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("filters out javascript: links", () => {
+ const html = `<a href="javascript:void(0)">noop</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("filters out tel: links", () => {
+ const html = `<a href="tel:+15555555555">call</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("filters out ftp: links", () => {
+ const html = `<a href="ftp://files.example.com/data">ftp</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("filters out data: links", () => {
+ const html = `<a href="data:text/plain;base64,abc">data</a>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("handles single-quoted href attributes", () => {
+ const html = `<a href='https://example.com/single'>link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/single"]);
+ });
+
+ it("handles unquoted href attributes", () => {
+ const html = `<a href=https://example.com/noquote>link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/noquote"]);
+ });
+
+ it("ignores tags that aren't <a>", () => {
+ const html = `
+ <img src="https://example.com/img.png">
+ <link href="https://example.com/style.css">
+ <a href="https://example.com/real">real</a>
+ `;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/real"]);
+ });
+
+ it("handles <a> tags with extra attributes", () => {
+ const html = `<a class="nav" id="main" href="https://example.com/page" target="_blank">link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("handles href before other attributes", () => {
+ const html = `<a href="https://example.com/page" class="nav">link</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/page"]);
+ });
+
+ it("strips trailing slash from non-root paths", () => {
+ const html = `<a href="https://example.com/docs/">docs</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/docs"]);
+ });
+
+ it("preserves trailing slash on root path", () => {
+ const html = `<a href="https://example.com/">home</a>`;
+ expect(extractLinks(html, BASE)).toEqual(["https://example.com/"]);
+ });
+
+ it("returns empty array for HTML with no links", () => {
+ const html = `<p>No links here at all.</p>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("returns empty array for empty string", () => {
+ expect(extractLinks("", BASE)).toEqual([]);
+ });
+
+ it("handles multiple links preserving discovery order", () => {
+ const html = `
+ <a href="https://example.com/a">a</a>
+ <a href="https://example.com/b">b</a>
+ <a href="https://example.com/c">c</a>
+ `;
+ expect(extractLinks(html, BASE)).toEqual([
+ "https://example.com/a",
+ "https://example.com/b",
+ "https://example.com/c",
+ ]);
+ });
+
+ it("handles malformed href gracefully", () => {
+ const html = `<a href="not a valid [url]">bad</a>`;
+ // Should not throw; just skip
+ expect(() => extractLinks(html, BASE)).not.toThrow();
+ });
+
+ it("skips <abbr> and <article> tags (not <a>)", () => {
+ const html = `<abbr href="https://example.com/x">X</abbr>`;
+ expect(extractLinks(html, BASE)).toEqual([]);
+ });
+
+ it("handles https links alongside http", () => {
+ const html = `
+ <a href="http://example.com/http">http</a>
+ <a href="https://example.com/https">https</a>
+ `;
+ const links = extractLinks(html, BASE);
+ expect(links).toContain("http://example.com/http");
+ expect(links).toContain("https://example.com/https");
+ });
+});
diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts
new file mode 100644
index 0000000..1e0f65a
--- /dev/null
+++ b/tests/unit/mcp-server.test.ts
@@ -0,0 +1,209 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { createTestDbWithVec } from "../fixtures/test-db.js";
+import { MockEmbeddingProvider } from "../fixtures/mock-provider.js";
+import { initLogger } from "../../src/logger.js";
+import { errorResponse, withErrorHandling, type ToolResult } from "../../src/mcp/errors.js";
+import { LibScopeError, ValidationError, DocumentNotFoundError } from "../../src/errors.js";
+import type Database from "better-sqlite3";
+
+describe("MCP server helpers", () => {
+ beforeEach(() => {
+ initLogger("silent");
+ });
+
+ describe("errorResponse", () => {
+ it("returns isError: true with text content", () => {
+ const result = errorResponse(new Error("something went wrong"));
+ expect(result.isError).toBe(true);
+ expect(result.content).toHaveLength(1);
+ expect(result.content[0]!.type).toBe("text");
+ });
+
+ it("formats LibScopeError using just the message", () => {
+ const result = errorResponse(new ValidationError("invalid input"));
+ expect(result.content[0]!.text).toBe("Error: invalid input");
+ });
+
+ it("formats a generic Error using name: message", () => {
+ const err = new TypeError("bad type");
+ const result = errorResponse(err);
+ expect(result.content[0]!.text).toBe("Error: TypeError: bad type");
+ });
+
+ it("formats non-Error values using String()", () => {
+ const result = errorResponse("raw string error");
+ expect(result.content[0]!.text).toContain("raw string error");
+ });
+
+ it("formats null/undefined without throwing", () => {
+ expect(() => errorResponse(null)).not.toThrow();
+ expect(() => errorResponse(undefined)).not.toThrow();
+ });
+ });
+
+ describe("withErrorHandling", () => {
+ it("returns the handler result when no error is thrown", async () => {
+ const expected: ToolResult = { content: [{ type: "text", text: "ok" }] };
+ const wrapped = withErrorHandling(() => expected);
+ const result = await wrapped({});
+ expect(result).toEqual(expected);
+ });
+
+ it("catches synchronous throws and returns an error response", async () => {
+ const wrapped = withErrorHandling(() => {
+ throw new ValidationError("bad input");
+ });
+ const result = await wrapped({});
+ expect(result.isError).toBe(true);
+ expect(result.content[0]!.text).toContain("bad input");
+ });
+
+ it("catches rejected promises and returns an error response", async () => {
+ const wrapped = withErrorHandling(() => {
+ return Promise.reject(new DocumentNotFoundError("doc-123"));
+ });
+ const result = await wrapped({});
+ expect(result.isError).toBe(true);
+ });
+
+ it("passes params to the inner handler", async () => {
+ const handler = vi.fn().mockReturnValue({ content: [{ type: "text", text: "done" }] });
+ const wrapped = withErrorHandling(handler);
+ const params = { docId: "abc", query: "test" };
+ await wrapped(params);
+ expect(handler).toHaveBeenCalledWith(params);
+ });
+
+ it("returns isError: true for LibScopeError subclasses", async () => {
+ const wrapped = withErrorHandling(() => {
+ throw new LibScopeError("base lib error");
+ });
+ const result = await wrapped({});
+ expect(result.isError).toBe(true);
+ expect(result.content[0]!.text).toBe("Error: base lib error");
+ });
+ });
+});
+
+// Integration-style tests for MCP tool behaviors using the underlying core functions
+// These verify the business logic that MCP tools delegate to.
+describe("MCP tool business logic", () => {
+ let db: Database.Database;
+ let provider: MockEmbeddingProvider;
+
+ beforeEach(() => {
+ initLogger("silent");
+ db = createTestDbWithVec();
+ provider = new MockEmbeddingProvider();
+ });
+
+ afterEach(() => {
+ db.close();
+ });
+
+ it("search returns empty response when no documents are indexed", async () => {
+ const { searchDocuments } = await import("../../src/core/search.js");
+ const { results, totalCount } = await searchDocuments(db, provider, { query: "anything" });
+ expect(results).toHaveLength(0);
+ expect(totalCount).toBe(0);
+ });
+
+ it("indexDocument then getDocument returns indexed document", async () => {
+ const { indexDocument } = await import("../../src/core/indexing.js");
+ const { getDocument } = await import("../../src/core/documents.js");
+
+ const indexed = await indexDocument(db, provider, {
+ title: "Test Doc",
+ content: "Some content for testing.",
+ sourceType: "manual",
+ });
+
+ expect(indexed.id).toBeTruthy();
+
+ const fetched = getDocument(db, indexed.id);
+ expect(fetched.title).toBe("Test Doc");
+ expect(fetched.content).toBe("Some content for testing.");
+ });
+
+ it("deleteDocument removes a document", async () => {
+ const { indexDocument } = await import("../../src/core/indexing.js");
+ const { deleteDocument, getDocument } = await import("../../src/core/documents.js");
+
+ const indexed = await indexDocument(db, provider, {
+ title: "Delete Me",
+ content: "This will be deleted.",
+ sourceType: "manual",
+ });
+
+ deleteDocument(db, indexed.id);
+
+ expect(() => getDocument(db, indexed.id)).toThrow(DocumentNotFoundError);
+ });
+
+ it("listDocuments returns paginated documents", async () => {
+ const { indexDocument } = await import("../../src/core/indexing.js");
+ const { listDocuments } = await import("../../src/core/documents.js");
+
+ await indexDocument(db, provider, {
+ title: "Doc A",
+ content: "Content A",
+ sourceType: "library",
+ library: "react",
+ });
+ await indexDocument(db, provider, {
+ title: "Doc B",
+ content: "Content B",
+ sourceType: "library",
+ library: "vue",
+ });
+
+ const all = listDocuments(db, {});
+ expect(all.length).toBeGreaterThanOrEqual(2);
+
+ const limited = listDocuments(db, { limit: 1 });
+ expect(limited).toHaveLength(1);
+ });
+
+ it("getDocumentRatings returns zero ratings for new document", async () => {
+ const { indexDocument } = await import("../../src/core/indexing.js");
+ const { getDocumentRatings } = await import("../../src/core/ratings.js");
+
+ const indexed = await indexDocument(db, provider, {
+ title: "Rate Me",
+ content: "Rateable content.",
+ sourceType: "manual",
+ });
+
+ const ratings = getDocumentRatings(db, indexed.id);
+ expect(ratings.totalRatings).toBe(0);
+ expect(ratings.averageRating).toBe(0);
+ });
+
+ it("rateDocument stores a rating and updates average", async () => {
+ const { indexDocument } = await import("../../src/core/indexing.js");
+ const { rateDocument, getDocumentRatings } = await import("../../src/core/ratings.js");
+
+ const indexed = await indexDocument(db, provider, {
+ title: "Rate Me",
+ content: "Rateable content.",
+ sourceType: "manual",
+ });
+
+ rateDocument(db, { documentId: indexed.id, rating: 4, feedback: "good doc" });
+ const ratings = getDocumentRatings(db, indexed.id);
+ expect(ratings.totalRatings).toBe(1);
+ expect(ratings.averageRating).toBe(4);
+ });
+
+ it("listTopics returns empty array when no topics exist", async () => {
+ const { listTopics } = await import("../../src/core/topics.js");
+ const topics = listTopics(db);
+ expect(topics).toEqual([]);
+ });
+
+ it("errorResponse for DocumentNotFoundError returns proper message", () => {
+ const result = errorResponse(new DocumentNotFoundError("missing-id"));
+ expect(result.isError).toBe(true);
+ expect(result.content[0]!.text).toContain("missing-id");
+ });
+});
diff --git a/tests/unit/packs.test.ts b/tests/unit/packs.test.ts
index 90edf79..45bf0f5 100644
--- a/tests/unit/packs.test.ts
+++ b/tests/unit/packs.test.ts
@@ -1,7 +1,8 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { writeFileSync, existsSync, mkdtempSync } from "node:fs";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { writeFileSync, existsSync, mkdtempSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
+import { gzipSync, gunzipSync } from "node:zlib";
import type Database from "better-sqlite3";
import { createTestDbWithVec } from "../fixtures/test-db.js";
import { MockEmbeddingProvider } from "../fixtures/mock-provider.js";
@@ -11,6 +12,7 @@ import {
listInstalledPacks,
createPack,
listAvailablePacks,
+ createPackFromSource,
} from "../../src/core/packs.js";
import type { KnowledgePack } from "../../src/core/packs.js";
import { indexDocument } from "../../src/core/indexing.js";
@@ -443,4 +445,626 @@ describe("knowledge packs", () => {
);
});
});
+
+ describe("createPackFromSource", () => {
+ let sourceDir: string;
+
+ beforeEach(() => {
+ sourceDir = mkdtempSync(join(tmpdir(), "libscope-pack-source-"));
+ });
+
+ it("should create a pack from a folder of markdown files", async () => {
+ writeFileSync(join(sourceDir, "guide.md"), "# Guide\n\nThis is a guide.");
+ writeFileSync(join(sourceDir, "api.md"), "# API\n\nEndpoint reference.");
+
+ const pack = await createPackFromSource({
+ name: "test-from-folder",
+ from: [sourceDir],
+ });
+
+ expect(pack.name).toBe("test-from-folder");
+ expect(pack.documents).toHaveLength(2);
+ expect(pack.documents.map((d) => d.title).sort()).toEqual(["api", "guide"]);
+ expect(pack.documents[0]!.content).toBeTruthy();
+ expect(pack.documents[0]!.source).toMatch(/^file:\/\//);
+ expect(pack.version).toBe("1.0.0");
+ expect(pack.metadata.author).toBe("libscope");
+ });
+
+ it("should write pack to outputPath", async () => {
+ writeFileSync(join(sourceDir, "doc.md"), "# Doc\n\nContent here.");
+ const outputPath = join(tempDir, "output-pack.json");
+
+ const pack = await createPackFromSource({
+ name: "output-test",
+ from: [sourceDir],
+ outputPath,
+ });
+
+ expect(existsSync(outputPath)).toBe(true);
+ const written = JSON.parse(readFileSync(outputPath, "utf-8")) as KnowledgePack;
+ expect(written.name).toBe("output-test");
+ expect(written.documents).toHaveLength(1);
+ expect(pack.documents).toHaveLength(1);
+ });
+
+ it("should filter by extensions", async () => {
+ writeFileSync(join(sourceDir, "readme.md"), "# Readme");
+ writeFileSync(join(sourceDir, "page.html"), "<h1>Page</h1><p>Content</p>");
+ writeFileSync(join(sourceDir, "data.json"), '{"key": "value"}');
+
+ const pack = await createPackFromSource({
+ name: "ext-filter",
+ from: [sourceDir],
+ extensions: [".md"],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("readme");
+ });
+
+ it("should handle extensions without leading dot", async () => {
+ writeFileSync(join(sourceDir, "readme.md"), "# Readme\n\nContent");
+
+ const pack = await createPackFromSource({
+ name: "ext-no-dot",
+ from: [sourceDir],
+ extensions: ["md"],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ });
+
+ it("should exclude files matching patterns", async () => {
+ writeFileSync(join(sourceDir, "guide.md"), "# Guide\n\nContent");
+ writeFileSync(join(sourceDir, "draft.md"), "# Draft\n\nNot ready");
+
+ const pack = await createPackFromSource({
+ name: "exclude-test",
+ from: [sourceDir],
+ exclude: ["draft.md"],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("guide");
+ });
+
+ it("should recurse into subdirectories by default", async () => {
+ const { mkdirSync } = await import("node:fs");
+ const subDir = join(sourceDir, "sub");
+ mkdirSync(subDir);
+ writeFileSync(join(sourceDir, "root.md"), "# Root");
+ writeFileSync(join(subDir, "nested.md"), "# Nested\n\nDeep content");
+
+ const pack = await createPackFromSource({
+ name: "recursive-test",
+ from: [sourceDir],
+ });
+
+ expect(pack.documents).toHaveLength(2);
+ expect(pack.documents.map((d) => d.title).sort()).toEqual(["nested", "root"]);
+ });
+
+ it("should not recurse when recursive is false", async () => {
+ const { mkdirSync } = await import("node:fs");
+ const subDir = join(sourceDir, "sub");
+ mkdirSync(subDir);
+ writeFileSync(join(sourceDir, "root.md"), "# Root");
+ writeFileSync(join(subDir, "nested.md"), "# Nested");
+
+ const pack = await createPackFromSource({
+ name: "no-recurse",
+ from: [sourceDir],
+ recursive: false,
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("root");
+ });
+
+ it("should throw for empty pack name", async () => {
+ await expect(createPackFromSource({ name: " ", from: [sourceDir] })).rejects.toThrow(
+ /Pack name is required/,
+ );
+ });
+
+ it("should throw for empty from array", async () => {
+ await expect(createPackFromSource({ name: "test", from: [] })).rejects.toThrow(
+ /At least one --from source is required/,
+ );
+ });
+
+ it("should throw for non-existent source path", async () => {
+ await expect(
+ createPackFromSource({ name: "test", from: ["/nonexistent/path/xyz"] }),
+ ).rejects.toThrow(/does not exist/);
+ });
+
+ it("should throw when no documents could be created", async () => {
+ // Empty directory — no parseable files
+ await expect(createPackFromSource({ name: "empty", from: [sourceDir] })).rejects.toThrow(
+ /No documents could be created/,
+ );
+ });
+
+ it("should skip files without a parser", async () => {
+ writeFileSync(join(sourceDir, "data.bin"), "binary stuff");
+ writeFileSync(join(sourceDir, "readme.md"), "# Readme\n\nContent");
+
+ const pack = await createPackFromSource({
+ name: "skip-unsupported",
+ from: [sourceDir],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("readme");
+ });
+
+ it("should skip files with empty content after parsing", async () => {
+ writeFileSync(join(sourceDir, "empty.md"), " ");
+ writeFileSync(join(sourceDir, "real.md"), "# Real\n\nActual content");
+
+ const pack = await createPackFromSource({
+ name: "skip-empty",
+ from: [sourceDir],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("real");
+ });
+
+ it("should accept a single file as source", async () => {
+ const filePath = join(sourceDir, "single.md");
+ writeFileSync(filePath, "# Single File\n\nJust one file.");
+
+ const pack = await createPackFromSource({
+ name: "single-file",
+ from: [filePath],
+ });
+
+ expect(pack.documents).toHaveLength(1);
+ expect(pack.documents[0]!.title).toBe("single");
+ });
+
+ it("should accept multiple sources", async () => {
+ const dir2 = mkdtempSync(join(tmpdir(), "libscope-pack-source2-"));
+ writeFileSync(join(sourceDir, "a.md"), "# A\n\nFrom dir 1");
+ writeFileSync(join(dir2, "b.md"), "# B\n\nFrom dir 2");
+
+ const pack = await createPackFromSource({
+ name: "multi-source",
+ from: [sourceDir, dir2],
+ });
+
+ expect(pack.documents).toHaveLength(2);
+ });
+
+ it("should call onProgress callback", async () => {
+ writeFileSync(join(sourceDir, "a.md"), "# A");
+ writeFileSync(join(sourceDir, "b.md"), "# B");
+
+ const progress: Array<{ file: string; index: number; total: number }> = [];
+
+ await createPackFromSource({
+ name: "progress-test",
+ from: [sourceDir],
+ onProgress: (info) => progress.push(info),
+ });
+
+ expect(progress).toHaveLength(2);
+ expect(progress[0]!.index).toBe(0);
+ expect(progress[0]!.total).toBe(2);
+ expect(progress[1]!.index).toBe(1);
+ });
+
+ it("should set custom version, description, author", async () => {
+ writeFileSync(join(sourceDir, "doc.md"), "# Doc\n\nContent");
+
+ const pack = await createPackFromSource({
+ name: "custom-meta",
+ from: [sourceDir],
+ version: "2.0.0",
+ description: "Custom desc",
+ author: "Test Author",
+ });
+
+ expect(pack.version).toBe("2.0.0");
+ expect(pack.description).toBe("Custom desc");
+ expect(pack.metadata.author).toBe("Test Author");
+ });
+
+ it("should produce a valid pack that passes validatePack", async () => {
+ writeFileSync(join(sourceDir, "doc.md"), "# Doc\n\nSome content here");
+ const outputPath = join(tempDir, "validate-test.json");
+
+ await createPackFromSource({
+ name: "validate-test",
+ from: [sourceDir],
+ outputPath,
+ });
+
+ // Read and re-validate through installPack (which calls validatePack internally)
+ const result = await installPack(db, provider, outputPath);
+ expect(result.packName).toBe("validate-test");
+ expect(result.documentsInstalled).toBe(1);
+ });
+
+ it("should handle HTML files", async () => {
+ writeFileSync(
+ join(sourceDir, "page.html"),
+ "<html><head><title>TestHello
World