From 8c2af43b85eed6a0243116de234ab6e8f778d8a0 Mon Sep 17 00:00:00 2001 From: Henrique Silveira Neves Date: Fri, 24 Jul 2026 21:16:39 -0300 Subject: [PATCH 1/5] fix: materialize internal import endpoints --- graphify/diagnostics.py | 78 +++- graphify/extract.py | 408 ++++++++++++++---- graphify/extractors/resolution.py | 18 +- graphify/manifest_ingest.py | 4 + graphify/skill-agents.md | 6 +- graphify/skill-amp.md | 6 +- graphify/skill-claw.md | 6 +- graphify/skill-codex.md | 6 +- graphify/skill-copilot.md | 6 +- graphify/skill-droid.md | 6 +- graphify/skill-kilo.md | 6 +- graphify/skill-kiro.md | 6 +- graphify/skill-opencode.md | 6 +- graphify/skill-pi.md | 6 +- graphify/skill-trae.md | 6 +- graphify/skill-vscode.md | 6 +- graphify/skill-windows.md | 6 +- graphify/skill.md | 6 +- tests/test_endpoint_classification.py | 61 +++ tests/test_multigraph_diagnostics.py | 49 ++- tests/test_src_layout_import_resolution.py | 236 ++++++++++ .../expected/graphify__skill-agents.md | 6 +- .../skillgen/expected/graphify__skill-amp.md | 6 +- .../skillgen/expected/graphify__skill-claw.md | 6 +- .../expected/graphify__skill-codex.md | 6 +- .../expected/graphify__skill-copilot.md | 6 +- .../expected/graphify__skill-droid.md | 6 +- .../skillgen/expected/graphify__skill-kilo.md | 6 +- .../skillgen/expected/graphify__skill-kiro.md | 6 +- .../expected/graphify__skill-opencode.md | 6 +- tools/skillgen/expected/graphify__skill-pi.md | 6 +- .../skillgen/expected/graphify__skill-trae.md | 6 +- .../expected/graphify__skill-vscode.md | 6 +- .../expected/graphify__skill-windows.md | 6 +- tools/skillgen/expected/graphify__skill.md | 6 +- tools/skillgen/fragments/core/core.md | 6 +- 36 files changed, 871 insertions(+), 157 deletions(-) create mode 100644 tests/test_endpoint_classification.py diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..d070a2cc1 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -10,6 +10,7 @@ from typing import Any import networkx as nx +from graphify.ids import normalize_id _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") @@ -42,7 +43,7 @@ def _node_ids(extraction: dict[str, Any]) -> set[str]: } -def _canonical_edge(edge: Any) -> dict[str, str]: +def _canonical_edge(edge: Any) -> dict[str, Any]: if not isinstance(edge, dict): return { "source": "", @@ -52,6 +53,8 @@ def _canonical_edge(edge: Any) -> dict[str, str]: "source_file": "", "source_location": "", "context": "", + "external": False, + "unresolved_internal": False, "_invalid": "non_object_edge", } source = edge.get("source", edge.get("from")) @@ -64,10 +67,33 @@ def _canonical_edge(edge: Any) -> dict[str, str]: "source_file": _safe_text(edge.get("source_file")), "source_location": _safe_text(edge.get("source_location")), "context": _safe_text(edge.get("context")), + "external": edge.get("external") is True, + "unresolved_internal": edge.get("unresolved_internal") is True, "_invalid": "", } +def _malformed_endpoint(source: str, target: str, root: str | Path | None) -> bool: + """Detect machine-absolute endpoint ids that escaped canonical remapping.""" + endpoints = (source.replace("\\", "/"), target.replace("\\", "/")) + if any( + value.startswith("/") + or re.match(r"^[A-Za-z]:/", value) + for value in endpoints + ): + return True + if root is None: + return False + try: + prefix = normalize_id(str(Path(root).resolve())) + except OSError: + prefix = normalize_id(str(root)) + return bool(prefix) and any( + value == prefix or value.startswith(prefix + "_") + for value in endpoints + ) + + def _exact_signature(edge: Any) -> str: if not isinstance(edge, dict): return "" @@ -184,8 +210,29 @@ def diagnose_extraction( non_object_edges = 0 missing_endpoint_edges = 0 dangling_endpoint_edges = 0 + external_endpoint_edges = 0 + unresolved_internal_endpoint_edges = 0 + malformed_endpoint_edges = 0 + unclassified_endpoint_edges = 0 self_loop_edges = 0 valid_candidate_edges = 0 + endpoint_examples: dict[str, list[dict[str, str]]] = { + "external": [], + "unresolved_internal": [], + "malformed": [], + "unclassified": [], + } + + def _remember(category: str, edge: dict[str, Any]) -> None: + if max_examples <= 0 or len(endpoint_examples[category]) >= max_examples: + return + endpoint_examples[category].append({ + "source": edge["source"], + "target": edge["target"], + "relation": edge["relation"], + "source_file": edge["source_file"], + "source_location": edge["source_location"], + }) for edge in canonical_edges: if edge["_invalid"]: @@ -198,6 +245,18 @@ def diagnose_extraction( continue if source not in node_ids or target not in node_ids: dangling_endpoint_edges += 1 + if _malformed_endpoint(source, target, root): + malformed_endpoint_edges += 1 + _remember("malformed", edge) + elif edge["external"]: + external_endpoint_edges += 1 + _remember("external", edge) + elif edge["unresolved_internal"]: + unresolved_internal_endpoint_edges += 1 + _remember("unresolved_internal", edge) + else: + unclassified_endpoint_edges += 1 + _remember("unclassified", edge) continue if source == target: self_loop_edges += 1 @@ -252,6 +311,10 @@ def diagnose_extraction( "non_object_edges": non_object_edges, "missing_endpoint_edges": missing_endpoint_edges, "dangling_endpoint_edges": dangling_endpoint_edges, + "external_endpoint_edges": external_endpoint_edges, + "unresolved_internal_endpoint_edges": unresolved_internal_endpoint_edges, + "malformed_endpoint_edges": malformed_endpoint_edges, + "unclassified_endpoint_edges": unclassified_endpoint_edges, "self_loop_edges": self_loop_edges, "valid_candidate_edges": valid_candidate_edges, "exact_duplicate_edges": _count_extra(exact_counts), @@ -274,6 +337,7 @@ def diagnose_extraction( "post_build_error": build_error, "producer_suppression": scan_producer_suppression_sites(suppression_path), "examples": examples, + "endpoint_examples": endpoint_examples, } @@ -329,13 +393,14 @@ def diagnose_file( def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: return { - "schema_version": 1, + "schema_version": 2, "summary": { key: value for key, value in summary.items() - if key not in {"examples", "producer_suppression"} + if key not in {"examples", "endpoint_examples", "producer_suppression"} }, "examples": summary.get("examples", []), + "endpoint_examples": summary.get("endpoint_examples", {}), "producer_suppression": summary.get("producer_suppression", {}), "notes": [ "Diagnostics are read-only.", @@ -358,6 +423,13 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"valid_candidate_edges: {summary['valid_candidate_edges']}", f"missing_endpoint_edges: {summary['missing_endpoint_edges']}", f"dangling_endpoint_edges: {summary['dangling_endpoint_edges']}", + f"external_endpoint_edges: {summary.get('external_endpoint_edges', 0)}", + ( + "unresolved_internal_endpoint_edges: " + f"{summary.get('unresolved_internal_endpoint_edges', 0)}" + ), + f"malformed_endpoint_edges: {summary.get('malformed_endpoint_edges', 0)}", + f"unclassified_endpoint_edges: {summary.get('unclassified_endpoint_edges', 0)}", f"self_loop_edges: {summary['self_loop_edges']}", f"exact_duplicate_edges: {summary['exact_duplicate_edges']}", f"directed_unique_endpoint_pairs: {summary['directed_unique_endpoint_pairs']}", diff --git a/graphify/extract.py b/graphify/extract.py index bbfa301cc..ea31fb64f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -182,75 +182,251 @@ def _file_node_id(rel_path: Path) -> str: def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: - """Repoint Python absolute-import edges to the real file node under a nested - (e.g. ``src/``) package root (#2072). - - Absolute imports target an id derived from the dotted module path - (``_make_id('pkg.mod')`` -> ``pkg_mod``), but file-node ids are - scan-root-relative (``src_pkg_mod`` when the code lives under ``src/``), so - the edge dangles and is silently dropped — the graph loses most ``imports`` - edges purely because of where the scan started. Build an alias map from the - dotted-module id to the real file-node id by detecting each ``.py`` file's - package root (the contiguous run of ancestor dirs carrying ``__init__.py``) - and rewrite matching ``imports``/``imports_from`` edge targets. Guards: never - shadow an existing node id, and drop an alias claimed by more than one file - (ambiguous -> leave dangling, as before). Files whose package root IS the - scan root are skipped (ids already coincide).""" + """Resolve Python import edges against files in the importer's workspace. + + The old alias pass inferred a package root only from a contiguous chain of + ``__init__.py`` files. That loses imports in PEP 420 namespace packages and + mixed monorepos where each project has its own ``src/`` or application root. + Import handlers now retain the original module and relative level as + transient metadata. Resolve those specifiers to physical files, then map the + files directly to their canonical scan-root-relative node ids. + + A workspace is the nearest Python project manifest. For unconfigured + projects it is the first directory below the scan root. Multiple physical + matches are never guessed. Unresolved absolute imports whose top-level + module is absent from the workspace corpus are explicitly marked external; + unresolved relative/local imports remain visible as internal defects.""" try: root = Path(root).resolve() except OSError: root = Path(root) - node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} - alias_to_files: dict[str, set[str]] = {} - for p in paths: - if p.suffix.lower() not in (".py", ".pyi"): - continue + + python_paths = [ + Path(p) for p in paths if Path(p).suffix.lower() in (".py", ".pyi") + ] + if not python_paths: + return + + manifests = ("pyproject.toml", "setup.py", "setup.cfg") + + def _workspace_for(path: Path) -> Path: + try: + current = path.resolve().parent + except OSError: + current = path.parent + for candidate in (current, *current.parents): + try: + candidate.relative_to(root) + except ValueError: + break + if any((candidate / name).is_file() for name in manifests): + return candidate + if candidate == root: + break + try: + rel = current.relative_to(root) + except ValueError: + return root + return root / rel.parts[0] if rel.parts else root + + def _candidate(path: Path) -> Path | None: + if path.is_dir(): + for init_name in ("__init__.py", "__init__.pyi"): + init_path = path / init_name + if init_path.is_file(): + return init_path.resolve() + if path.is_file(): + return path.resolve() + for suffix in (".py", ".pyi"): + module_path = path.with_suffix(suffix) + if module_path.is_file(): + return module_path.resolve() + return None + + def _module_candidates( + module_name: str, + current_path: Path, + workspace: Path, + level: int, + ) -> set[Path]: + if level > 0: + base = current_path.resolve().parent + for _ in range(level - 1): + base = base.parent + candidate = base / module_name.replace(".", "/") if module_name else base + hit = _candidate(candidate) + return {hit} if hit is not None else set() + + rel_module = module_name.replace(".", "/") + search_roots: list[Path] = [workspace] + for ancestor in current_path.resolve().parents: + try: + ancestor.relative_to(workspace) + except ValueError: + break + if ancestor == workspace: + continue + # An absolute import starts at a sys.path root, not inside a regular + # package. Namespace/source roots have no __init__ and are valid. + if not ( + (ancestor / "__init__.py").is_file() + or (ancestor / "__init__.pyi").is_file() + ): + search_roots.append(ancestor) + hits = { + hit + for base in search_roots + if (hit := _candidate(base / rel_module)) is not None + } + return hits + + file_node_by_path: dict[Path, str] = {} + workspace_by_path: dict[Path, Path] = {} + module_paths_by_workspace: dict[Path, dict[str, set[Path]]] = {} + global_module_paths: dict[str, set[Path]] = {} + public_names_by_path: dict[Path, set[str]] = {} + for path in python_paths: try: - rel = Path(p).resolve().relative_to(root) + resolved = path.resolve() + rel = resolved.relative_to(root) except (ValueError, OSError): continue - parts = rel.parts - if len(parts) < 2: - continue # top-level file: scan-root-relative id already matches - d = Path(p).resolve().parent - levels = 0 - # Bounded by the number of dirs between the file and the scan root, so a - # pathological `/__init__.py` chain can't loop forever. - while levels < len(parts) - 1 and (d / "__init__.py").is_file(): - levels += 1 - d = d.parent - if levels == 0: - continue # not inside a package (namespace pkg / loose module) - mod_parts = parts[-(levels + 1):] # package dirs + the file itself - if len(mod_parts) == len(parts): - continue # package root == scan root: file-node id already coincides - file_node = _file_node_id(rel) - alias = _make_id(str(Path(*mod_parts).with_suffix(""))) - alias_to_files.setdefault(alias, set()).add(file_node) - if p.name in ("__init__.py", "__init__.pyi") and len(mod_parts) > 1: - # `import pkg` / `from pkg import x` targets the package-dir id. - pkg_alias = _make_id(str(Path(*mod_parts[:-1]))) - alias_to_files.setdefault(pkg_alias, set()).add(file_node) - alias_map = { - a: next(iter(fs)) - for a, fs in alias_to_files.items() - if len(fs) == 1 and a not in node_ids - } - if not alias_map: - return - for e in all_edges: - # Only repoint edges emitted from a Python file: a non-Python import edge - # (e.g. C# `using Pkg.Mod;`, Java/Go dotted imports) can have a dangling - # target string that coincides with a Python alias, and repointing it - # would fabricate a cross-language import edge (#2072 review). - if ( - isinstance(e, dict) - and e.get("relation") in ("imports", "imports_from") - and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) + workspace = _workspace_for(resolved) + file_node_by_path[resolved] = _file_node_id(rel) + workspace_by_path[resolved] = workspace + try: + workspace_rel = resolved.relative_to(workspace) + except ValueError: + workspace_rel = rel + module_parts = list(workspace_rel.with_suffix("").parts) + if module_parts and module_parts[-1] == "__init__": + module_parts.pop() + aliases = module_paths_by_workspace.setdefault(workspace, {}) + for index in range(len(module_parts)): + alias = ".".join(module_parts[index:]) + aliases.setdefault(alias, set()).add(resolved) + global_module_paths.setdefault(alias, set()).add(resolved) + parsed = _parse_python_tree(resolved) + if parsed is not None: + parsed_source, tree_root = parsed + public_names: set[str] = set() + for child in tree_root.children: + if child.type in ("class_definition", "function_definition"): + name_node = child.child_by_field_name("name") + if name_node is not None: + public_names.add(_read_text(name_node, parsed_source)) + elif child.type == "import_from_statement": + public_names.update( + local_name + for _, local_name in _python_imported_names( + child, parsed_source + ) + ) + public_names_by_path[resolved] = public_names + + for edge in all_edges: + if not ( + isinstance(edge, dict) + and edge.get("relation") in ("imports", "imports_from") + and str(edge.get("source_file", "")).lower().endswith((".py", ".pyi")) + and "_import_module" in edge ): - tgt = e.get("target") - if tgt in alias_map: - e["target"] = alias_map[tgt] + continue + module_name = str(edge.pop("_import_module", "")) + try: + level = int(edge.pop("_import_level", 0)) + except (TypeError, ValueError): + level = 0 + imported_names = { + str(name) + for name in edge.pop("_imported_names", []) + if name and name != "*" + } + source_path = Path(str(edge.get("source_file", ""))) + if not source_path.is_absolute(): + source_path = root / source_path + try: + source_path = source_path.resolve() + except OSError: + pass + workspace = workspace_by_path.get(source_path, _workspace_for(source_path)) + candidates = _module_candidates(module_name, source_path, workspace, level) + if level == 0: + candidates.update( + module_paths_by_workspace.get(workspace, {}).get(module_name, set()) + ) + if not candidates: + # Explicit runtime path bootstraps are common in monorepos + # (one project adds a sibling project's src/ to sys.path). A + # unique corpus-wide module is safe to connect; duplicates stay + # unresolved rather than crossing projects arbitrarily. + candidates.update(global_module_paths.get(module_name, set())) + elif not candidates and imported_names: + candidates.update( + module_paths_by_workspace.get(workspace, {}).get( + module_name.split(".")[-1], set() + ) + ) + # Importing a module with the same basename as the current file does not + # prove a self-import. It is commonly a stdlib/third-party name collision. + candidates.discard(source_path) + + top_level = module_name.split(".", 1)[0] if module_name else "" + if level == 0 and top_level in getattr(sys, "stdlib_module_names", ()): + edge["external"] = True + edge.pop("unresolved_internal", None) + continue + + if len(candidates) > 1 and imported_names: + symbol_matches = [ + candidate + for candidate in candidates + if imported_names <= public_names_by_path.get(candidate, set()) + ] + if len(symbol_matches) == 1: + candidates = {symbol_matches[0]} + elif len(symbol_matches) > 1: + package_matches = [ + candidate + for candidate in symbol_matches + if candidate.name in ("__init__.py", "__init__.pyi") + ] + if len(package_matches) == 1: + candidates = {package_matches[0]} + + if len(candidates) == 1: + target_path = next(iter(candidates)) + target_id = file_node_by_path.get(target_path) + if target_id is not None: + edge["target"] = target_id + edge.pop("external", None) + edge.pop("unresolved_internal", None) + continue + + if level > 0 or candidates: + edge["unresolved_internal"] = True + edge.pop("external", None) + if level > 0: + base = source_path.parent + for _ in range(level - 1): + base = base.parent + unresolved_path = ( + base / module_name.replace(".", "/") + if module_name else base + ) + try: + unresolved_rel = unresolved_path.resolve().relative_to(root) + edge["target"] = _make_id( + "ref_local", _file_stem(unresolved_rel) + ) + except (ValueError, OSError): + pass + else: + # Both stdlib and third-party packages are intentionally outside the + # corpus graph. Marking them makes diagnostics distinguish expected + # exclusions from lost internal structure. + edge["external"] = True + edge.pop("unresolved_internal", None) SEMANTIC_RELATIONS = frozenset({ @@ -325,6 +501,10 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, + # Transient provenance used after all files are known to + # resolve this import inside the importer's workspace. + "_import_module": module_name, + "_import_level": 0, } if raw_alias: # `import pkg.mod as alias` binds the local name `alias`, not @@ -334,30 +514,43 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s edge["local_alias"] = raw_alias.strip() edges.append(edge) elif t == "import_from_statement": - module_node = node.child_by_field_name("module_name") - if module_node: - raw = _read_text(module_node, source) - if raw.startswith("."): - # Relative import - resolve to full path so IDs match file node IDs - dots = len(raw) - len(raw.lstrip(".")) - module_name = raw.lstrip(".") - base = Path(str_path).parent - for _ in range(dots - 1): - base = base.parent - rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" - tgt_nid = _make_id(str(base / rel)) - else: - tgt_nid = _make_id(raw) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports_from", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) + parsed_module = _python_import_from_module(node, source) + if parsed_module is not None: + dots, module_name = parsed_module + imported_names = [ + name for name, _ in _python_imported_names(node, source) + ] + modules = [module_name] + # `from . import services` has no module name before `import`. + # Treat imported names as submodule candidates; symbol-only names + # that have no file remain classified as unresolved internal. + if dots > 0 and not module_name: + modules = [name for name, _ in _python_imported_names(node, source)] + for imported_module in modules: + if dots > 0: + # Relative import - resolve to full path so IDs match file node IDs + module_name = imported_module + base = Path(str_path).parent + for _ in range(dots - 1): + base = base.parent + rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" + tgt_nid = _make_id(str(base / rel)) + else: + module_name = imported_module + tgt_nid = _make_id(module_name) + edges.append({ + "source": file_nid, + "target": tgt_nid, + "relation": "imports_from", + "context": "import", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + "weight": 1.0, + "_import_module": module_name, + "_import_level": dots, + "_imported_names": imported_names, + }) def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: @@ -408,7 +601,28 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p # back onto the importer's own variant, a phantom self-loop (#1814). if resolved_path is not None: edge["target_file"] = str(resolved_path) + if not resolved_path.is_file(): + edge["unresolved_internal"] = True + else: + aliases = _load_tsconfig_aliases(Path(str_path).parent) + local_alias = any( + _match_tsconfig_alias(raw, pattern) is not None + for pattern in aliases + ) + workspace_packages = _load_workspace_packages(Path(str_path).parent) + local_workspace = any( + raw == name or raw.startswith(name + "/") + for name in workspace_packages + ) + if not raw.startswith((".", "/")) and not local_alias and not local_workspace: + edge["external"] = True + else: + edge["unresolved_internal"] = True edges.append(edge) + if resolved_path is not None and not resolved_path.is_file(): + # Keep the file-level dangling edge for diagnostics, but do not + # synthesize symbol edges for a file that is not in the corpus. + resolved_path = None # Emit symbol-level edges for named imports/re-exports from local/aliased files. # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) @@ -3967,6 +4181,7 @@ def add_existing_edge(edge: dict) -> None: _DISPATCH: dict[str, Any] = { ".py": extract_python, + ".pyi": extract_python, ".js": extract_js, ".jsx": extract_js, ".mjs": extract_js, @@ -4869,6 +5084,19 @@ def _learn(e: dict) -> None: if dec is not None: e["target"] = f"{dec[0]}_{dec[1]}" + # A missing local JS/TS target still carries target_file so diagnostics can + # distinguish it from a package import. Canonicalize that path even though + # no target node exists; otherwise the dangling endpoint embeds the absolute + # checkout prefix and is misclassified as malformed/machine-specific. + for edge in all_edges: + if not edge.get("unresolved_internal") or not edge.get("target_file"): + continue + try: + target_rel = Path(edge["target_file"]).resolve().relative_to(root) + except (ValueError, OSError): + continue + edge["target"] = _make_id("ref_local", _file_stem(target_rel)) + # Repoint Python absolute imports onto the real file nodes under a nested # (src/) package root before the resolver/import-evidence passes run, so the # graph is identical regardless of scan root (#2072). @@ -4895,9 +5123,12 @@ def _learn(e: dict) -> None: _rewire_unique_stub_nodes(all_nodes, all_edges) # Add cross-file class-level edges (Python only - uses Python parser internally) - py_paths = [p for p in paths if p.suffix == ".py"] + py_paths = [p for p in paths if p.suffix.lower() in (".py", ".pyi")] if py_paths: - py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] + py_results = [ + r for r, p in zip(per_file, paths) + if p.suffix.lower() in (".py", ".pyi") + ] try: cross_file_edges = _resolve_cross_file_imports(py_results, py_paths) all_edges.extend(cross_file_edges) @@ -5310,6 +5541,9 @@ def _portable_out_of_root_sf(p: Path) -> str: # so it cannot be popped at that earlier point without breaking the fix. for e in all_edges: e.pop("local_alias", None) + e.pop("_import_module", None) + e.pop("_import_level", None) + e.pop("_imported_names", None) # Tag AST provenance so the incremental watch rebuild can distinguish # AST-extracted nodes from semantic/LLM nodes. On a full re-extraction diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index e88e36372..0ccd7865e 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -1617,14 +1617,16 @@ def _probe_python_module_candidate(candidate: Path) -> Path | None: """Resolve one module-path candidate to a .py file (dir+__init__, exact, or with a .py suffix), or None.""" if candidate.is_dir(): - init_path = candidate / "__init__.py" - if init_path.is_file(): - return init_path + for init_name in ("__init__.py", "__init__.pyi"): + init_path = candidate / init_name + if init_path.is_file(): + return init_path if candidate.is_file(): return candidate - py_candidate = candidate.with_suffix(".py") - if py_candidate.is_file(): - return py_candidate + for suffix in (".py", ".pyi"): + module_candidate = candidate.with_suffix(suffix) + if module_candidate.is_file(): + return module_candidate return None @@ -1691,7 +1693,9 @@ def _collect_python_symbol_resolution_facts( root: Path, facts: _SymbolResolutionFacts, ) -> None: - py_paths = [path for path in paths if path.suffix == ".py"] + py_paths = [ + path for path in paths if path.suffix.lower() in (".py", ".pyi") + ] if not py_paths: return diff --git a/graphify/manifest_ingest.py b/graphify/manifest_ingest.py index ae3aa61fc..6266d2a83 100644 --- a/graphify/manifest_ingest.py +++ b/graphify/manifest_ingest.py @@ -105,6 +105,10 @@ def extract_package_manifest(path: Path) -> dict[str, Any]: "source_file": str_path, "source_location": "L1", "weight": 1.0, + # Dependency package nodes are materialized only when their own + # manifest is part of the corpus. Otherwise this is an expected + # external endpoint, not lost internal structure. + "external": True, }) return {"nodes": nodes, "edges": edges} diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index afb4ecc12..870c68952 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index afb4ecc12..870c68952 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index d98865cc8..ff44d276c 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 0c821a278..8ed58de66 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index d98865cc8..ff44d276c 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index c3815d556..6f24bbf2b 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index dbb4658ca..22a4c6896 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index d98865cc8..ff44d276c 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index cf5dae440..a148428b1 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -456,13 +456,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index d98865cc8..ff44d276c 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index b0cbeb122..472336828 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -462,13 +462,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 3e6bc6b7b..5841beacd 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -460,13 +460,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 574384576..056bef11d 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -486,13 +486,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill.md b/graphify/skill.md index d98865cc8..ff44d276c 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tests/test_endpoint_classification.py b/tests/test_endpoint_classification.py new file mode 100644 index 000000000..45306420e --- /dev/null +++ b/tests/test_endpoint_classification.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from graphify.diagnostics import diagnose_extraction +from graphify.extract import extract + + +def test_typescript_external_and_missing_local_imports_are_separate(tmp_path: Path) -> None: + source = tmp_path / "src" / "app.ts" + source.parent.mkdir(parents=True) + source.write_text( + "import {Component} from '@angular/core';\n" + "import {Missing} from './missing';\n", + encoding="utf-8", + ) + + result = extract( + [source], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + imports = [ + edge for edge in result["edges"] + if edge.get("relation") == "imports_from" + ] + + assert len(imports) == 2 + assert sum(edge.get("external") is True for edge in imports) == 1 + assert sum(edge.get("unresolved_internal") is True for edge in imports) == 1 + + summary = diagnose_extraction(result, root=tmp_path) + assert summary["external_endpoint_edges"] == 1 + assert summary["unresolved_internal_endpoint_edges"] == 1 + assert summary["unclassified_endpoint_edges"] == 0 + + +def test_unresolved_typescript_alias_is_internal(tmp_path: Path) -> None: + (tmp_path / "tsconfig.json").write_text( + '{"compilerOptions":{"baseUrl":".","paths":{"@app/*":["src/*"]}}}', + encoding="utf-8", + ) + source = tmp_path / "src" / "app.ts" + source.parent.mkdir() + source.write_text( + "import {Missing} from '@app/missing';\n", + encoding="utf-8", + ) + + result = extract( + [source], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + edge = next( + edge for edge in result["edges"] + if edge.get("relation") == "imports_from" + ) + + assert edge.get("unresolved_internal") is True + assert edge.get("external") is not True diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 8c39b8e23..7ad4517b7 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -14,6 +14,7 @@ format_diagnostic_report, scan_producer_suppression_sites, ) +from graphify.ids import normalize_id def _diagnostic_fixture() -> dict: @@ -92,6 +93,10 @@ def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: assert summary["valid_candidate_edges"] == 5 assert summary["missing_endpoint_edges"] == 1 assert summary["dangling_endpoint_edges"] == 1 + assert summary["unclassified_endpoint_edges"] == 1 + assert summary["external_endpoint_edges"] == 0 + assert summary["unresolved_internal_endpoint_edges"] == 0 + assert summary["malformed_endpoint_edges"] == 0 assert summary["self_loop_edges"] == 1 assert summary["exact_duplicate_edges"] == 1 assert summary["directed_unique_endpoint_pairs"] == 2 @@ -146,10 +151,50 @@ def test_diagnose_extraction_handles_malformed_shapes_without_crashing() -> None assert summary["non_object_edges"] == 2 assert summary["missing_endpoint_edges"] == 1 assert summary["dangling_endpoint_edges"] == 2 + assert summary["unclassified_endpoint_edges"] == 2 assert summary["valid_candidate_edges"] == 1 assert summary["post_build_error"].startswith("TypeError:") +def test_diagnose_extraction_classifies_endpoint_loss_categories(tmp_path: Path) -> None: + root_prefix = normalize_id(str(tmp_path)) + extraction = { + "nodes": [{"id": "app", "label": "app.py", "file_type": "code"}], + "edges": [ + { + "source": "app", + "target": "pathlib", + "relation": "imports", + "external": True, + }, + { + "source": "app", + "target": "models_base", + "relation": "imports_from", + "unresolved_internal": True, + }, + { + "source": f"{root_prefix}_pkg_app", + "target": "app", + "relation": "indirect_call", + }, + { + "source": "app", + "target": "mystery", + "relation": "references", + }, + ], + } + + summary = diagnose_extraction(extraction, root=tmp_path) + + assert summary["dangling_endpoint_edges"] == 4 + assert summary["external_endpoint_edges"] == 1 + assert summary["unresolved_internal_endpoint_edges"] == 1 + assert summary["malformed_endpoint_edges"] == 1 + assert summary["unclassified_endpoint_edges"] == 1 + + def test_diagnose_extraction_handles_non_list_nodes_and_edges() -> None: summary = diagnose_extraction( {"nodes": {"id": "a"}, "edges": {"source": "a", "target": "b"}}, @@ -239,7 +284,7 @@ def test_diagnostic_json_report_is_serializable(tmp_path: Path) -> None: summary = diagnose_file(graph_path, directed=True) payload = format_diagnostic_json(summary) - assert payload["schema_version"] == 1 + assert payload["schema_version"] == 2 assert payload["summary"]["raw_edge_count"] == 7 assert "producer_suppression" in payload json.dumps(payload) @@ -399,7 +444,7 @@ def test_diagnose_multigraph_cli_json_output(monkeypatch, tmp_path: Path, capsys mainmod.main() payload = json.loads(capsys.readouterr().out) - assert payload["schema_version"] == 1 + assert payload["schema_version"] == 2 assert payload["summary"]["directed_same_endpoint_collapsed_edges"] == 3 diff --git a/tests/test_src_layout_import_resolution.py b/tests/test_src_layout_import_resolution.py index beeb6b95b..fec414db3 100644 --- a/tests/test_src_layout_import_resolution.py +++ b/tests/test_src_layout_import_resolution.py @@ -13,6 +13,7 @@ from graphify.extract import extract from graphify.extractors.resolution import _resolve_python_module_path from graphify.build import build_from_json +from graphify.diagnostics import diagnose_extraction _FILES = { @@ -37,6 +38,31 @@ def _write(base: Path, prefix: str = "") -> list[Path]: return written +def _write_file(path: Path, body: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + return path + + +def _node_id(result: dict, label: str, source_file: str) -> str: + matches = [ + node["id"] + for node in result["nodes"] + if node.get("label") == label and node.get("source_file") == source_file + ] + assert len(matches) == 1 + return matches[0] + + +def _has_edge(result: dict, source: str, target: str, relation: str) -> bool: + return any( + edge.get("source") == source + and edge.get("target") == target + and edge.get("relation") == relation + for edge in result["edges"] + ) + + def _import_edges(G): """(relation, source, target) for import edges, present-endpoints only.""" return { @@ -102,6 +128,9 @@ def test_import_edges_identical_from_root_or_src(tmp_path): def test_ambiguous_package_alias_is_not_repointed(tmp_path): """A dotted-module id claimed by two different files (two src roots with the same package) must stay dangling rather than pick an arbitrary file.""" + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='ambiguous'\nversion='1'\n" + ) for sub in ("a", "b"): d = tmp_path / sub / "src" / "pkg" d.mkdir(parents=True) @@ -151,3 +180,210 @@ def test_non_python_import_edge_is_not_repointed(tmp_path): assert not any(v == "src_pkg_mod" and u == "app_cs" for _, u, v in _import_edges(G)), ( "non-Python import edge was repointed onto a Python file (#2072 review)" ) + + +def test_namespace_package_import_resolves_without_init_file(tmp_path): + project = tmp_path / "api" + project.mkdir() + (project / "pyproject.toml").write_text("[project]\nname='api'\nversion='1'\n") + model = _write_file(project / "src/models/base.py", "class Base:\n pass\n") + service = _write_file( + project / "src/services/use.py", "from models.base import Base\n" + ) + + result = extract( + [model, service], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + model_id = _node_id(result, "base.py", "api/src/models/base.py") + service_id = _node_id(result, "use.py", "api/src/services/use.py") + + assert _has_edge(result, service_id, model_id, "imports_from") + summary = diagnose_extraction(result, root=tmp_path) + assert summary["unresolved_internal_endpoint_edges"] == 0 + + +def test_same_module_name_resolves_inside_each_monorepo_workspace(tmp_path): + paths = [] + expected = [] + for project_name in ("alpha", "beta"): + project = tmp_path / project_name + (project / "pyproject.toml").parent.mkdir(parents=True, exist_ok=True) + (project / "pyproject.toml").write_text( + f"[project]\nname='{project_name}'\nversion='1'\n" + ) + model = _write_file( + project / "src/models/base.py", + f"class {project_name.title()}Base:\n pass\n", + ) + service = _write_file( + project / "src/services/use.py", "import models.base\n" + ) + paths.extend((model, service)) + expected.append((project_name, model, service)) + + result = extract( + paths, + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + for project_name, _, _ in expected: + model_id = _node_id( + result, "base.py", f"{project_name}/src/models/base.py" + ) + service_id = _node_id( + result, "use.py", f"{project_name}/src/services/use.py" + ) + assert _has_edge(result, service_id, model_id, "imports") + + +def test_project_without_manifest_uses_first_scan_root_directory(tmp_path): + model = _write_file( + tmp_path / "legacy/src/models/base.py", "class Base:\n pass\n" + ) + service = _write_file( + tmp_path / "legacy/src/app.py", "from models.base import Base\n" + ) + + result = extract( + [model, service], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + + assert _has_edge( + result, + _node_id(result, "app.py", "legacy/src/app.py"), + _node_id(result, "base.py", "legacy/src/models/base.py"), + "imports_from", + ) + + +def test_python_stub_module_and_external_import_are_classified(tmp_path): + project = tmp_path / "typed" + (project / "pyproject.toml").parent.mkdir(parents=True, exist_ok=True) + (project / "pyproject.toml").write_text("[project]\nname='typed'\nversion='1'\n") + stub = _write_file( + project / "src/contracts/types.pyi", "class Payload: ...\n" + ) + consumer = _write_file( + project / "src/app.py", + "from contracts.types import Payload\n" + "import pathlib\n" + "import third_party_sdk\n", + ) + + result = extract( + [stub, consumer], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + assert _has_edge( + result, + _node_id(result, "app.py", "typed/src/app.py"), + _node_id(result, "types.pyi", "typed/src/contracts/types.pyi"), + "imports_from", + ) + external = [ + edge for edge in result["edges"] + if edge.get("target") in {"pathlib", "third_party_sdk"} + ] + assert len(external) == 2 + assert all(edge.get("external") is True for edge in external) + summary = diagnose_extraction(result, root=tmp_path) + assert summary["external_endpoint_edges"] == 2 + assert summary["unresolved_internal_endpoint_edges"] == 0 + assert summary["unclassified_endpoint_edges"] == 0 + + +def test_unresolved_relative_import_is_internal_not_external(tmp_path): + source = _write_file( + tmp_path / "pkg/__init__.py", "from .missing import value\n" + ) + + result = extract( + [source], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + edge = next(e for e in result["edges"] if e["relation"] == "imports_from") + assert edge.get("unresolved_internal") is True + assert edge.get("external") is not True + + +def test_imported_symbol_disambiguates_stale_relative_module_path(tmp_path): + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='moved-models'\nversion='1'\n" + ) + wrong = _write_file( + tmp_path / "models/legacy/scenario.py", + "class OtherScenario:\n pass\n", + ) + intended = _write_file( + tmp_path / "models/scenario/scenario.py", + "class Scenario:\n pass\n", + ) + consumer = _write_file( + tmp_path / "models/node/node.py", + "from .scenario import Scenario\n", + ) + + result = extract( + [wrong, intended, consumer], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + + assert _has_edge( + result, + _node_id(result, "node.py", "models/node/node.py"), + _node_id(result, "scenario.py", "models/scenario/scenario.py"), + "imports_from", + ) + + +def test_imported_symbol_disambiguates_absolute_package_facade(tmp_path): + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='facades'\nversion='1'\n" + ) + alpha = _write_file( + tmp_path / "alpha/models/__init__.py", + "from .entity import Alpha\n", + ) + alpha_entity = _write_file( + tmp_path / "alpha/models/entity.py", + "class Alpha:\n pass\n", + ) + beta = _write_file( + tmp_path / "beta/models/__init__.py", + "from .entity import Beta\n", + ) + beta_entity = _write_file( + tmp_path / "beta/models/entity.py", + "class Beta:\n pass\n", + ) + consumer = _write_file( + tmp_path / "consumer.py", + "from models import Beta\n", + ) + + result = extract( + [alpha, alpha_entity, beta, beta_entity, consumer], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + + assert _has_edge( + result, + _node_id(result, "consumer.py", "consumer.py"), + "beta_models_init", + "imports_from", + ) diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index afb4ecc12..870c68952 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index afb4ecc12..870c68952 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index d98865cc8..ff44d276c 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 0c821a278..8ed58de66 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index d98865cc8..ff44d276c 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index c3815d556..6f24bbf2b 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -461,13 +461,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index dbb4658ca..22a4c6896 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index d98865cc8..ff44d276c 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index cf5dae440..a148428b1 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -456,13 +456,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index d98865cc8..ff44d276c 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index b0cbeb122..472336828 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -462,13 +462,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 3e6bc6b7b..5841beacd 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -460,13 +460,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 574384576..056bef11d 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -486,13 +486,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index d98865cc8..ff44d276c 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -464,13 +464,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index e28910728..e2d90b867 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -399,13 +399,15 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(en summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), + ('malformed_endpoint_edges', 'malformed endpoint edges'), + ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` From d48c20d2c1e498d6ddb937fc35a213447738686e Mon Sep 17 00:00:00 2001 From: Henrique Silveira Neves Date: Fri, 24 Jul 2026 22:15:51 -0300 Subject: [PATCH 2/5] feat: preserve parallel relations with multidigraph --- CHANGELOG.md | 12 ++ README.md | 1 + graphify/analyze.py | 10 +- graphify/build.py | 189 ++++++++++++++++-- graphify/cli.py | 171 ++++++++++++++-- graphify/cluster.py | 13 +- graphify/diagnostics.py | 100 ++++++++- graphify/export.py | 42 +++- graphify/exporters/html.py | 35 +++- graphify/global_graph.py | 49 ++++- graphify/serve.py | 105 ++++++---- graphify/skill-agents.md | 19 +- graphify/skill-aider.md | 22 +- graphify/skill-amp.md | 19 +- graphify/skill-claw.md | 19 +- graphify/skill-codex.md | 19 +- graphify/skill-copilot.md | 19 +- graphify/skill-devin.md | 24 +-- graphify/skill-droid.md | 19 +- graphify/skill-kilo.md | 19 +- graphify/skill-kiro.md | 19 +- graphify/skill-opencode.md | 19 +- graphify/skill-pi.md | 19 +- graphify/skill-trae.md | 19 +- graphify/skill-vscode.md | 19 +- graphify/skill-windows.md | 19 +- graphify/skill.md | 19 +- graphify/skills/agents/references/update.md | 13 +- graphify/skills/amp/references/update.md | 13 +- graphify/skills/claude/references/update.md | 13 +- graphify/skills/claw/references/update.md | 13 +- graphify/skills/codex/references/update.md | 13 +- graphify/skills/copilot/references/update.md | 13 +- graphify/skills/droid/references/update.md | 13 +- graphify/skills/kilo/references/update.md | 13 +- graphify/skills/kiro/references/update.md | 13 +- graphify/skills/opencode/references/update.md | 13 +- graphify/skills/pi/references/update.md | 13 +- graphify/skills/trae/references/update.md | 13 +- graphify/skills/vscode/references/update.md | 13 +- graphify/skills/windows/references/update.md | 13 +- graphify/watch.py | 43 +++- graphify/wiki.py | 28 ++- tests/test_build.py | 120 +++++++++++ .../test_build_merge_hyperedges_and_prune.py | 34 ++++ tests/test_explain_cli.py | 35 ++++ tests/test_extract_code_only_cli.py | 19 ++ tests/test_merge_graphs_cli.py | 20 +- tests/test_multigraph_diagnostics.py | 18 ++ tests/test_path_cli.py | 33 +++ tests/test_query_cli.py | 33 +++ tests/test_skillgen.py | 8 +- .../expected/graphify__skill-agents.md | 19 +- .../expected/graphify__skill-aider.md | 22 +- .../skillgen/expected/graphify__skill-amp.md | 19 +- .../skillgen/expected/graphify__skill-claw.md | 19 +- .../expected/graphify__skill-codex.md | 19 +- .../expected/graphify__skill-copilot.md | 19 +- .../expected/graphify__skill-devin.md | 24 +-- .../expected/graphify__skill-droid.md | 19 +- .../skillgen/expected/graphify__skill-kilo.md | 19 +- .../skillgen/expected/graphify__skill-kiro.md | 19 +- .../expected/graphify__skill-opencode.md | 19 +- tools/skillgen/expected/graphify__skill-pi.md | 19 +- .../skillgen/expected/graphify__skill-trae.md | 19 +- .../expected/graphify__skill-vscode.md | 19 +- .../expected/graphify__skill-windows.md | 19 +- tools/skillgen/expected/graphify__skill.md | 19 +- ...ify__skills__agents__references__update.md | 13 +- ...aphify__skills__amp__references__update.md | 13 +- ...ify__skills__claude__references__update.md | 13 +- ...phify__skills__claw__references__update.md | 13 +- ...hify__skills__codex__references__update.md | 13 +- ...fy__skills__copilot__references__update.md | 13 +- ...hify__skills__droid__references__update.md | 13 +- ...phify__skills__kilo__references__update.md | 13 +- ...phify__skills__kiro__references__update.md | 13 +- ...y__skills__opencode__references__update.md | 13 +- ...raphify__skills__pi__references__update.md | 13 +- ...phify__skills__trae__references__update.md | 13 +- ...ify__skills__vscode__references__update.md | 13 +- ...fy__skills__windows__references__update.md | 13 +- tools/skillgen/fragments/core/aider.md | 22 +- tools/skillgen/fragments/core/core.md | 19 +- tools/skillgen/fragments/core/devin.md | 24 +-- .../fragments/references/shared/update.md | 13 +- tools/skillgen/gen.py | 5 +- 87 files changed, 1701 insertions(+), 488 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 531071fda..1b0512d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## Unreleased + +- Feat: add opt-in `--multigraph` builds backed by a persistent directed + `MultiDiGraph`. Canonically distinct relations between the same endpoints + retain stable keys; exact duplicate occurrences aggregate through + `occurrence_count`. Incremental updates, watch/hook rebuilds, cluster-only, + no-cluster, graph merges, query/path/explain, HTML, wiki, Obsidian, GraphML, + graph databases, MCP, reports, and diagnostics preserve or explicitly reject + unsupported simplification. Community detection, cohesion, god nodes, and + centrality use an unweighted simple undirected topology projection so + multiplicity does not distort analysis. The default remains `Graph`. + ## 0.9.26 (2026-07-25) - Fix: `graphify query`/`explain` no longer fabricate `indirect_call` edges to class definitions (#2137, thanks @Rishet11). The callable guard admitted classes, so passing a class as a value (`select(Model)`, `db.get(Model, id)`, `except (ErrorA, ErrorB)`, `getattr(obj, "Name", 0)`) produced a false inferred call edge in both the intra-file and cross-file paths. Classes are now tracked separately and excluded from `indirect_call`; direct instantiation still emits its `calls` edge. The suppression is context-blind, so a genuine higher-order class callback that is actually invoked (e.g. `map(Point, coords)`) also loses its edge, which is the intended tradeoff. diff --git a/README.md b/README.md index 0459f57cf..8b97420ca 100644 --- a/README.md +++ b/README.md @@ -734,6 +734,7 @@ graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow lo graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt +graphify extract ./src --multigraph # preserve directed parallel relations with stable edge keys graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) diff --git a/graphify/analyze.py b/graphify/analyze.py index ec1e61a99..04290c926 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -3,7 +3,7 @@ from pathlib import Path import networkx as nx -from graphify.build import edge_data +from graphify.build import analysis_projection, edge_data # Builtin/mock names that can appear as annotation-derived nodes in pre-existing # graphs. Excluded from god-node ranking so they don't displace real abstractions @@ -104,7 +104,8 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) + topology = analysis_projection(G) + degree = {node: topology.degree(node) for node in G.nodes} sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) result = [] for node_id, deg in sorted_nodes: @@ -281,7 +282,7 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: Each result includes a 'why' field explaining what makes it non-obvious. """ node_community = _node_community_map(communities) - degrees = dict(G.degree()) + degrees = dict(analysis_projection(G).degree()) candidates = [] for u, v, data in G.edges(data=True): @@ -347,7 +348,8 @@ def _cross_community_surprises( return [] if G.number_of_nodes() > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) + topology = analysis_projection(G) + betweenness = nx.edge_betweenness_centrality(topology) top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] result = [] for (u, v), score in top_edges: diff --git a/graphify/build.py b/graphify/build.py index 44e8c441c..5be6042aa 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -21,6 +21,7 @@ # before any graph construction happens. # from __future__ import annotations +import hashlib import json import math import os @@ -292,6 +293,76 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: return [raw] +def _canonical_json_value(value: object) -> object: + """Return a JSON-stable value for persistent edge identity.""" + if isinstance(value, dict): + return { + str(key): _canonical_json_value(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, (list, tuple)): + return [_canonical_json_value(item) for item in value] + if isinstance(value, set): + return sorted( + (_canonical_json_value(item) for item in value), + key=lambda item: json.dumps(item, sort_keys=True, default=str), + ) + if value is None or isinstance(value, (str, int, float, bool)): + return value + return str(value) + + +def canonical_edge_key(source: str, target: str, attrs: dict) -> str: + """Stable key for one canonical directed relationship. + + The key includes every persisted semantic attribute, while excluding + transport metadata and the duplicate counter. It is therefore stable across + extraction order and checkout paths once ``source_file`` has been + relativized, but changes when relation, context, location, confidence, or + another semantic attribute changes. + """ + semantic = { + key: value + for key, value in attrs.items() + if key not in {"key", "occurrence_count", "_src", "_tgt"} + } + payload = { + "source": source, + "target": target, + "attributes": _canonical_json_value(semantic), + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + return "edge_" + hashlib.sha256(encoded).hexdigest() + + +def analysis_projection(G: nx.Graph) -> nx.Graph: + """Return the simple undirected topology used for graph analysis. + + Parallel relations and opposite directions intentionally contribute one + unweighted endpoint pair. The canonical graph remains untouched. + """ + projected = nx.Graph() + projected.add_nodes_from( + (node, dict(attrs)) + for node, attrs in sorted(G.nodes(data=True), key=lambda row: str(row[0])) + ) + pairs = { + (source, target) if str(source) <= str(target) else (target, source) + for source, target in G.edges() + } + projected.add_edges_from( + (source, target, {"weight": 1.0}) + for source, target in sorted(pairs, key=lambda pair: (str(pair[0]), str(pair[1]))) + ) + return projected + + def dedupe_nodes(nodes: list[dict]) -> list[dict]: """Collapse nodes sharing an ``id``, last-writer-wins on attributes. @@ -487,14 +558,26 @@ def _doc_twin_remap(nodes: list) -> dict[str, str]: return remap -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: +def build_from_json( + extraction: dict, + *, + directed: bool = False, + multigraph: bool = False, + root: str | Path | None = None, +) -> nx.Graph: """Build a NetworkX graph from an extraction dict. + multigraph=True produces a MultiDiGraph and implies directed=True. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). """ + if multigraph: + from graphify.multigraph_compat import require_multigraph_capabilities + + require_multigraph_capabilities() + directed = True _root = str(Path(root).resolve()) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: @@ -594,7 +677,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + G: nx.Graph + if multigraph: + G = nx.MultiDiGraph() + elif directed: + G = nx.DiGraph() + else: + G = nx.Graph() for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted # by a buggy LLM extraction) so NetworkX add_node never raises @@ -761,14 +850,19 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run # and makes the serialized graph churn. Sorting fixes the last-write outcome. - for edge in sorted( - extraction.get("edges", []), - key=lambda e: ( - str(e.get("source", e.get("from", ""))), - str(e.get("target", e.get("to", ""))), - str(e.get("relation", "")), - ), - ): + def _edge_sort_key(edge: dict) -> tuple: + base = ( + str(edge.get("source", edge.get("from", ""))), + str(edge.get("target", edge.get("to", ""))), + str(edge.get("relation", "")), + ) + if not multigraph: + return base + return base + ( + json.dumps(edge, sort_keys=True, ensure_ascii=False, default=str), + ) + + for edge in sorted(extraction.get("edges", []), key=_edge_sort_key): if "source" not in edge and "from" in edge: edge["source"] = edge["from"] if "target" not in edge and "to" in edge: @@ -816,7 +910,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # strings, NaN/inf, negatives — while numeric strings coerce cleanly. # Repair (not drop) the key so graph.json round-trips a clean value and a # cluster-only/--update reload never re-ingests the null. - attrs = {k: v for k, v in edge.items() if k not in ("source", "target", "target_file", "local_alias")} + attrs = { + k: v + for k, v in edge.items() + if k not in ("source", "target", "target_file", "local_alias", "key") + } for _num_key in ("weight", "confidence_score"): if _num_key in attrs: try: @@ -889,7 +987,23 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat existing.get("_src") == tgt and existing.get("_tgt") == src ): continue - G.add_edge(src, tgt, **attrs) + if multigraph: + edge_key = canonical_edge_key(str(src), str(tgt), attrs) + incoming_count = attrs.pop("occurrence_count", 1) + try: + incoming_count = max(1, int(incoming_count)) + except (TypeError, ValueError): + incoming_count = 1 + if G.has_edge(src, tgt, edge_key): + existing = G[src][tgt][edge_key] + existing["occurrence_count"] = ( + int(existing.get("occurrence_count", 1)) + incoming_count + ) + else: + attrs["occurrence_count"] = incoming_count + G.add_edge(src, tgt, key=edge_key, **attrs) + else: + G.add_edge(src, tgt, **attrs) hyperedges = extraction.get("hyperedges", []) if hyperedges: # Relativize hyperedge source_file the same way nodes and edges are @@ -944,12 +1058,14 @@ def build( extractions: list[dict], *, directed: bool = False, + multigraph: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, ) -> nx.Graph: """Merge multiple extraction results into one graph. + multigraph=True produces a MultiDiGraph and implies directed=True. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. dedup=True (default) runs entity deduplication before building the graph. @@ -976,7 +1092,12 @@ def build( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed, root=root) + return build_from_json( + combined, + directed=directed, + multigraph=multigraph, + root=root, + ) def _norm_label(label: str | None) -> str: @@ -1047,6 +1168,7 @@ def build_merge( prune_sources: list[str] | None = None, *, directed: bool = False, + multigraph: bool | None = None, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, @@ -1083,11 +1205,28 @@ def build_merge( existing_edges = list(data.get(links_key, [])) existing_hyperedges = list(data.get("hyperedges", [])) had_graph = True + existing_multigraph = data.get("multigraph") is True + existing_directed = data.get("directed") is True else: existing_nodes = [] existing_edges = [] existing_hyperedges = [] had_graph = False + existing_multigraph = False + existing_directed = False + + if existing_multigraph: + if multigraph is False: + raise ValueError( + "graphify: refusing to simplify an existing MultiDiGraph; " + "re-run with --multigraph or omit the explicit downgrade." + ) + multigraph = True + directed = True + elif multigraph is None: + multigraph = False + if existing_directed: + directed = True # Effective root for relativizing absolute source_file / prune paths back to the # stored relative source_file keys. When the caller passes root we use it; @@ -1131,7 +1270,14 @@ def _kept(item: dict) -> bool: base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, + directed=directed, + multigraph=multigraph, + dedup=dedup, + dedup_llm_backend=dedup_llm_backend, + root=root, + ) # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes @@ -1216,10 +1362,17 @@ def _prune_match(sf: "str | None") -> bool: file=sys.stderr, ) - edges_to_remove = [ - (u, v) for u, v, d in G.edges(data=True) - if _prune_match(d.get("source_file")) - ] + if G.is_multigraph(): + edges_to_remove = [ + (u, v, key) + for u, v, key, d in G.edges(keys=True, data=True) + if _prune_match(d.get("source_file")) + ] + else: + edges_to_remove = [ + (u, v) for u, v, d in G.edges(data=True) + if _prune_match(d.get("source_file")) + ] if edges_to_remove: G.remove_edges_from(edges_to_remove) print( diff --git a/graphify/cli.py b/graphify/cli.py index 91df09672..712100977 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1177,8 +1177,30 @@ def dispatch_command(cmd: str) -> None: else: datas = edge_datas(G, v, u) forward = False - rels = sorted({d.get("relation") for d in datas if d.get("relation")}) - rel = "/".join(rels) if rels else "related" + relation_details = [] + for data in sorted( + datas, + key=lambda item: ( + str(item.get("relation", "")), + str(item.get("source_file", "")), + str(item.get("source_location", "")), + str(item.get("context", "")), + ), + ): + relation = data.get("relation") or "related" + context = data.get("context") + location = data.get("source_location") + source_file = data.get("source_file") + occurrences = data.get("occurrence_count", 1) + detail = relation + if context: + detail += f":{context}" + if source_file or location: + detail += f"@{source_file or ''}{':' + str(location) if location else ''}" + if occurrences != 1: + detail += f"x{occurrences}" + relation_details.append(detail) + rel = " | ".join(relation_details) if relation_details else "related" confs = sorted({d.get("confidence") for d in datas if d.get("confidence")}) conf_str = f" [{'/'.join(confs)}]" if confs else "" if i == 0: @@ -1218,8 +1240,9 @@ def dispatch_command(cmd: str) -> None: _raw = json.loads(gp.read_text(encoding="utf-8")) if "links" not in _raw and "edges" in _raw: _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} + # Force a directed multigraph view so every stored parallel relation is + # available even for legacy simple-graph files. + _raw = {**_raw, "directed": True, "multigraph": True} try: G = json_graph.node_link_graph(_raw, edges="links") except TypeError: @@ -1262,12 +1285,16 @@ def dispatch_command(cmd: str) -> None: except Exception: pass print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data + from graphify.build import edge_datas connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) + connections.extend( + ("out", nb, data) for data in edge_datas(G, nid, nb) + ) for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) + connections.extend( + ("in", nb, data) for data in edge_datas(G, nb, nid) + ) if connections: print(f"\nConnections ({len(connections)}):") connections.sort(key=lambda c: G.degree(c[1]), reverse=True) @@ -1281,7 +1308,14 @@ def dispatch_command(cmd: str) -> None: loc = edata.get("source_location") or "" sfile = edata.get("source_file") or "" at = f" {sfile}:{loc}" if loc else "" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]{at}") + context = edata.get("context") or "" + context_text = f" context={context}" if context else "" + occurrences = edata.get("occurrence_count", 1) + occurrence_text = f" x{occurrences}" if occurrences != 1 else "" + print( + f" {arrow} {G.nodes[nb].get('label', nb)} " + f"[{rel}] [{conf}]{context_text}{occurrence_text}{at}" + ) if len(connections) > 20: remainder = connections[20:] print(f" ... and {len(remainder)} more") @@ -1319,7 +1353,7 @@ def dispatch_command(cmd: str) -> None: print( "Usage: graphify diagnose multigraph " "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", + "[--directed] [--undirected] [--multigraph] [--extract-path path]", file=sys.stderr, ) sys.exit(1) @@ -1327,6 +1361,7 @@ def dispatch_command(cmd: str) -> None: graph_path = Path(_default_graph_path()) max_examples = 5 directed: bool | None = None + multigraph: bool | None = None direction_flag: str | None = None json_output = False extract_path: Path | None = None @@ -1373,6 +1408,9 @@ def dispatch_command(cmd: str) -> None: sys.exit(1) direction_flag = "undirected" directed = False + elif arg == "--multigraph": + multigraph = True + directed = True elif arg == "--extract-path": i += 1 if i >= len(sys.argv): @@ -1394,6 +1432,7 @@ def dispatch_command(cmd: str) -> None: summary = diagnose_file( graph_path, directed=directed, + multigraph=multigraph, root=Path(".").resolve(), max_examples=max_examples, extract_path=extract_path, @@ -1558,7 +1597,12 @@ def dispatch_command(cmd: str) -> None: ) _raw = json.loads(graph_json.read_text(encoding="utf-8")) _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) + _multigraph = bool(_raw.get("multigraph", False)) + G = build_from_json( + _raw, + directed=_directed, + multigraph=_multigraph, + ) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") stages.mark("load") print("Re-clustering...") @@ -1782,6 +1826,7 @@ def dispatch_command(cmd: str) -> None: elif cmd == "update": force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") no_cluster = False + multigraph: bool | None = None args = sys.argv[2:] watch_arg: str | None = None for a in args: @@ -1791,6 +1836,9 @@ def dispatch_command(cmd: str) -> None: if a == "--no-cluster": no_cluster = True continue + if a == "--multigraph": + multigraph = True + continue if a.startswith("-"): print(f"error: unknown update option: {a}", file=sys.stderr) sys.exit(2) @@ -1817,7 +1865,13 @@ def dispatch_command(cmd: str) -> None: # Interactive CLI: block on the per-repo lock rather than skip, so the # user sees their explicit `graphify update` complete instead of # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + ok = _rebuild_code( + watch_path, + force=force, + no_cluster=no_cluster, + multigraph=multigraph, + block_on_lock=True, + ) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") if not ( @@ -1977,17 +2031,22 @@ def _load_graph(p: str): args = sys.argv[2:] graph_paths: list[Path] = [] out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" + merge_multigraph = False i = 0 while i < len(args): if args[i] == "--out" and i + 1 < len(args): out_path = Path(args[i + 1]) i += 2 + elif args[i] == "--multigraph": + merge_multigraph = True + i += 1 else: graph_paths.append(Path(args[i])) i += 1 if len(graph_paths) < 2: print( - "Usage: graphify merge-graphs [...] [--out merged.json]", + "Usage: graphify merge-graphs [...] " + "[--out merged.json] [--multigraph]", file=sys.stderr, ) sys.exit(1) @@ -2010,6 +2069,13 @@ def _load_graph(p: str): except TypeError: G = _jg.node_link_graph(data) graphs.append(G) + if any(graph.is_multigraph() for graph in graphs) and not merge_multigraph: + print( + "error: one or more inputs are multigraphs; pass --multigraph " + "to preserve parallel relations.", + file=sys.stderr, + ) + sys.exit(2) # nx.compose requires all graphs to be the same type. When input graphs # come from different sources (e.g. an AST-only run vs a full LLM run) one # may be a MultiGraph and another a Graph. Normalise everything to Graph @@ -2033,9 +2099,40 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": naive_tags = [gp.parent.parent.name for gp in graph_paths] if len(set(naive_tags)) != len(naive_tags): print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = _nx.Graph() + from graphify.build import canonical_edge_key as _edge_key + + def _to_multidigraph(g: "_nx.Graph") -> "_nx.MultiDiGraph": + if isinstance(g, _nx.MultiDiGraph): + return g + result = _nx.MultiDiGraph() + result.add_nodes_from(g.nodes(data=True)) + if g.is_multigraph(): + rows = g.edges(keys=True, data=True) + for u, v, key, attrs in rows: + src = attrs.get("_src", u) + tgt = attrs.get("_tgt", v) + result.add_edge(src, tgt, key=str(key), **dict(attrs)) + else: + for u, v, attrs in g.edges(data=True): + attrs = dict(attrs) + src = attrs.get("_src", u) + tgt = attrs.get("_tgt", v) + result.add_edge( + src, + tgt, + key=_edge_key(str(src), str(tgt), attrs), + **attrs, + ) + return result + + merged = _nx.MultiDiGraph() if merge_multigraph else _nx.Graph() for G, repo_tag in zip(graphs, repo_tags): - prefixed = _to_simple(_prefix(G, repo_tag)) + prefixed_raw = _prefix(G, repo_tag) + prefixed = ( + _to_multidigraph(prefixed_raw) + if merge_multigraph + else _to_simple(prefixed_raw) + ) merged = _nx.compose(merged, prefixed) try: out_data = _jg.node_link_data(merged, edges="links") @@ -2408,10 +2505,13 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": args = sys.argv[3:] source = None tag = None + use_multigraph = False i = 0 while i < len(args): if args[i] == "--as" and i + 1 < len(args): tag = args[i + 1]; i += 2 + elif args[i] == "--multigraph": + use_multigraph = True; i += 1 elif not source: source = Path(args[i]); i += 1 else: @@ -2421,7 +2521,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": sys.exit(1) tag = tag or source.parent.parent.name try: - result = _global_add(source, tag) + result = _global_add(source, tag, multigraph=use_multigraph) if result["skipped"]: print(f"'{tag}' unchanged since last add - global graph not modified.") else: @@ -2462,7 +2562,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print( "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] " - "[--no-gitignore] [--code-only] " + "[--multigraph] [--no-gitignore] [--code-only] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", file=sys.stderr, @@ -2487,6 +2587,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": cli_cargo: bool = False cli_allow_partial: bool = False no_cluster = False + cli_multigraph = False dedup_llm = False google_workspace = False global_merge = False @@ -2554,6 +2655,8 @@ def _parse_float(name: str, raw: str) -> float: out_dir = Path(a.split("=", 1)[1]); i += 1 elif a == "--no-cluster": no_cluster = True; i += 1 + elif a == "--multigraph": + cli_multigraph = True; i += 1 elif a == "--dedup-llm": dedup_llm = True; i += 1 elif a == "--code-only": @@ -2645,6 +2748,7 @@ def _parse_float(name: str, raw: str) -> float: _write_build_config as _write_build_cfg, _read_build_excludes as _read_build_ex, _read_build_gitignore as _read_build_gi, + _read_build_multigraph as _read_build_mg, ) # #1971 persistence: an explicit --no-gitignore persists False; a later # flag-less `graphify extract` must NOT clobber it back to True, which @@ -2656,10 +2760,12 @@ def _parse_float(name: str, raw: str) -> float: _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) # An explicit list replaces the persisted one; omission reuses it. _effective_excludes = cli_excludes or _read_build_ex(graphify_out) + _effective_multigraph = cli_multigraph or _read_build_mg(graphify_out) _write_build_cfg( graphify_out, excludes=cli_excludes or None, gitignore=False if no_gitignore else None, + multigraph=True if cli_multigraph else None, ) stages = _StageTimer(cli_timing) @@ -3217,7 +3323,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) sys.exit(1) - if no_cluster: + if no_cluster and not _effective_multigraph: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. # Dedupe nodes (by id) and parallel edges so the raw output matches the @@ -3344,7 +3450,11 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.global_graph import global_add as _global_add _tag = global_repo_tag or target.name try: - result = _global_add(graphify_out / "graph.json", _tag) + result = _global_add( + graphify_out / "graph.json", + _tag, + multigraph=_effective_multigraph, + ) if result["skipped"]: print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") else: @@ -3380,10 +3490,17 @@ def _invalidate_file_manifest_for_db_graph() -> None: prune_sources=_prune_sources or None, dedup=True, dedup_llm_backend=dedup_backend, + multigraph=True if _effective_multigraph else None, root=target, ) else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + G = _build( + [merged], + dedup=True, + dedup_llm_backend=dedup_backend, + multigraph=_effective_multigraph, + root=target, + ) stages.mark("build") if G.number_of_nodes() == 0: print( @@ -3394,7 +3511,15 @@ def _invalidate_file_manifest_for_db_graph() -> None: ) sys.exit(1) - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) + communities = ( + {} + if no_cluster + else _cluster( + G, + resolution=cli_resolution, + exclude_hubs_percentile=cli_exclude_hubs, + ) + ) stages.mark("cluster") cohesion = _score_all(G, communities) try: @@ -3462,7 +3587,11 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.global_graph import global_add as _global_add _tag = global_repo_tag or target.name try: - result = _global_add(graphify_out / "graph.json", _tag) + result = _global_add( + graphify_out / "graph.json", + _tag, + multigraph=_effective_multigraph, + ) if result["skipped"]: print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") else: diff --git a/graphify/cluster.py b/graphify/cluster.py index 682210700..37bc670b7 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -6,6 +6,7 @@ import json import sys import networkx as nx +from graphify.build import analysis_projection def _suppress_output(): @@ -95,6 +96,7 @@ def label_communities_by_hub( Used as the default (no-backend) labeler; an LLM naming pass, when configured, overrides these with richer names. """ + topology = analysis_projection(G) labels: dict[int, str] = {} for cid, members in communities.items(): present = [n for n in members if n in G] @@ -102,7 +104,7 @@ def label_communities_by_hub( labels[cid] = f"Community {cid}" continue # highest degree wins; ties broken by node id (ascending) for determinism - hub = min(present, key=lambda n: (-G.degree(n), str(n))) + hub = min(present, key=lambda n: (-topology.degree(n), str(n))) name = str(G.nodes[hub].get("label") or hub).strip() if name.endswith("()"): name = name[:-2] @@ -154,8 +156,7 @@ def cluster( """ if G.number_of_nodes() == 0: return {} - if G.is_directed(): - G = G.to_undirected() + G = analysis_projection(G) if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} @@ -259,14 +260,16 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: n = len(community_nodes) if n <= 1: return 1.0 - subgraph = G.subgraph(community_nodes) + topology = analysis_projection(G) if (G.is_directed() or G.is_multigraph()) else G + subgraph = topology.subgraph(community_nodes) actual = subgraph.number_of_edges() possible = n * (n - 1) / 2 return actual / possible if possible > 0 else 0.0 def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: - return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} + topology = analysis_projection(G) + return {cid: cohesion_score(topology, nodes) for cid, nodes in communities.items()} def remap_communities_to_previous( diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index d070a2cc1..92eb6a555 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -104,6 +104,10 @@ def _exact_signature(edge: Any) -> str: normalized["target"] = normalized["to"] normalized.pop("from", None) normalized.pop("to", None) + normalized.pop("key", None) + normalized.pop("occurrence_count", None) + normalized.pop("_src", None) + normalized.pop("_tgt", None) return json.dumps( normalized, sort_keys=True, @@ -183,6 +187,7 @@ def diagnose_extraction( extraction: dict[str, Any], *, directed: bool = True, + multigraph: bool = False, root: str | Path | None = None, max_examples: int = 5, extract_path: str | Path | None = None, @@ -202,7 +207,7 @@ def diagnose_extraction( if isinstance(n, dict) and n.get("verification") == "unverified" ) - exact_counts: Counter[str] = Counter(_exact_signature(edge) for edge in raw_edges) + exact_counts: Counter[str] = Counter() directed_pairs: Counter[tuple[str, str]] = Counter() undirected_pairs: Counter[tuple[str, str]] = Counter() grouped: dict[tuple[str, str], list[dict[str, str]]] = defaultdict(list) @@ -216,6 +221,8 @@ def diagnose_extraction( unclassified_endpoint_edges = 0 self_loop_edges = 0 valid_candidate_edges = 0 + encoded_duplicate_occurrences = 0 + valid_signatures_by_pair: dict[tuple[str, str], set[str]] = defaultdict(set) endpoint_examples: dict[str, list[dict[str, str]]] = { "external": [], "unresolved_internal": [], @@ -234,7 +241,7 @@ def _remember(category: str, edge: dict[str, Any]) -> None: "source_location": edge["source_location"], }) - for edge in canonical_edges: + for raw_edge, edge in zip(raw_edges, canonical_edges): if edge["_invalid"]: non_object_edges += 1 continue @@ -261,11 +268,37 @@ def _remember(category: str, edge: dict[str, Any]) -> None: if source == target: self_loop_edges += 1 valid_candidate_edges += 1 + if isinstance(raw_edge, dict): + try: + encoded_duplicate_occurrences += max( + 0, int(raw_edge.get("occurrence_count", 1)) - 1 + ) + except (TypeError, ValueError): + pass + signature = _exact_signature(raw_edge) + exact_counts[signature] += 1 directed_pair = (source, target) undirected_pair = (source, target) if source <= target else (target, source) directed_pairs[directed_pair] += 1 undirected_pairs[undirected_pair] += 1 grouped[directed_pair].append(edge) + valid_signatures_by_pair[directed_pair].add(signature) + + canonical_distinct_candidate_edges = len(exact_counts) + exact_duplicate_occurrences = ( + _count_extra(exact_counts) + encoded_duplicate_occurrences + ) + distinct_parallel_edge_instances = sum( + max(0, len(signatures) - 1) + for signatures in valid_signatures_by_pair.values() + ) + orientations_by_pair: dict[tuple[str, str], set[tuple[str, str]]] = defaultdict(set) + for source, target in valid_signatures_by_pair: + pair = (source, target) if source <= target else (target, source) + orientations_by_pair[pair].add((source, target)) + opposite_direction_endpoint_pairs = sum( + 1 for orientations in orientations_by_pair.values() if len(orientations) > 1 + ) examples: list[dict[str, Any]] = [] if max_examples > 0: @@ -293,7 +326,12 @@ def _remember(category: str, edge: dict[str, Any]) -> None: post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) + graph: nx.Graph = build_from_json( + graph_input, + directed=directed, + multigraph=multigraph, + root=root, + ) graph_type = type(graph).__name__ post_build_edge_count = graph.number_of_edges() post_build_node_count = graph.number_of_nodes() @@ -304,6 +342,23 @@ def _remember(category: str, edge: dict[str, Any]) -> None: Path(extract_path) if extract_path else Path(__file__).with_name("extract.py") ) + post_build_preserved_parallel_edges = None + post_build_lost_distinct_edges = None + if post_build_edge_count is not None: + if graph_type in {"MultiGraph", "MultiDiGraph"}: + built_pairs: Counter[tuple[str, str]] = Counter() + for source, target in graph.edges(): + built_pairs[(str(source), str(target))] += 1 + post_build_preserved_parallel_edges = min( + distinct_parallel_edge_instances, + _count_extra(built_pairs), + ) + else: + post_build_preserved_parallel_edges = 0 + post_build_lost_distinct_edges = max( + 0, canonical_distinct_candidate_edges - post_build_edge_count + ) + return { "node_count": len(node_ids), "unverified_node_count": unverified_node_count, @@ -317,7 +372,11 @@ def _remember(category: str, edge: dict[str, Any]) -> None: "unclassified_endpoint_edges": unclassified_endpoint_edges, "self_loop_edges": self_loop_edges, "valid_candidate_edges": valid_candidate_edges, - "exact_duplicate_edges": _count_extra(exact_counts), + "exact_duplicate_edges": exact_duplicate_occurrences, + "canonical_distinct_candidate_edges": canonical_distinct_candidate_edges, + "exact_duplicate_occurrences": exact_duplicate_occurrences, + "distinct_parallel_edge_instances": distinct_parallel_edge_instances, + "opposite_direction_endpoint_pairs": opposite_direction_endpoint_pairs, "directed_unique_endpoint_pairs": len(directed_pairs), "directed_same_endpoint_collapsed_edges": _count_extra(directed_pairs), "undirected_unique_endpoint_pairs": len(undirected_pairs), @@ -334,6 +393,8 @@ def _remember(category: str, edge: dict[str, Any]) -> None: "post_build_graph_type": graph_type, "post_build_node_count": post_build_node_count, "post_build_edge_count": post_build_edge_count, + "post_build_preserved_parallel_edges": post_build_preserved_parallel_edges, + "post_build_lost_distinct_edges": post_build_lost_distinct_edges, "post_build_error": build_error, "producer_suppression": scan_producer_suppression_sites(suppression_path), "examples": examples, @@ -363,6 +424,7 @@ def diagnose_file( path: str | Path, *, directed: bool | None = None, + multigraph: bool | None = None, root: str | Path | None = None, max_examples: int = 5, extract_path: str | Path | None = None, @@ -378,16 +440,24 @@ def diagnose_file( effective_directed = raw_directed if isinstance(raw_directed, bool) else True else: effective_directed = directed + if multigraph is None: + effective_multigraph = data.get("multigraph") is True + else: + effective_multigraph = multigraph + if effective_multigraph: + effective_directed = True summary = diagnose_extraction( data, directed=effective_directed, + multigraph=effective_multigraph, root=root, max_examples=max_examples, extract_path=extract_path, ) summary["input_path"] = str(path) summary["effective_directed"] = effective_directed + summary["effective_multigraph"] = effective_multigraph return summary @@ -417,6 +487,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"input: {summary.get('input_path', '')}", "input_stage: provided JSON (normal graph.json is post-build)", f"effective_directed: {summary.get('effective_directed', '')}", + f"effective_multigraph: {summary.get('effective_multigraph', '')}", f"nodes: {summary['node_count']}", f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", f"raw_edges: {summary['raw_edge_count']}", @@ -432,6 +503,19 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"unclassified_endpoint_edges: {summary.get('unclassified_endpoint_edges', 0)}", f"self_loop_edges: {summary['self_loop_edges']}", f"exact_duplicate_edges: {summary['exact_duplicate_edges']}", + ( + "canonical_distinct_candidate_edges: " + f"{summary.get('canonical_distinct_candidate_edges', 0)}" + ), + f"exact_duplicate_occurrences: {summary.get('exact_duplicate_occurrences', 0)}", + ( + "distinct_parallel_edge_instances: " + f"{summary.get('distinct_parallel_edge_instances', 0)}" + ), + ( + "opposite_direction_endpoint_pairs: " + f"{summary.get('opposite_direction_endpoint_pairs', 0)}" + ), f"directed_unique_endpoint_pairs: {summary['directed_unique_endpoint_pairs']}", ( "directed_same_endpoint_collapsed_edges: " @@ -449,6 +533,14 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"context_variant_groups: {summary['context_variant_groups']}", f"post_build_graph_type: {summary['post_build_graph_type']}", f"post_build_edges: {summary['post_build_edge_count']}", + ( + "post_build_preserved_parallel_edges: " + f"{summary.get('post_build_preserved_parallel_edges', 0)}" + ), + ( + "post_build_lost_distinct_edges: " + f"{summary.get('post_build_lost_distinct_edges', 0)}" + ), f"producer_suppression_sites: {suppression.get('total_sites', 0)}", ] if summary.get("post_build_error"): diff --git a/graphify/export.py b/graphify/export.py index e1f2caa99..e7ca53f97 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -15,7 +15,7 @@ from networkx.readwrite import json_graph from graphify.security import sanitize_label from graphify.analyze import _node_community_map -from graphify.build import edge_data +from graphify.build import edge_data, edge_datas from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 @@ -512,11 +512,25 @@ def safe_name(label: str) -> str: node_filename = _dedup_node_filenames(G, safe_name) + def _incident(node_id: str): + if G.is_directed(): + for neighbor in G.successors(node_id): + for attrs in edge_datas(G, node_id, neighbor): + yield "out", neighbor, attrs + for neighbor in G.predecessors(node_id): + for attrs in edge_datas(G, neighbor, node_id): + yield "in", neighbor, attrs + else: + for neighbor in G.neighbors(node_id): + for attrs in edge_datas(G, node_id, neighbor): + yield "undirected", neighbor, attrs + # Helper: compute dominant confidence for a node across all its edges def _dominant_confidence(node_id: str) -> str: - confs = [] - for u, v, edata in G.edges(node_id, data=True): - confs.append(edata.get("confidence", "EXTRACTED")) + confs = [ + edata.get("confidence", "EXTRACTED") + for _direction, _neighbor, edata in _incident(node_id) + ] if not confs: return "EXTRACTED" return Counter(confs).most_common(1)[0][0] @@ -567,16 +581,24 @@ def _dominant_confidence(node_id: str) -> str: lines.append(f" - {tag}") lines += ["---", "", f"# {label}", ""] - # Outgoing edges as wikilinks - neighbors = list(G.neighbors(node_id)) - if neighbors: + incident = sorted( + _incident(node_id), + key=lambda item: ( + G.nodes[item[1]].get("label", item[1]), + item[0], + item[2].get("relation", ""), + ), + ) + if incident: lines.append("## Connections") - for neighbor in sorted(neighbors, key=lambda n: G.nodes[n].get("label", n)): - edata = edge_data(G, node_id, neighbor) + for direction, neighbor, edata in incident: neighbor_label = node_filename[neighbor] relation = edata.get("relation", "") confidence = edata.get("confidence", "EXTRACTED") - lines.append(f"- [[{neighbor_label}]] - `{relation}` [{confidence}]") + arrow = "→" if direction == "out" else "←" if direction == "in" else "—" + lines.append( + f"- {arrow} [[{neighbor_label}]] - `{relation}` [{confidence}]" + ) lines.append("") # Inline tags at bottom of note body (for Obsidian tag panel) diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 59c0e52e3..f96f7e173 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -487,21 +487,48 @@ def to_html( # canonicalizes endpoint order, which would otherwise flip the arrow # for `calls` and `rationale_for` in the rendered graph (#563). vis_edges = [] - for u, v, data in G.edges(data=True): + if G.is_multigraph(): + edge_rows = list(G.edges(keys=True, data=True)) + else: + edge_rows = [(u, v, None, data) for u, v, data in G.edges(data=True)] + pair_positions: dict[tuple[str, str], int] = {} + for u, v, key, data in edge_rows: confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") true_src = data.get("_src", u) true_tgt = data.get("_tgt", v) - vis_edges.append({ + pair = tuple(sorted((str(true_src), str(true_tgt)))) + position = pair_positions.get(pair, 0) + pair_positions[pair] = position + 1 + context = data.get("context") or "" + source_file = data.get("source_file") or "" + source_location = data.get("source_location") or "" + occurrences = data.get("occurrence_count", 1) + details = [f"{relation} [{confidence}]"] + if context: + details.append(f"context: {context}") + if source_file or source_location: + details.append(f"source: {source_file}:{source_location}") + if occurrences != 1: + details.append(f"occurrences: {occurrences}") + edge_payload = { "from": true_src, "to": true_tgt, "label": relation, - "title": _html.escape(f"{relation} [{confidence}]"), + "title": _html.escape("\n".join(details)), "dashes": confidence != "EXTRACTED", "width": 2 if confidence == "EXTRACTED" else 1, "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, "confidence": confidence, - }) + } + if key is not None: + edge_payload["id"] = str(key) + edge_payload["smooth"] = { + "enabled": True, + "type": "curvedCW" if position % 2 == 0 else "curvedCCW", + "roundness": 0.12 + 0.08 * (position // 2), + } + vis_edges.append(edge_payload) # Build community legend data legend_data = [] diff --git a/graphify/global_graph.py b/graphify/global_graph.py index eddd0c92a..d8606d401 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -76,7 +76,7 @@ def _file_hash(path: Path) -> str: return h.hexdigest()[:16] -def global_add(source_path: Path, repo_tag: str) -> dict: +def global_add(source_path: Path, repo_tag: str, *, multigraph: bool = False) -> dict: """Add or update a project graph in the global graph. Returns a summary dict with keys: repo_tag, nodes_added, nodes_removed, skipped. @@ -112,12 +112,33 @@ def global_add(source_path: Path, repo_tag: str) -> dict: src_G = _jg.node_link_graph(data, edges="links") except TypeError: src_G = _jg.node_link_graph(data) + if src_G.is_multigraph() and not multigraph: + raise ValueError( + "source graph is a multigraph; pass --multigraph to preserve " + "parallel relations in the global graph" + ) # Prefix IDs for cross-project isolation prefixed = prefix_graph_for_global(src_G, repo_tag) # Load global graph and prune stale nodes for this repo G = _load_global_graph() + if multigraph and not isinstance(G, nx.MultiDiGraph): + from graphify.build import canonical_edge_key + + upgraded = nx.MultiDiGraph() + upgraded.add_nodes_from(G.nodes(data=True)) + for source, target, attrs in G.edges(data=True): + copied = dict(attrs) + true_source = copied.get("_src", source) + true_target = copied.get("_tgt", target) + upgraded.add_edge( + true_source, + true_target, + key=canonical_edge_key(str(true_source), str(true_target), copied), + **copied, + ) + G = upgraded removed = prune_repo_from_graph(G, repo_tag) # Merge external-library nodes (no source_file) by label to avoid duplication @@ -137,11 +158,27 @@ def global_add(source_path: Path, repo_tag: str) -> dict: for node, data in prefixed.nodes(data=True): if node not in remap: G.add_node(node, **data) - for u, v, data in prefixed.edges(data=True): - u = remap.get(u, u) - v = remap.get(v, v) - if u != v: # don't introduce self-loops via remapping - G.add_edge(u, v, **data) + if G.is_multigraph(): + if prefixed.is_multigraph(): + rows = prefixed.edges(keys=True, data=True) + else: + from graphify.build import canonical_edge_key + + rows = ( + (u, v, canonical_edge_key(str(u), str(v), attrs), attrs) + for u, v, attrs in prefixed.edges(data=True) + ) + for u, v, key, data in rows: + u = remap.get(u, u) + v = remap.get(v, v) + if u != v: + G.add_edge(u, v, key=key, **data) + else: + for u, v, data in prefixed.edges(data=True): + u = remap.get(u, u) + v = remap.get(v, v) + if u != v: # don't introduce self-loops via remapping + G.add_edge(u, v, **data) added = prefixed.number_of_nodes() - len(remap) _save_global_graph(G) diff --git a/graphify/serve.py b/graphify/serve.py index f32a91673..1e4ab4f6a 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -737,6 +737,22 @@ def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> return H +def _bidirectional_neighbors(G: nx.Graph, node: str): + """Yield each adjacent node once, traversing both directions.""" + if not G.is_directed(): + yield from G.neighbors(node) + return + seen: set[str] = set() + for neighbor in G.successors(node): + if neighbor not in seen: + seen.add(neighbor) + yield neighbor + for neighbor in G.predecessors(node): + if neighbor not in seen: + seen.add(neighbor) + yield neighbor + + def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: # Compute hub threshold: nodes above this degree are not expanded as transit. # p99 of degree distribution, floored at 50 to avoid over-blocking small graphs. @@ -758,7 +774,7 @@ def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis # is the starting node should still be explored). if n not in seed_set and G.degree(n) >= hub_threshold: continue - for neighbor in G.neighbors(n): + for neighbor in _bidirectional_neighbors(G, n): if neighbor not in visited: next_frontier.add(neighbor) edges_seen.append((n, neighbor)) @@ -786,7 +802,7 @@ def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis visited.add(node) if node not in seed_set and G.degree(node) >= hub_threshold: continue - for neighbor in G.neighbors(node): + for neighbor in _bidirectional_neighbors(G, node): if neighbor not in visited: stack.append((neighbor, d + 1)) edges_seen.append((node, neighbor)) @@ -858,39 +874,50 @@ def _adj(n): lines.append(line) for u, v in edges: if u in nodes and v in nodes: - raw = G[u][v] - d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw + from graphify.build import edge_datas + + oriented_datas: list[tuple[str, str, dict]] = [] + if G.has_edge(u, v): + oriented_datas.extend((u, v, data) for data in edge_datas(G, u, v)) + if G.is_directed() and G.has_edge(v, u): + oriented_datas.extend((v, u, data) for data in edge_datas(G, v, u)) + for stored_src, stored_tgt, d in oriented_datas: # (u, v) is BFS/DFS visit order, not necessarily the true edge # direction: on an undirected graph G.neighbors() walks callers # and callees alike, so a caller->callee edge renders backwards # whenever the callee is visited first. _src/_tgt (stashed on the # edge data by the `query` CLI loader) carry the real direction; # fall back to (u, v) for graphs/edges that don't set them. - src = d.get("_src", u) - tgt = d.get("_tgt", v) + src = d.get("_src", stored_src) + tgt = d.get("_tgt", stored_tgt) # Guard against a stray/dangling _src/_tgt (hand-edited or adversarial # graph.json): only trust them when they name exactly this edge's # endpoints, else fall back to (u, v). Without this, G.nodes[src] # would KeyError on an unknown id (#2080 review). - if {src, tgt} != {u, v}: - src, tgt = u, v - context = d.get("context") - context_suffix = f" context={sanitize_label(str(context))}" if context else "" + if {src, tgt} != {u, v}: + src, tgt = stored_src, stored_tgt + context = d.get("context") + context_suffix = f" context={sanitize_label(str(context))}" if context else "" # The relation SITE (call/import/reference line in the source's # file), not a def line — so "who calls X" cites a clickable call # location, not the caller's def (#BUG1). - _loc = str(d.get("source_location") or "") - at_suffix = ( - f" at={sanitize_label(str(d.get('source_file') or ''))}:{sanitize_label(_loc)}" - if _loc else "" - ) - line = ( - f"EDGE {sanitize_label(G.nodes[src].get('label', src))} " - f"--{sanitize_label(str(d.get('relation', '')))} " - f"[{sanitize_label(str(d.get('confidence', '')))}{context_suffix}]--> " - f"{sanitize_label(G.nodes[tgt].get('label', tgt))}{at_suffix}" - ) - lines.append(line) + _loc = str(d.get("source_location") or "") + at_suffix = ( + f" at={sanitize_label(str(d.get('source_file') or ''))}:{sanitize_label(_loc)}" + if _loc else "" + ) + occurrence_count = d.get("occurrence_count", 1) + occurrence_suffix = ( + f" occurrences={occurrence_count}" if occurrence_count != 1 else "" + ) + line = ( + f"EDGE {sanitize_label(G.nodes[src].get('label', src))} " + f"--{sanitize_label(str(d.get('relation', '')))} " + f"[{sanitize_label(str(d.get('confidence', '')))}" + f"{context_suffix}{occurrence_suffix}]--> " + f"{sanitize_label(G.nodes[tgt].get('label', tgt))}{at_suffix}" + ) + lines.append(line) output = "\n".join(lines) if len(output) > char_budget: cut_at = output[:char_budget].rfind("\n") @@ -1394,23 +1421,25 @@ def _edge_at(d: dict) -> str: if loc else "" ) for nb in G.successors(nid): - d = edge_data(G, nid, nb) - rel = d.get("relation", "") - if rel_filter and rel_filter not in rel.lower(): - continue - lines.append( - f" --> {sanitize_label(G.nodes[nb].get('label', nb))} " - f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" - ) + for d in edge_datas(G, nid, nb): + rel = d.get("relation", "") + if rel_filter and rel_filter not in rel.lower(): + continue + lines.append( + f" --> {sanitize_label(G.nodes[nb].get('label', nb))} " + f"[{sanitize_label(str(rel))}] " + f"[{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" + ) for nb in G.predecessors(nid): - d = edge_data(G, nb, nid) - rel = d.get("relation", "") - if rel_filter and rel_filter not in rel.lower(): - continue - lines.append( - f" <-- {sanitize_label(G.nodes[nb].get('label', nb))} " - f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" - ) + for d in edge_datas(G, nb, nid): + rel = d.get("relation", "") + if rel_filter and rel_filter not in rel.lower(): + continue + lines.append( + f" <-- {sanitize_label(G.nodes[nb].get('label', nb))} " + f"[{sanitize_label(str(rel))}] " + f"[{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" + ) budget = int(arguments.get("token_budget", 2000)) return _cut_lines_to_budget( lines, budget, "Narrow with relation_filter or use get_node for a specific symbol" diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 870c68952..7352492e4 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 4f03ccbae..7be15f626 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -387,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -403,7 +403,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -464,7 +464,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +502,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -533,7 +533,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -556,7 +556,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -574,7 +574,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -597,7 +597,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -618,7 +618,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -824,7 +824,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -848,7 +848,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 870c68952..7352492e4 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index ff44d276c..21742630f 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 8ed58de66..bdc603b2c 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index ff44d276c..21742630f 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index e3e6d2dec..22e87e13e 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -452,7 +452,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -468,7 +468,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -529,7 +529,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -567,7 +567,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -598,7 +598,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -652,7 +652,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -675,7 +675,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -692,7 +692,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -715,7 +715,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -736,7 +736,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -961,7 +961,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -984,7 +984,7 @@ from pathlib import Path old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 6f24bbf2b..d22d36045 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 22a4c6896..2ff196fd4 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index ff44d276c..21742630f 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index a148428b1..7ea78e4d4 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -381,7 +382,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -399,7 +400,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -453,7 +454,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -461,9 +462,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -490,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index ff44d276c..21742630f 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 472336828..ab5d1928c 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -387,7 +388,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -405,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -459,7 +460,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -467,9 +468,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -496,7 +503,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 5841beacd..d87691eae 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -385,7 +386,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -403,7 +404,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -457,7 +458,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -465,9 +466,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -494,7 +501,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 056bef11d..e638a685e 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -411,7 +412,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -429,7 +430,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -483,7 +484,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -491,9 +492,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -520,7 +527,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skill.md b/graphify/skill.md index ff44d276c..21742630f 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd412..7eef933c5 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/graphify/watch.py b/graphify/watch.py index 1ef1ebd4d..944edc034 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -81,12 +81,13 @@ def _write_build_config( *, excludes: "list[str] | None", gitignore: bool | None = None, + multigraph: bool | None = None, ) -> None: """Persist corpus-shaping options under ``out_dir``. Best effort and non clobbering: omitted options retain their existing values. """ - if not excludes and gitignore is None: + if not excludes and gitignore is None and multigraph is None: return try: out_dir.mkdir(parents=True, exist_ok=True) @@ -101,6 +102,8 @@ def _write_build_config( config["excludes"] = list(excludes) if gitignore is not None: config["gitignore"] = gitignore + if multigraph is not None: + config["multigraph"] = multigraph path.write_text(json.dumps(config), encoding="utf-8") except OSError: pass @@ -133,6 +136,19 @@ def _read_build_gitignore(out_dir: Path) -> bool: return True +def _read_build_multigraph(out_dir: Path) -> bool: + """Return whether rebuilds must preserve MultiDiGraph output.""" + try: + path = out_dir / _BUILD_CONFIG_FILENAME + if path.is_file(): + cfg = json.loads(path.read_text(encoding="utf-8")) + if isinstance(cfg, dict) and isinstance(cfg.get("multigraph"), bool): + return cfg["multigraph"] + except (OSError, json.JSONDecodeError): + pass + return False + + def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: """Concatenate path lists, preserving order and dropping duplicates. @@ -843,6 +859,7 @@ def _rebuild_code( follow_symlinks: bool = False, force: bool = False, no_cluster: bool = False, + multigraph: bool | None = None, acquire_lock: bool = True, block_on_lock: bool = False, ) -> bool: @@ -903,6 +920,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + multigraph=multigraph, acquire_lock=False, ) # Late-arrival drain: another hook may have queued work while we @@ -920,6 +938,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + multigraph=multigraph, acquire_lock=False, ) and ok return ok @@ -1135,7 +1154,19 @@ def _add_deleted_source(path: Path) -> None: rebuilt_sources |= set(deleted_paths) out.mkdir(exist_ok=True) - if no_cluster: + effective_multigraph = ( + multigraph + if multigraph is not None + else ( + bool(existing_graph_data.get("multigraph")) + if isinstance(existing_graph_data, dict) + else _read_build_multigraph(out) + ) + ) + if effective_multigraph: + _write_build_config(out, excludes=None, multigraph=True) + + if no_cluster and not effective_multigraph: # Normalise to "links" key so schema is consistent with the full clustered path. # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); # without it, --no-cluster + repeated `update` accumulate duplicates and edge @@ -1206,7 +1237,11 @@ def _add_deleted_source(path: Path) -> None: "total_words": detected.get("total_words", 0), } - G = build_from_json(result) + G = build_from_json( + result, + directed=effective_multigraph, + multigraph=effective_multigraph, + ) candidate_topology = _topology_from_graph(G) if existing_graph_data: try: @@ -1232,7 +1267,7 @@ def _add_deleted_source(path: Path) -> None: print("[graphify watch] No code-graph topology changes detected; outputs left untouched.") return True - communities = cluster(G) + communities = {} if no_cluster else cluster(G) previous_node_community = _node_community_map(existing_graph_data) if previous_node_community: communities = remap_communities_to_previous(communities, previous_node_community) diff --git a/graphify/wiki.py b/graphify/wiki.py index cb9c6cf35..dfebe2d71 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -6,7 +6,22 @@ from urllib.parse import quote import networkx as nx -from graphify.build import edge_data +from graphify.build import edge_datas + + +def _incident_edges(G: nx.Graph, node_id: str): + """Yield (neighbor, attrs) for every incoming and outgoing edge.""" + if G.is_directed(): + for neighbor in G.successors(node_id): + for attrs in edge_datas(G, node_id, neighbor): + yield neighbor, attrs + for neighbor in G.predecessors(node_id): + for attrs in edge_datas(G, neighbor, node_id): + yield neighbor, attrs + else: + for neighbor in G.neighbors(node_id): + for attrs in edge_datas(G, node_id, neighbor): + yield neighbor, attrs def _safe_filename(name: str) -> str: @@ -76,8 +91,7 @@ def _community_article( # Edge confidence breakdown conf_counts: Counter = Counter() for nid in nodes: - for neighbor in G.neighbors(nid): - ed = edge_data(G, nid, neighbor) + for _neighbor, ed in _incident_edges(G, nid): conf_counts[ed.get("confidence", "EXTRACTED")] += 1 total_edges = sum(conf_counts.values()) or 1 @@ -146,9 +160,13 @@ def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str], node_commun # Group neighbors by relation type by_relation: dict[str, list[str]] = {} - for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True): + incident = sorted( + _incident_edges(G, nid), + key=lambda item: G.degree(item[0]), + reverse=True, + ) + for neighbor, ed in incident: nd = G.nodes[neighbor] - ed = edge_data(G, nid, neighbor) rel = ed.get("relation", "related") neighbor_label = nd.get("label", neighbor) conf = ed.get("confidence", "") diff --git a/tests/test_build.py b/tests/test_build.py index 3d8582c35..95d153fd9 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1061,3 +1061,123 @@ def test_build_from_json_prunes_dangling_hyperedge_members(capsys): assert set(hes) == {"he_partial"}, "an all-dangling hyperedge must be dropped" assert hes["he_partial"]["nodes"] == ["alpha", "beta"] assert "he_all_ghost" in capsys.readouterr().err + + +def test_multigraph_preserves_distinct_parallel_relations_and_directions(): + extraction = { + "nodes": [ + {"id": "a", "label": "A", "source_file": "a.py", "file_type": "code"}, + {"id": "b", "label": "B", "source_file": "b.py", "file_type": "code"}, + ], + "edges": [ + { + "source": "a", "target": "b", "relation": "references", + "confidence": "EXTRACTED", "source_file": "a.py", + "source_location": "L10", "context": "parameter_type", + }, + { + "source": "a", "target": "b", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "a.py", + "source_location": "L20", "context": "call", + }, + { + "source": "b", "target": "a", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "b.py", + "source_location": "L30", "context": "call", + }, + ], + } + + graph = build_from_json(extraction, multigraph=True) + + assert isinstance(graph, nx.MultiDiGraph) + assert graph.number_of_edges("a", "b") == 2 + assert graph.number_of_edges("b", "a") == 1 + assert { + (data["relation"], data["source_location"], data["context"]) + for data in edge_datas(graph, "a", "b") + } == { + ("references", "L10", "parameter_type"), + ("calls", "L20", "call"), + } + + +def test_multigraph_aggregates_exact_duplicates_with_occurrence_count(): + edge = { + "source": "a", "target": "b", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "a.py", + "source_location": "L20", "context": "call", + } + extraction = { + "nodes": [ + {"id": "a", "label": "A", "source_file": "a.py", "file_type": "code"}, + {"id": "b", "label": "B", "source_file": "b.py", "file_type": "code"}, + ], + "edges": [dict(edge), dict(edge), dict(edge)], + } + + graph = build_from_json(extraction, multigraph=True) + + assert graph.number_of_edges("a", "b") == 1 + assert edge_datas(graph, "a", "b")[0]["occurrence_count"] == 3 + + +def test_multigraph_keys_are_stable_across_order_and_checkout_root(tmp_path): + left = tmp_path / "left" + right = tmp_path / "right" + left.mkdir() + right.mkdir() + + def extraction(root): + return { + "nodes": [ + { + "id": "a", "label": "A", + "source_file": str(root / "src" / "a.py"), "file_type": "code", + "_origin": "ast", + }, + { + "id": "b", "label": "B", + "source_file": str(root / "src" / "b.py"), "file_type": "code", + "_origin": "ast", + }, + ], + "edges": [ + { + "source": "a", "target": "b", "relation": "calls", + "confidence": "EXTRACTED", + "source_file": str(root / "src" / "a.py"), + "source_location": "L20", "context": "call", + }, + { + "source": "a", "target": "b", "relation": "references", + "confidence": "EXTRACTED", + "source_file": str(root / "src" / "a.py"), + "source_location": "L10", "context": "parameter_type", + }, + ], + } + + first = build_from_json(extraction(left), multigraph=True, root=left) + second_input = extraction(right) + second_input["edges"].reverse() + second = build_from_json(second_input, multigraph=True, root=right) + + assert set(first["a"]["b"]) == set(second["a"]["b"]) + assert all(str(left) not in key and str(right) not in key for key in first["a"]["b"]) + + +def test_analysis_projection_counts_each_endpoint_pair_once(): + graph = nx.MultiDiGraph() + graph.add_nodes_from(["a", "b"]) + graph.add_edge("a", "b", key="one") + graph.add_edge("a", "b", key="two") + graph.add_edge("b", "a", key="reverse") + + from graphify.build import analysis_projection + + projected = analysis_projection(graph) + + assert isinstance(projected, nx.Graph) + assert projected.number_of_edges() == 1 + assert projected["a"]["b"]["weight"] == 1.0 diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index d3629a73c..05673ac3c 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -264,3 +264,37 @@ def test_prune_reextracted_absolute_node_not_deleted(tmp_path): G = build_merge([new_chunk], graph_path, prune_sources=["mod.py"], dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "gone" in labels, "re-extracted file wrongly pruned across mismatched forms (#2012/#1796)" + + +def test_build_merge_preserves_existing_multigraph_mode(tmp_path): + from graphify.export import to_json + + graph_path = tmp_path / "graphify-out" / "graph.json" + graph_path.parent.mkdir() + initial = build_merge( + [{ + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + ], + "edges": [ + { + "source": "a", "target": "b", "relation": "calls", + "source_file": "a.py", "source_location": "L1", + }, + { + "source": "a", "target": "b", "relation": "references", + "source_file": "a.py", "source_location": "L2", + }, + ], + }], + graph_path=graph_path, + multigraph=True, + ) + to_json(initial, {}, str(graph_path), force=True) + + merged = build_merge([], graph_path=graph_path) + + assert merged.is_directed() + assert merged.is_multigraph() + assert merged.number_of_edges("a", "b") == 2 diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 94ba22b9c..79ce9189b 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -152,6 +152,41 @@ def test_explain_connection_shows_call_site_line(monkeypatch, tmp_path, capsys): assert "state.py" in out and "L56" in out +def test_explain_lists_every_parallel_edge(monkeypatch, tmp_path, capsys): + graph_data = { + "directed": True, "multigraph": True, "graph": {}, + "nodes": [ + {"id": "stage", "label": "StageTrackPaths.__init__", + "source_file": "stage_track_paths.py", "community": 0}, + {"id": "path", "label": "Path", + "source_file": "pathlib.pyi", "community": 0}, + ], + "links": [ + { + "source": "stage", "target": "path", "key": "references", + "relation": "references", "confidence": "EXTRACTED", + "context": "parameter_type", "source_file": "stage_track_paths.py", + "source_location": "L982", "occurrence_count": 6, + }, + { + "source": "stage", "target": "path", "key": "calls", + "relation": "calls", "confidence": "EXTRACTED", + "context": "call", "source_file": "stage_track_paths.py", + "source_location": "L1002", "occurrence_count": 1, + }, + ], + } + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(graph_data)) + + out = _run(monkeypatch, graph_path, "StageTrackPaths", capsys) + + assert "[references]" in out and "context=parameter_type x6" in out + assert "stage_track_paths.py:L982" in out + assert "[calls]" in out and "context=call" in out + assert "stage_track_paths.py:L1002" in out + + # --- #2009: high-degree nodes must not silently hide the cut connections ------ def _write_high_degree_graph(tmp_path, n_callers=30, files=None): diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 93fec48d3..01ac5f723 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -69,6 +69,25 @@ def test_extract_usage_advertises_code_only(tmp_path): ) +def test_extract_multigraph_persists_mode_with_no_cluster(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text( + "class B:\n pass\n\n" + "def use(value: B):\n return B()\n" + ) + + result = _run(repo, "--code-only", "--multigraph", "--no-cluster") + + assert result.returncode == 0, result.stderr + graph = json.loads((repo / "graphify-out" / "graph.json").read_text()) + assert graph["directed"] is True + assert graph["multigraph"] is True + assert all("key" in edge for edge in graph["links"]) + config = json.loads((repo / "graphify-out" / ".graphify_build.json").read_text()) + assert config["multigraph"] is True + + def _run_relative_out(repo: Path, *extra: str): """Like _run but with a RELATIVE GRAPHIFY_OUT so --out/--output controls the parent dir (an absolute GRAPHIFY_OUT would override the flag).""" diff --git a/tests/test_merge_graphs_cli.py b/tests/test_merge_graphs_cli.py index 5ff06b391..d92e3af31 100644 --- a/tests/test_merge_graphs_cli.py +++ b/tests/test_merge_graphs_cli.py @@ -1,9 +1,10 @@ -"""`graphify merge-graphs` tolerates inputs that disagree on graph type (#1606). +"""`graphify merge-graphs` preserves graph type intentionally (#1606). Per-repo graph.json files written by different extract paths at different times don't always agree on the `directed` / `multigraph` flags. compose requires one uniform type, so a mixed set used to crash with an unhandled NetworkXError. The -handler now normalizes every input to a plain undirected Graph before composing. +Simple inputs still normalize to Graph. Multigraph inputs require an explicit +``--multigraph`` so parallel relationships are never simplified silently. """ from __future__ import annotations @@ -37,15 +38,22 @@ def test_merge_graphs_mixed_directed_and_multigraph(tmp_path): _write(c, directed=False, multigraph=True, node_id="z") # MultiGraph out = tmp_path / "merged.json" - r = _run(["merge-graphs", str(a), str(b), str(c), "--out", str(out)], tmp_path) + refused = _run(["merge-graphs", str(a), str(b), str(c), "--out", str(out)], tmp_path) + assert refused.returncode == 2 + assert "pass --multigraph" in refused.stderr + + r = _run([ + "merge-graphs", str(a), str(b), str(c), + "--out", str(out), "--multigraph", + ], tmp_path) assert r.returncode == 0, f"merge crashed: {r.stderr}" assert out.exists() data = json.loads(out.read_text()) ids = {n["id"] for n in data["nodes"]} - # every input's node survives, normalized into one undirected simple graph + # every input's node survives in one directed multigraph assert {"r1::x", "r2::y", "r3::z"} <= ids or len(ids) == 3 - assert data.get("directed") is False - assert data.get("multigraph") is False + assert data.get("directed") is True + assert data.get("multigraph") is True def test_merge_graphs_same_named_repo_dirs_do_not_collapse(tmp_path): diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 7ad4517b7..897d3de31 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -99,6 +99,10 @@ def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: assert summary["malformed_endpoint_edges"] == 0 assert summary["self_loop_edges"] == 1 assert summary["exact_duplicate_edges"] == 1 + assert summary["canonical_distinct_candidate_edges"] == 4 + assert summary["exact_duplicate_occurrences"] == 1 + assert summary["distinct_parallel_edge_instances"] == 2 + assert summary["opposite_direction_endpoint_pairs"] == 0 assert summary["directed_unique_endpoint_pairs"] == 2 assert summary["directed_same_endpoint_collapsed_edges"] == 3 assert summary["same_endpoint_group_count"] == 1 @@ -106,6 +110,20 @@ def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: assert summary["source_location_variant_groups"] == 1 assert summary["post_build_graph_type"] == "DiGraph" assert summary["post_build_edge_count"] == 2 + assert summary["post_build_lost_distinct_edges"] == 2 + + +def test_diagnose_extraction_multigraph_reports_zero_distinct_loss() -> None: + summary = diagnose_extraction( + _diagnostic_fixture(), + directed=True, + multigraph=True, + ) + + assert summary["post_build_graph_type"] == "MultiDiGraph" + assert summary["post_build_edge_count"] == 4 + assert summary["post_build_preserved_parallel_edges"] == 2 + assert summary["post_build_lost_distinct_edges"] == 0 def test_diagnose_extraction_accepts_node_link_links_key() -> None: diff --git a/tests/test_path_cli.py b/tests/test_path_cli.py index 2584ce4b2..63b4fa58d 100644 --- a/tests/test_path_cli.py +++ b/tests/test_path_cli.py @@ -176,6 +176,39 @@ def test_path_relation_matches_stored_edge_not_fabricated(monkeypatch, tmp_path, assert "calls" not in out +def test_path_shows_parallel_relation_context_location_and_occurrences( + monkeypatch, tmp_path, capsys +): + data = { + "directed": True, "multigraph": True, "graph": {}, + "nodes": [ + {"id": "a", "label": "Alpha", "source_file": "a.py"}, + {"id": "b", "label": "Beta", "source_file": "b.py"}, + ], + "links": [ + { + "source": "a", "target": "b", "key": "calls", + "relation": "calls", "confidence": "EXTRACTED", + "context": "call", "source_file": "a.py", + "source_location": "L20", + }, + { + "source": "a", "target": "b", "key": "references", + "relation": "references", "confidence": "EXTRACTED", + "context": "parameter_type", "source_file": "a.py", + "source_location": "L10", "occurrence_count": 6, + }, + ], + } + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(data)) + + out = _run(monkeypatch, graph_path, "Alpha", "Beta", capsys) + + assert "calls:call@a.py:L20" in out + assert "references:parameter_type@a.py:L10x6" in out + + def test_path_relation_fallback_related_when_missing(monkeypatch, tmp_path, capsys): """#2074: an edge with no stored relation prints an honest 'related', not an empty '---->' arrow and not a fabricated relation.""" diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index 0db4e6fa8..8366b2995 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -106,6 +106,39 @@ def test_query_cli_preserves_calls_direction_when_seeded_on_caller(monkeypatch, assert "callee_fn --calls" not in out +def test_query_cli_traverses_directed_graph_both_ways_and_lists_parallel_edges( + monkeypatch, tmp_path, capsys +): + graph = nx.MultiDiGraph() + graph.add_node("caller", label="caller_fn", source_file="a.py", community=0) + graph.add_node("callee", label="callee_fn", source_file="b.py", community=1) + graph.add_edge( + "caller", "callee", key="calls", relation="calls", + confidence="EXTRACTED", context="call", source_file="a.py", + source_location="L20", + ) + graph.add_edge( + "caller", "callee", key="references", relation="references", + confidence="EXTRACTED", context="parameter_type", source_file="a.py", + source_location="L10", occurrence_count=2, + ) + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links"))) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "query", "callee_fn", "--graph", str(graph_path)], + ) + + mainmod.main() + out = capsys.readouterr().out + + assert "caller_fn --calls" in out + assert "caller_fn --references" in out + assert "occurrences=2" in out + + def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): """#F4: query CLI must refuse to parse a graph.json that exceeds the cap.""" import pytest diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 0c09e601e..65bb5fa0d 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -513,8 +513,9 @@ def test_monoliths_carry_the_1392_runbook_fixes(): # #6/#7 directed propagation: no bare build_from_json call survives, and # the IS_DIRECTED substitution instruction is present. assert "directed=IS_DIRECTED" in body + assert "multigraph=IS_MULTIGRAPH" in body assert "build_from_json(extraction)" not in body - assert "Substitute it everywhere it appears" in body + assert "Substitute both placeholders everywhere they appear" in body # #10 content-only semantic scope: code is no longer flattened in. assert "for cat in ('document', 'paper', 'image')" in body @@ -526,7 +527,10 @@ def test_monoliths_carry_the_1392_runbook_fixes(): # #18/#20 zero-node guard before any write, report/analysis gated on # to_json's return. lines = body.splitlines() - build_i = next(i for i, l in enumerate(lines) if "G = build_from_json(extraction, directed=IS_DIRECTED)" in l) + build_i = next( + i for i, l in enumerate(lines) + if "G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH)" in l + ) guard_i = next(i for i, l in enumerate(lines[build_i:], build_i) if "number_of_nodes() == 0" in l) report_i = next(i for i, l in enumerate(lines[build_i:], build_i) if "GRAPH_REPORT.md').write_text(report)" in l) wrote_i = next(i for i, l in enumerate(lines[build_i:], build_i) if l.strip().startswith("wrote = to_json(")) diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 870c68952..7352492e4 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 4f03ccbae..7be15f626 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -387,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -403,7 +403,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -464,7 +464,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +502,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -533,7 +533,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -556,7 +556,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -574,7 +574,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -597,7 +597,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -618,7 +618,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -824,7 +824,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -848,7 +848,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 870c68952..7352492e4 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index ff44d276c..21742630f 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 8ed58de66..bdc603b2c 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index ff44d276c..21742630f 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index e3e6d2dec..22e87e13e 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -452,7 +452,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -468,7 +468,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -529,7 +529,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -567,7 +567,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -598,7 +598,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -652,7 +652,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -675,7 +675,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -692,7 +692,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -715,7 +715,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -736,7 +736,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -961,7 +961,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -984,7 +984,7 @@ from pathlib import Path old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 6f24bbf2b..d22d36045 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -386,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -404,7 +405,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -458,7 +459,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -466,9 +467,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -495,7 +502,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 22a4c6896..2ff196fd4 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index ff44d276c..21742630f 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index a148428b1..7ea78e4d4 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -381,7 +382,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -399,7 +400,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -453,7 +454,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -461,9 +462,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -490,7 +497,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index ff44d276c..21742630f 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 472336828..ab5d1928c 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -387,7 +388,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -405,7 +406,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -459,7 +460,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -467,9 +468,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -496,7 +503,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 5841beacd..d87691eae 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -385,7 +386,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -403,7 +404,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -457,7 +458,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -465,9 +466,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -494,7 +501,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 056bef11d..e638a685e 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -411,7 +412,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -429,7 +430,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -483,7 +484,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -491,9 +492,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -520,7 +527,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index ff44d276c..21742630f 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -18,6 +18,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -389,7 +390,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -407,7 +408,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -461,7 +462,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -469,9 +470,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -498,7 +505,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 4f03ccbae..7be15f626 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -387,7 +387,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -403,7 +403,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -464,7 +464,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) detection = json.loads(Path('.graphify_detect.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -502,7 +502,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -533,7 +533,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -556,7 +556,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -574,7 +574,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -597,7 +597,7 @@ extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -618,7 +618,7 @@ from pathlib import Path extraction = json.loads(Path('.graphify_extract.json').read_text()) analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -824,7 +824,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -848,7 +848,7 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index e2d90b867..753a36bf6 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -15,6 +15,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --multigraph # preserve parallel directed relations in a MultiDiGraph /graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON @@ -324,7 +325,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** the code blocks below pass `directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH` to `build_from_json()`. Replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; this implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise replace it with `False`. Replace `IS_DIRECTED` with `True` if `--directed` was given, otherwise `False` (the default undirected `Graph`). Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -342,7 +343,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc # root= mirrors the --update runbook (#1361): relativize source_file to the same # base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -396,7 +397,7 @@ from pathlib import Path from graphify.diagnostics import diagnose_extraction, format_diagnostic_report extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') +summary = diagnose_extraction(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH, root='INPUT_PATH') print(format_diagnostic_report(summary)) flags = [f'{summary[k]} {label}' for k, label in ( ('unresolved_internal_endpoint_edges', 'unresolved-internal endpoint edges'), @@ -404,9 +405,15 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('unclassified_endpoint_edges', 'unclassified endpoint edges'), ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ('post_build_lost_distinct_edges', 'distinct relations lost after build'), ) if summary.get(k, 0)] +if not IS_MULTIGRAPH: + flags.extend( + f'{summary[k]} {label}' for k, label in ( + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), + ) if summary.get(k, 0) + ) print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') " ``` @@ -433,7 +440,7 @@ detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(enc analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) # root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) +G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index e3e6d2dec..22e87e13e 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -452,7 +452,7 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. +**Before starting:** replace `IS_MULTIGRAPH` with `True` when `--multigraph` was given; it implies `IS_DIRECTED=True` and builds a `MultiDiGraph`. Otherwise use `False`. Replace `IS_DIRECTED` with `True` for `--directed`, otherwise `False`. Substitute both placeholders everywhere they appear. ```bash mkdir -p graphify-out @@ -468,7 +468,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Guard BEFORE any write: an empty extraction must not clobber a good graph.json / # GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). if G.number_of_nodes() == 0: @@ -529,7 +529,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} @@ -567,7 +567,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -598,7 +598,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -652,7 +652,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -675,7 +675,7 @@ from graphify.build import build_from_json from graphify.export import to_cypher from pathlib import Path -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) to_cypher(G, 'graphify-out/cypher.txt') print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') " @@ -692,7 +692,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) @@ -715,7 +715,7 @@ extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} labels = {int(k): v for k, v in labels_raw.items()} @@ -736,7 +736,7 @@ from pathlib import Path extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) +G = build_from_json(extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) communities = {int(k): v for k, v in analysis['communities'].items()} to_graphml(G, communities, 'graphify-out/graph.graphml') @@ -961,7 +961,7 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') # Load new extraction new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) +G_new = build_from_json(new_extraction, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -984,7 +984,7 @@ from pathlib import Path old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json(new_extract, directed=IS_DIRECTED, multigraph=IS_MULTIGRAPH) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 3632fd412..7eef933c5 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -106,15 +106,16 @@ prune = list(deleted) or None # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). +# Replace IS_MULTIGRAPH with True for a graph created with --multigraph. It implies +# directed mode and preserves stable keyed parallel relations across the merge. +# Replace IS_DIRECTED with True for --directed, otherwise False. G = build_merge( [new_extraction], graph_path='graphify-out/graph.json', prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -181,7 +182,11 @@ from pathlib import Path # Load old graph (before update) from backup written before merge old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +G_new = build_from_json( + new_extract, + directed=IS_DIRECTED, + multigraph=IS_MULTIGRAPH, +) if old_data: G_old = json_graph.node_link_graph(old_data, edges='links') diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 732215e8a..e25098bca 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -776,8 +776,9 @@ def _is_directed_fix_line(line: str) -> bool: """ return ( "build_from_json(" in line and "import" not in line - ) or "directed=IS_DIRECTED" in line or ( - "IS_DIRECTED" in line and "Substitute it everywhere" in line + ) or "directed=IS_DIRECTED" in line or "multigraph=IS_MULTIGRAPH" in line or ( + ("IS_DIRECTED" in line or "IS_MULTIGRAPH" in line) + and ("Substitute it everywhere" in line or "Substitute both placeholders" in line) ) From d266d8c667d71c5b2894efce7c51f22d067ad6a6 Mon Sep 17 00:00:00 2001 From: Henrique Silveira Neves Date: Fri, 24 Jul 2026 22:23:06 -0300 Subject: [PATCH 3/5] docs: expose multigraph flags in CLI help --- graphify/__main__.py | 4 ++++ tests/test_multigraph_help.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/test_multigraph_help.py diff --git a/graphify/__main__.py b/graphify/__main__.py index 924ae986d..26986f133 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -521,6 +521,7 @@ def _run_cli() -> None: print(" --max-examples N max same-endpoint examples to print (default 5)") print(" --directed force directed post-build simulation") print(" --undirected force undirected post-build simulation") + print(" --multigraph force MultiDiGraph post-build simulation") print(" (default follows JSON directed flag;") print(" raw extraction with no flag defaults directed)") print(" --extract-path PATH extractor source for suppression scan") @@ -528,6 +529,7 @@ def _run_cli() -> None: print(" merge-driver git merge driver: union-merge two graph.json files (set up via hook install)") print(" merge-graphs merge two or more graph.json files into one cross-repo graph") print(" --out output path (default: graphify-out/merged-graph.json)") + print(" --multigraph preserve parallel relations in a directed multigraph") print(" --branch checkout a specific branch (default: repo default)") print(" --out clone to a custom directory (default: ~/.graphify/repos//)") print(" add fetch a URL and save it to ./raw, then update the graph") @@ -536,6 +538,7 @@ def _run_cli() -> None: print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") print(" update re-extract code files and update the graph (no LLM needed)") + print(" --multigraph preserve parallel directed relations (also preserves existing mode)") print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" --no-cluster skip clustering, write raw extraction only") @@ -609,6 +612,7 @@ def _run_cli() -> None: print(" --api-timeout S per-request timeout in seconds for the LLM client (default: 600)") print(" --out DIR, --output DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") + print(" --multigraph preserve parallel directed relations in a MultiDiGraph") print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") print(" --no-cluster skip clustering, write raw extraction only") print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") diff --git a/tests/test_multigraph_help.py b/tests/test_multigraph_help.py new file mode 100644 index 000000000..967c2e153 --- /dev/null +++ b/tests/test_multigraph_help.py @@ -0,0 +1,18 @@ +"""The top-level CLI help advertises every MultiDiGraph entry point.""" + +from __future__ import annotations + +import sys + + +def test_main_help_documents_multigraph_flags(capsys, monkeypatch): + from graphify.__main__ import main + + monkeypatch.setattr(sys, "argv", ["graphify", "--help"]) + main() + output = capsys.readouterr().out + + assert "force MultiDiGraph post-build simulation" in output + assert "preserve parallel relations in a directed multigraph" in output + assert "preserve parallel directed relations (also preserves existing mode)" in output + assert "preserve parallel directed relations in a MultiDiGraph" in output From fb77da2b4fa17adcf13e89b19136a37fa8f8c04b Mon Sep 17 00:00:00 2001 From: Henrique Silveira Neves Date: Fri, 24 Jul 2026 22:53:35 -0300 Subject: [PATCH 4/5] fix: classify repeated edge occurrences --- CHANGELOG.md | 8 + graphify/build.py | 32 ++- graphify/cli.py | 32 ++- graphify/diagnostics.py | 85 +++++++- graphify/export.py | 31 ++- graphify/exporters/graphdb.py | 31 ++- graphify/exporters/html.py | 16 ++ graphify/extract.py | 196 +++++++++++++++++- graphify/extractors/engine.py | 113 ++++++++-- graphify/extractors/json_config.py | 52 ++++- graphify/serve.py | 20 +- graphify/skill-agents.md | 8 +- graphify/skill-aider.md | 2 + graphify/skill-amp.md | 8 +- graphify/skill-claw.md | 8 +- graphify/skill-codex.md | 8 +- graphify/skill-copilot.md | 8 +- graphify/skill-devin.md | 2 + graphify/skill-droid.md | 8 +- graphify/skill-kilo.md | 8 +- graphify/skill-kiro.md | 8 +- graphify/skill-opencode.md | 8 +- graphify/skill-pi.md | 8 +- graphify/skill-trae.md | 8 +- graphify/skill-vscode.md | 8 +- graphify/skill-windows.md | 8 +- graphify/skill.md | 8 +- tests/test_edge_occurrences.py | 122 +++++++++++ tests/test_explain_cli.py | 7 + tests/test_export.py | 45 ++++ tests/test_extract.py | 11 +- tests/test_path_cli.py | 4 + tests/test_query_cli.py | 5 + tests/test_zero_node_no_cache.py | 12 ++ .../expected/graphify__skill-agents.md | 8 +- .../expected/graphify__skill-aider.md | 2 + .../skillgen/expected/graphify__skill-amp.md | 8 +- .../skillgen/expected/graphify__skill-claw.md | 8 +- .../expected/graphify__skill-codex.md | 8 +- .../expected/graphify__skill-copilot.md | 8 +- .../expected/graphify__skill-devin.md | 2 + .../expected/graphify__skill-droid.md | 8 +- .../skillgen/expected/graphify__skill-kilo.md | 8 +- .../skillgen/expected/graphify__skill-kiro.md | 8 +- .../expected/graphify__skill-opencode.md | 8 +- tools/skillgen/expected/graphify__skill-pi.md | 8 +- .../skillgen/expected/graphify__skill-trae.md | 8 +- .../expected/graphify__skill-vscode.md | 8 +- .../expected/graphify__skill-windows.md | 8 +- tools/skillgen/expected/graphify__skill.md | 8 +- tools/skillgen/fragments/core/aider.md | 2 + tools/skillgen/fragments/core/core.md | 8 +- tools/skillgen/fragments/core/devin.md | 2 + tools/skillgen/gen.py | 8 + 54 files changed, 988 insertions(+), 86 deletions(-) create mode 100644 tests/test_edge_occurrences.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0512d23..802667f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## Unreleased +- Fix: AST edges now retain portable token-level occurrence evidence. Python + annotations use their real line/column span and parameter name, repeated + evidence on one relation is aggregated deterministically, and duplicate + emissions introduced by post-resolution endpoint rewrites are suppressed. + Diagnostics separate legitimate source repetition from producer defects. + Deliberately unindexed data JSON is now a cached intentional skip rather than + a recurring zero-node warning, with failures and unsupported files reported + separately. - Feat: add opt-in `--multigraph` builds backed by a persistent directed `MultiDiGraph`. Canonically distinct relations between the same endpoints retain stable keys; exact duplicate occurrences aggregate through diff --git a/graphify/build.py b/graphify/build.py index 5be6042aa..5f4817b6b 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -324,7 +324,7 @@ def canonical_edge_key(source: str, target: str, attrs: dict) -> str: semantic = { key: value for key, value in attrs.items() - if key not in {"key", "occurrence_count", "_src", "_tgt"} + if key not in {"key", "occurrences", "occurrence_count", "_src", "_tgt"} } payload = { "source": source, @@ -989,6 +989,7 @@ def _edge_sort_key(edge: dict) -> tuple: continue if multigraph: edge_key = canonical_edge_key(str(src), str(tgt), attrs) + incoming_occurrences = attrs.pop("occurrences", []) incoming_count = attrs.pop("occurrence_count", 1) try: incoming_count = max(1, int(incoming_count)) @@ -996,11 +997,32 @@ def _edge_sort_key(edge: dict) -> tuple: incoming_count = 1 if G.has_edge(src, tgt, edge_key): existing = G[src][tgt][edge_key] - existing["occurrence_count"] = ( - int(existing.get("occurrence_count", 1)) + incoming_count - ) + merged_occurrences = list(existing.get("occurrences", [])) + seen = { + json.dumps(item, sort_keys=True, default=str) + for item in merged_occurrences + } + for item in incoming_occurrences: + fingerprint = json.dumps(item, sort_keys=True, default=str) + if fingerprint not in seen: + seen.add(fingerprint) + merged_occurrences.append(item) + if merged_occurrences: + merged_occurrences.sort( + key=lambda item: json.dumps(item, sort_keys=True, default=str) + ) + existing["occurrences"] = merged_occurrences + existing["occurrence_count"] = len(merged_occurrences) + else: + existing["occurrence_count"] = ( + int(existing.get("occurrence_count", 1)) + incoming_count + ) else: - attrs["occurrence_count"] = incoming_count + if incoming_occurrences: + attrs["occurrences"] = incoming_occurrences + attrs["occurrence_count"] = len(incoming_occurrences) + else: + attrs["occurrence_count"] = incoming_count G.add_edge(src, tgt, key=edge_key, **attrs) else: G.add_edge(src, tgt, **attrs) diff --git a/graphify/cli.py b/graphify/cli.py index 712100977..21860999b 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -85,6 +85,27 @@ def _default_graph_path() -> str: return str(Path(_GRAPHIFY_OUT) / "graph.json") +def _format_occurrence_details(data: dict, limit: int = 3) -> str: + occurrences = data.get("occurrences") + if not isinstance(occurrences, list) or not occurrences: + return "" + rendered = [] + for occurrence in occurrences[:limit]: + if not isinstance(occurrence, dict): + continue + detail = str( + occurrence.get("source_span") + or occurrence.get("source_location") + or "unlocated" + ) + if occurrence.get("parameter"): + detail += f"({occurrence['parameter']})" + rendered.append(detail) + if len(occurrences) > limit: + rendered.append(f"+{len(occurrences) - limit}") + return ",".join(rendered) + + def _stamped_manifest_files( files_by_type: dict[str, list[str]], sem_result: dict, @@ -1199,6 +1220,9 @@ def dispatch_command(cmd: str) -> None: detail += f"@{source_file or ''}{':' + str(location) if location else ''}" if occurrences != 1: detail += f"x{occurrences}" + evidence = _format_occurrence_details(data) + if evidence: + detail += f"[{evidence}]" relation_details.append(detail) rel = " | ".join(relation_details) if relation_details else "related" confs = sorted({d.get("confidence") for d in datas if d.get("confidence")}) @@ -1312,9 +1336,11 @@ def dispatch_command(cmd: str) -> None: context_text = f" context={context}" if context else "" occurrences = edata.get("occurrence_count", 1) occurrence_text = f" x{occurrences}" if occurrences != 1 else "" + evidence = _format_occurrence_details(edata) + evidence_text = f" evidence={evidence}" if evidence else "" print( f" {arrow} {G.nodes[nb].get('label', nb)} " - f"[{rel}] [{conf}]{context_text}{occurrence_text}{at}" + f"[{rel}] [{conf}]{context_text}{occurrence_text}{evidence_text}{at}" ) if len(connections) > 20: remainder = connections[20:] @@ -3271,6 +3297,10 @@ def _progress(idx: int, total: int, _result: dict) -> None: "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), "hyperedges": list(sem_result.get("hyperedges", [])), + "extraction_diagnostics": dict( + ast_result.get("extraction_diagnostics", {}) + ), + "file_outcomes": list(ast_result.get("file_outcomes", [])), "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), } diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index 92eb6a555..e78cf66bb 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -285,8 +285,36 @@ def _remember(category: str, edge: dict[str, Any]) -> None: valid_signatures_by_pair[directed_pair].add(signature) canonical_distinct_candidate_edges = len(exact_counts) - exact_duplicate_occurrences = ( - _count_extra(exact_counts) + encoded_duplicate_occurrences + extraction_diagnostics = extraction.get("extraction_diagnostics", {}) + suppressed_producer_duplicates = int( + extraction_diagnostics.get( + "suppressed_producer_duplicate_occurrences", 0 + ) or 0 + ) + if extraction_diagnostics: + exact_duplicate_occurrences = ( + int( + extraction_diagnostics.get( + "legitimate_repeated_source_occurrences", 0 + ) or 0 + ) + + suppressed_producer_duplicates + + int( + extraction_diagnostics.get("unlocated_duplicate_occurrences", 0) + or 0 + ) + ) + else: + exact_duplicate_occurrences = ( + _count_extra(exact_counts) + encoded_duplicate_occurrences + ) + file_outcomes = [ + outcome for outcome in extraction.get("file_outcomes", []) + if isinstance(outcome, dict) + ] + outcome_counts = Counter( + str(outcome.get("status") or "unexpected_empty") + for outcome in file_outcomes ) distinct_parallel_edge_instances = sum( max(0, len(signatures) - 1) @@ -375,6 +403,31 @@ def _remember(category: str, edge: dict[str, Any]) -> None: "exact_duplicate_edges": exact_duplicate_occurrences, "canonical_distinct_candidate_edges": canonical_distinct_candidate_edges, "exact_duplicate_occurrences": exact_duplicate_occurrences, + "legitimate_repeated_source_occurrences": int( + extraction_diagnostics.get( + "legitimate_repeated_source_occurrences", + encoded_duplicate_occurrences, + ) or 0 + ), + "suppressed_producer_duplicate_occurrences": suppressed_producer_duplicates, + "unlocated_duplicate_occurrences": int( + extraction_diagnostics.get("unlocated_duplicate_occurrences", 0) or 0 + ), + "post_normalization_exact_duplicate_edges": int( + extraction_diagnostics.get("post_normalization_exact_duplicate_edges", 0) + or 0 + ), + "post_normalization_unclassified_duplicates": int( + extraction_diagnostics.get( + "post_normalization_unclassified_duplicates", 0 + ) or 0 + ), + "zero_node_files": len(file_outcomes), + "intentionally_skipped_files": outcome_counts["skipped_intentional"], + "failed_extraction_files": outcome_counts["failed"], + "unexpected_empty_files": outcome_counts["unexpected_empty"], + "unsupported_extraction_files": outcome_counts["unsupported"], + "file_outcome_examples": file_outcomes[:max_examples], "distinct_parallel_edge_instances": distinct_parallel_edge_instances, "opposite_direction_endpoint_pairs": opposite_direction_endpoint_pairs, "directed_unique_endpoint_pairs": len(directed_pairs), @@ -467,11 +520,15 @@ def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: "summary": { key: value for key, value in summary.items() - if key not in {"examples", "endpoint_examples", "producer_suppression"} + if key not in { + "examples", "endpoint_examples", "producer_suppression", + "file_outcome_examples", + } }, "examples": summary.get("examples", []), "endpoint_examples": summary.get("endpoint_examples", {}), "producer_suppression": summary.get("producer_suppression", {}), + "file_outcome_examples": summary.get("file_outcome_examples", []), "notes": [ "Diagnostics are read-only.", "A normal graph.json is already post-build and cannot recover raw producer edges.", @@ -508,6 +565,28 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"{summary.get('canonical_distinct_candidate_edges', 0)}" ), f"exact_duplicate_occurrences: {summary.get('exact_duplicate_occurrences', 0)}", + ( + "legitimate_repeated_source_occurrences: " + f"{summary.get('legitimate_repeated_source_occurrences', 0)}" + ), + ( + "suppressed_producer_duplicate_occurrences: " + f"{summary.get('suppressed_producer_duplicate_occurrences', 0)}" + ), + f"unlocated_duplicate_occurrences: {summary.get('unlocated_duplicate_occurrences', 0)}", + ( + "post_normalization_exact_duplicate_edges: " + f"{summary.get('post_normalization_exact_duplicate_edges', 0)}" + ), + ( + "post_normalization_unclassified_duplicates: " + f"{summary.get('post_normalization_unclassified_duplicates', 0)}" + ), + f"zero_node_files: {summary.get('zero_node_files', 0)}", + f"intentionally_skipped_files: {summary.get('intentionally_skipped_files', 0)}", + f"failed_extraction_files: {summary.get('failed_extraction_files', 0)}", + f"unexpected_empty_files: {summary.get('unexpected_empty_files', 0)}", + f"unsupported_extraction_files: {summary.get('unsupported_extraction_files', 0)}", ( "distinct_parallel_edge_instances: " f"{summary.get('distinct_parallel_edge_instances', 0)}" diff --git a/graphify/export.py b/graphify/export.py index e7ca53f97..172d0d42b 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -15,7 +15,7 @@ from networkx.readwrite import json_graph from graphify.security import sanitize_label from graphify.analyze import _node_community_map -from graphify.build import edge_data, edge_datas +from graphify.build import canonical_edge_key, edge_data, edge_datas from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 @@ -392,17 +392,30 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: ) lines.append(f"MERGE (n:{ftype} {{id: '{node_id_esc}', label: '{label}'}});") lines.append("") - for u, v, data in G.edges(data=True): + graph_edges = ( + G.edges(keys=True, data=True) + if G.is_multigraph() + else ((u, v, canonical_edge_key(u, v, data), data) for u, v, data in G.edges(data=True)) + ) + for u, v, edge_key, data in graph_edges: rel = _cypher_label( (data.get("relation", "RELATES_TO") or "RELATES_TO").upper(), "RELATES_TO", ) + edge_key_esc = _cypher_escape(str(edge_key)) conf = _cypher_escape(data.get("confidence", "EXTRACTED")) + occurrence_count = int(data.get("occurrence_count", 1) or 1) + occurrences_json = _cypher_escape( + json.dumps(data.get("occurrences", []), ensure_ascii=False, sort_keys=True) + ) u_esc = _cypher_escape(u) v_esc = _cypher_escape(v) lines.append( f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " - f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" + f"MERGE (a)-[r:{rel} {{graphify_key: '{edge_key_esc}'}}]->(b) " + f"SET r.confidence = '{conf}', " + f"r.occurrence_count = {occurrence_count}, " + f"r.occurrences_json = '{occurrences_json}';" ) with open(output_path, "w", encoding="utf-8") as f: # nosec f.write("\n".join(lines)) @@ -1025,9 +1038,15 @@ def _graphml_safe(val): for node_id in H.nodes(): for key, val in list(H.nodes[node_id].items()): H.nodes[node_id][key] = _graphml_safe(val) - for u, v in H.edges(): - for key, val in list(H.edges[u, v].items()): - H.edges[u, v][key] = _graphml_safe(val) + if H.is_multigraph(): + edge_records = H.edges(keys=True, data=True) + for _, _, _, attrs in edge_records: + for key, val in list(attrs.items()): + attrs[key] = _graphml_safe(val) + else: + for _, _, attrs in H.edges(data=True): + for key, val in list(attrs.items()): + attrs[key] = _graphml_safe(val) # Write atomically: a mid-serialization error otherwise leaves a 0-byte # .graphml on disk that downstream tooling mistakes for a completed export diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..cbed48c81 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -1,7 +1,9 @@ """graphdb — moved verbatim from graphify/export.py.""" from __future__ import annotations +import json from graphify.analyze import _node_community_map +from graphify.build import canonical_edge_key import networkx as nx import re @@ -59,17 +61,27 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + graph_edges = ( + G.edges(keys=True, data=True) + if G.is_multigraph() + else ((u, v, canonical_edge_key(u, v, data), data) for u, v, data in G.edges(data=True)) + ) + for u, v, edge_key, data in graph_edges: rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") } + if isinstance(data.get("occurrences"), list): + props["occurrences_json"] = json.dumps( + data["occurrences"], ensure_ascii=False, sort_keys=True + ) session.run( f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", + f"MERGE (a)-[r:{rel} {{graphify_key: $edge_key}}]->(b) SET r += $props", src=u, tgt=v, + edge_key=str(edge_key), props=props, ) edges_pushed += 1 @@ -157,16 +169,25 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + graph_edges = ( + G.edges(keys=True, data=True) + if G.is_multigraph() + else ((u, v, canonical_edge_key(u, v, data), data) for u, v, data in G.edges(data=True)) + ) + for u, v, edge_key, data in graph_edges: rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") } + if isinstance(data.get("occurrences"), list): + props["occurrences_json"] = json.dumps( + data["occurrences"], ensure_ascii=False, sort_keys=True + ) graph.query( f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - {"src": u, "tgt": v, "props": props}, + f"MERGE (a)-[r:{rel} {{graphify_key: $edge_key}}]->(b) SET r += $props", + {"src": u, "tgt": v, "edge_key": str(edge_key), "props": props}, ) edges_pushed += 1 diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index f96f7e173..0957d875e 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -511,6 +511,22 @@ def to_html( details.append(f"source: {source_file}:{source_location}") if occurrences != 1: details.append(f"occurrences: {occurrences}") + occurrence_items = data.get("occurrences", []) + if isinstance(occurrence_items, list) and occurrence_items: + evidence = [] + for occurrence in occurrence_items: + if not isinstance(occurrence, dict): + continue + item = str( + occurrence.get("source_span") + or occurrence.get("source_location") + or "unlocated" + ) + if occurrence.get("parameter"): + item += f" ({occurrence['parameter']})" + evidence.append(item) + if evidence: + details.append("evidence: " + ", ".join(evidence)) edge_payload = { "from": true_src, "to": true_tgt, diff --git a/graphify/extract.py b/graphify/extract.py index ea31fb64f..d1c0df0b6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2243,6 +2243,123 @@ def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: nodes[:] = [node for node in nodes if node.get("id") not in drop_ids] +def _normalize_edge_occurrences(edges: list[dict]) -> dict[str, int]: + """Merge source evidence after every resolver has canonicalized endpoints. + + Distinct source tokens on the same line are legitimate occurrences. The + same token emitted by two resolver paths is a producer duplicate and is + suppressed. Edges without token spans stay auditable through synthetic + unlocated occurrences rather than being silently discarded. + """ + grouped: dict[str, dict] = {} + seen_occurrences: dict[str, set[str]] = {} + suppressed = 0 + unlocated = 0 + + for ordinal, original in enumerate(edges): + edge = dict(original) + explicit = edge.pop("occurrences", None) + span = edge.pop("source_span", None) + edge.pop("occurrence_count", None) + identity = json.dumps( + edge, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str + ) + is_existing_identity = identity in grouped + if not is_existing_identity: + grouped[identity] = edge + grouped[identity]["occurrences"] = [] + seen_occurrences[identity] = set() + + occurrences = explicit if isinstance(explicit, list) else None + if not occurrences: + occurrence = { + "source_location": edge.get("source_location"), + "kind": edge.get("context") or edge.get("relation") or "relation", + } + if span: + occurrence["source_span"] = span + else: + occurrence["unlocated"] = True + occurrence["_ordinal"] = ordinal + occurrences = [occurrence] + + target = grouped[identity]["occurrences"] + fingerprints = seen_occurrences[identity] + for raw_occurrence in occurrences: + occurrence = dict(raw_occurrence) + if occurrence.get("unlocated") and any( + existing.get("source_location") == occurrence.get("source_location") + and existing.get("kind") == occurrence.get("kind") + for existing in target + ): + suppressed += 1 + continue + if occurrence.get("unlocated") and target: + unlocated += 1 + fingerprint_payload = { + key: value for key, value in occurrence.items() if key != "_ordinal" + } + fingerprint = json.dumps( + fingerprint_payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ) + if not occurrence.get("unlocated") and fingerprint in fingerprints: + suppressed += 1 + continue + fingerprints.add(fingerprint) + target.append(occurrence) + + normalized: list[dict] = [] + for edge in grouped.values(): + occurrences = edge["occurrences"] + occurrences.sort( + key=lambda item: ( + str(item.get("source_span") or ""), + str(item.get("parameter") or ""), + str(item.get("kind") or ""), + int(item.get("_ordinal", 0)), + ) + ) + for occurrence in occurrences: + occurrence.pop("_ordinal", None) + edge["occurrence_count"] = len(occurrences) + normalized.append(edge) + repeated_groups: Counter[str] = Counter() + for edge in normalized: + base = { + key: value for key, value in edge.items() + if key not in { + "source_location", "occurrences", "occurrence_count", "_src", "_tgt" + } + } + for occurrence in edge.get("occurrences", []): + base["occurrence_owner_location"] = ( + occurrence.get("owner_location") + or occurrence.get("source_location") + or edge.get("source_location") + ) + coarse_key = json.dumps( + base, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ) + repeated_groups[coarse_key] += 1 + legitimate = sum(max(0, count - 1) for count in repeated_groups.values()) + edges[:] = normalized + return { + "legitimate_repeated_source_occurrences": legitimate, + "suppressed_producer_duplicate_occurrences": suppressed, + "unlocated_duplicate_occurrences": unlocated, + "post_normalization_exact_duplicate_edges": 0, + "post_normalization_unclassified_duplicates": 0, + } + + def _augment_js_reexport_edges( paths: list[Path], nodes: list[dict], @@ -2433,6 +2550,7 @@ def _key(label: str) -> str: "confidence_score": 1.0 if type_qualified else 0.8, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -2541,6 +2659,7 @@ def _emit_call(caller: str, target_nid: "str | None", rc: dict) -> None: "confidence_score": 1.0, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -2668,6 +2787,7 @@ def _key(label: str) -> str: "confidence_score": 1.0, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -2796,6 +2916,7 @@ def _key(label: str) -> str: "confidence_score": 1.0 if type_qualified else 0.8, "source_file": src_file, "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -2910,6 +3031,7 @@ def _key(label: str) -> str: "confidence_score": 1.0 if type_qualified else 0.8, "source_file": src_file, "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -3109,6 +3231,7 @@ def _key(label: str) -> str: "confidence_score": 1.0 if type_qualified else 0.8, "source_file": src_file, "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -4487,7 +4610,11 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: # (e.g. a transient batch/parallel hiccup). Caching it makes the empty # byte-stable across runs and silently blinds affected/explain to and # through the file (#1666); skipping the write lets a rerun self-heal. - if not bypass_cache and "error" not in result and result.get("nodes"): + if ( + not bypass_cache + and "error" not in result + and (result.get("nodes") or result.get("status") == "skipped_intentional") + ): save_cached(path, result, root, cache_root=cache_location) return idx, result @@ -4617,13 +4744,22 @@ def _extract_sequential( ) extractor = _get_extractor(path) if extractor is None: - per_file[idx] = {"nodes": [], "edges": []} + per_file[idx] = { + "nodes": [], + "edges": [], + "status": "unsupported", + "reason": "no structural extractor", + } continue bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES # XAML boundary anchors on `root` (the corpus), not the cache location. result = _safe_extract_with_xaml_root(extractor, path, root) # See _extract_single_file: don't cache an anomalous zero-node result (#1666). - if not bypass_cache and "error" not in result and result.get("nodes"): + if ( + not bypass_cache + and "error" not in result + and (result.get("nodes") or result.get("status") == "skipped_intentional") + ): save_cached(path, result, root, cache_root=cache_location) per_file[idx] = result if total_files >= _PROGRESS_INTERVAL: @@ -4740,13 +4876,38 @@ def extract( if per_file[i] is None: per_file[i] = {"nodes": [], "edges": []} - # #1666: surface any source file an extractor accepted but that produced zero + file_outcomes: list[dict[str, str]] = [] + for i, path in enumerate(paths): + result = per_file[i] or {} + if result.get("nodes"): + continue + status = result.get("status") + if status not in { + "skipped_intentional", "failed", "unexpected_empty", "unsupported" + }: + status = "failed" if result.get("error") else "unexpected_empty" + file_outcomes.append({ + "source_file": str(path), + "status": status, + "reason": str( + result.get("reason") + or result.get("error") + or "extractor returned no nodes" + ), + }) + + # #1666: surface only anomalous empty results. Deliberate data/config skips + # are classified and cached, so they neither warn nor retry forever. # nodes (not even a file node). Such a file is silently absent from the graph, # so affected/explain are blind to and through it with no other signal. _empty_sources: list[str] = [] for i, _p in enumerate(paths): _res = per_file[i] or {} - if _res.get("nodes") or _res.get("error"): + if ( + _res.get("nodes") + or _res.get("error") + or _res.get("status") in {"skipped_intentional", "unsupported"} + ): continue if _get_extractor(_p) is not None: _empty_sources.append(str(_p)) @@ -4761,6 +4922,18 @@ def extract( file=sys.stderr, flush=True, ) + intentional_skips = [ + outcome for outcome in file_outcomes + if outcome["status"] == "skipped_intentional" + ] + if intentional_skips: + print( + f" info: {len(intentional_skips)} source file(s) intentionally skipped " + "by structural extraction policy (no graph loss warning).", + file=sys.stderr, + flush=True, + ) + # #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST # extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently # contributes zero nodes. The #1666 warning above deliberately skips these (it @@ -5420,6 +5593,7 @@ def _has_import_evidence(candidate_id: str) -> bool: "confidence_score": 0.8, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) continue @@ -5456,6 +5630,7 @@ def _has_import_evidence(candidate_id: str) -> bool: "confidence_score": confidence_score, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), + "source_span": rc.get("source_span"), "weight": 1.0, }) @@ -5545,6 +5720,8 @@ def _portable_out_of_root_sf(p: Path) -> str: e.pop("_import_level", None) e.pop("_imported_names", None) + occurrence_diagnostics = _normalize_edge_occurrences(all_edges) + # Tag AST provenance so the incremental watch rebuild can distinguish # AST-extracted nodes from semantic/LLM nodes. On a full re-extraction # the watcher drops any AST-marked node missing from the fresh output @@ -5557,9 +5734,18 @@ def _portable_out_of_root_sf(p: Path) -> str: for e in all_edges: e["_origin"] = "ast" + for outcome in file_outcomes: + outcome_path = Path(outcome["source_file"]) + try: + outcome["source_file"] = outcome_path.resolve().relative_to(root).as_posix() + except (ValueError, OSError, RuntimeError): + outcome["source_file"] = outcome_path.name + return { "nodes": all_nodes, "edges": all_edges, + "extraction_diagnostics": occurrence_diagnostics, + "file_outcomes": file_outcomes, "input_tokens": 0, "output_tokens": 0, } diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index 361aa5815..ef7608301 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -65,7 +65,29 @@ def _semantic_reference_edge( "NonCallableMagicMock", "PropertyMock", "patch", "sentinel", }) -def _python_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: +def _source_occurrence(node, *, kind: str, parameter: str | None = None) -> dict: + """Portable source evidence for one AST token/expression.""" + occurrence = { + "source_location": f"L{node.start_point[0] + 1}", + "source_span": ( + f"L{node.start_point[0] + 1}:C{node.start_point[1] + 1}-" + f"L{node.end_point[0] + 1}:C{node.end_point[1] + 1}" + ), + "kind": kind, + } + if parameter: + occurrence["parameter"] = parameter + return occurrence + + +def _python_collect_type_refs( + node, + source: bytes, + generic: bool, + out: list[dict], + *, + parameter: str | None = None, +) -> None: """Walk a Python type annotation; append (name, role) where role is 'type' or 'generic_arg'. Builtin/typing containers (list, dict, Optional, Union, …) are not emitted as refs themselves, @@ -77,42 +99,62 @@ def _python_collect_type_refs(node, source: bytes, generic: bool, out: list[tupl if t == "type": for c in node.children: if c.is_named: - _python_collect_type_refs(c, source, generic, out) + _python_collect_type_refs(c, source, generic, out, parameter=parameter) return if t == "identifier": name = _read_text(node, source) if name and name not in _PYTHON_TYPE_CONTAINERS and name not in _PYTHON_ANNOTATION_NOISE: - out.append((name, "generic_arg" if generic else "type")) + out.append({ + "name": name, + "role": "generic_arg" if generic else "type", + "occurrence": _source_occurrence( + node, kind="generic_arg" if generic else "type", parameter=parameter + ), + }) return if t == "attribute": tail = _read_text(node, source).rsplit(".", 1)[-1] if tail and tail not in _PYTHON_TYPE_CONTAINERS and tail not in _PYTHON_ANNOTATION_NOISE: - out.append((tail, "generic_arg" if generic else "type")) + out.append({ + "name": tail, + "role": "generic_arg" if generic else "type", + "occurrence": _source_occurrence( + node, kind="generic_arg" if generic else "type", parameter=parameter + ), + }) return if t == "generic_type": for c in node.children: if c.type == "identifier": container = _read_text(c, source) if container and container not in _PYTHON_TYPE_CONTAINERS and container not in _PYTHON_ANNOTATION_NOISE: - out.append((container, "generic_arg" if generic else "type")) + out.append({ + "name": container, + "role": "generic_arg" if generic else "type", + "occurrence": _source_occurrence( + c, kind="generic_arg" if generic else "type", parameter=parameter + ), + }) elif c.type == "type_parameter": for sub in c.children: if sub.is_named: - _python_collect_type_refs(sub, source, True, out) + _python_collect_type_refs( + sub, source, True, out, parameter=parameter + ) return if t == "subscript": value = node.child_by_field_name("value") if value is not None: - _python_collect_type_refs(value, source, generic, out) + _python_collect_type_refs(value, source, generic, out, parameter=parameter) for c in node.children: if c is value or not c.is_named: continue - _python_collect_type_refs(c, source, True, out) + _python_collect_type_refs(c, source, True, out, parameter=parameter) return if node.is_named: for c in node.children: if c.is_named: - _python_collect_type_refs(c, source, generic, out) + _python_collect_type_refs(c, source, generic, out, parameter=parameter) def _csharp_pre_scan_interfaces(root_node, source: bytes) -> set[str]: """Return names declared as `interface` in this C# compilation unit.""" @@ -974,15 +1016,23 @@ def _scala_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple if c.is_named: _scala_collect_type_refs(c, source, generic, out) -def _python_collect_param_refs(params_node, source: bytes) -> list[tuple[str, str]]: +def _python_collect_param_refs(params_node, source: bytes) -> list[dict]: """Collect type refs from each typed parameter under a `parameters` node.""" - out: list[tuple[str, str]] = [] + out: list[dict] = [] if params_node is None: return out for child in params_node.children: if child.type in ("typed_parameter", "typed_default_parameter"): type_node = child.child_by_field_name("type") - _python_collect_type_refs(type_node, source, False, out) + name_node = child.child_by_field_name("name") + if name_node is None: + name_node = next( + (c for c in child.children if c.type == "identifier"), None + ) + parameter = _read_text(name_node, source) if name_node is not None else None + _python_collect_type_refs( + type_node, source, False, out, parameter=parameter + ) return out def _python_param_names(params_node, source: bytes) -> set[str]: @@ -3182,24 +3232,38 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: if config.ts_module == "tree_sitter_python": params_node = node.child_by_field_name("parameters") - for ref_name, role in _python_collect_param_refs(params_node, source): + for ref in _python_collect_param_refs(params_node, source): + ref_name, role = ref["name"], ref["role"] + occurrence = dict(ref["occurrence"]) + occurrence["owner_location"] = f"L{line}" ctx = "generic_arg" if role == "generic_arg" else "parameter_type" - target_nid = ensure_named_node(ref_name, line) + ref_line = occurrence["source_location"] + target_nid = ensure_named_node(ref_name, int(ref_line[1:])) if target_nid != func_nid: - edges.append( - _semantic_reference_edge(func_nid, target_nid, ctx, str_path, line) + edge = _semantic_reference_edge( + func_nid, target_nid, ctx, str_path, ref_line ) + edge["occurrences"] = [occurrence] + edge["occurrence_count"] = 1 + edges.append(edge) return_type_node = node.child_by_field_name("return_type") if return_type_node is not None: - return_refs: list[tuple[str, str]] = [] + return_refs: list[dict] = [] _python_collect_type_refs(return_type_node, source, False, return_refs) - for ref_name, role in return_refs: + for ref in return_refs: + ref_name, role = ref["name"], ref["role"] + occurrence = dict(ref["occurrence"]) + occurrence["owner_location"] = f"L{line}" ctx = "generic_arg" if role == "generic_arg" else "return_type" - target_nid = ensure_named_node(ref_name, line) + ref_line = occurrence["source_location"] + target_nid = ensure_named_node(ref_name, int(ref_line[1:])) if target_nid != func_nid: - edges.append( - _semantic_reference_edge(func_nid, target_nid, ctx, str_path, line) + edge = _semantic_reference_edge( + func_nid, target_nid, ctx, str_path, ref_line ) + edge["occurrences"] = [occurrence] + edge["occurrence_count"] = 1 + edges.append(edge) if config.ts_module == "tree_sitter_c_sharp": csharp_type_params = _csharp_type_parameters_in_scope(node, source) @@ -4070,6 +4134,10 @@ def walk_calls( "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", + "occurrences": [ + _source_occurrence(node, kind="call") + ], + "occurrence_count": 1, "weight": 1.0, }) elif callee_name and not tgt_nid: @@ -4080,6 +4148,9 @@ def walk_calls( "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", + "source_span": _source_occurrence( + node, kind="call" + )["source_span"], "receiver": swift_receiver or member_receiver, } # Ruby: attach the receiver's inferred type from the method's diff --git a/graphify/extractors/json_config.py b/graphify/extractors/json_config.py index 6a9b641a9..e71aa0b67 100644 --- a/graphify/extractors/json_config.py +++ b/graphify/extractors/json_config.py @@ -21,6 +21,20 @@ "extends", "$ref", "$schema", "compilerOptions", }) + +def _is_config_json_name(path: Path) -> bool: + name = path.name.casefold() + return name in _CONFIG_JSON_NAMES or name.endswith( + ( + ".eslintrc.json", + ".prettierrc.json", + ".babelrc.json", + "tsconfig.json", + "jsconfig.json", + ) + ) + + def _is_config_json(path: Path, obj_node, source: bytes) -> bool: """True if a .json file is a recognized config/manifest worth AST-extracting. @@ -28,12 +42,7 @@ def _is_config_json(path: Path, obj_node, source: bytes) -> bool: so arbitrarily-named config files (e.g. ``api.tsconfig.json``, ``foo.eslintrc.json``) are still picked up. Returns False for data JSON so it is skipped by the structural pass (#1224).""" - name = path.name.casefold() - if name in _CONFIG_JSON_NAMES: - return True - # Common compound config names: *.eslintrc.json, *.prettierrc.json, etc. - if name.endswith((".eslintrc.json", ".prettierrc.json", ".babelrc.json", - "tsconfig.json", "jsconfig.json")): + if _is_config_json_name(path): return True # Top-level key probe: scan the root object's immediate keys (no deep walk). for top_key in obj_node.children: @@ -71,7 +80,20 @@ def extract_json(path: Path) -> dict: with path.open("rb") as _f: source = _f.read(_JSON_MAX_BYTES + 1) if len(source) > _JSON_MAX_BYTES: - return {"nodes": [], "edges": [], "error": "json file too large to index"} + if not _is_config_json_name(path): + return { + "nodes": [], + "edges": [], + "status": "skipped_intentional", + "skipped": "data json outside structural indexing scope", + "reason": "data json outside structural indexing scope", + } + return { + "nodes": [], + "edges": [], + "status": "failed", + "error": "recognized config json file too large to index", + } language = Language(tsjson.language()) parser = Parser(language) tree = parser.parse(source) @@ -207,10 +229,22 @@ def walk_object(obj_node, parent_nid: str, parent_key: str | None, # datasets, GeoJSON, API dumps) is skipped so it doesn't explode into # orphan key-nodes (#1224); it's left to the LLM semantic pass. if not _is_config_json(path, doc, source): - return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"} + return { + "nodes": [], + "edges": [], + "status": "skipped_intentional", + "skipped": "data json outside structural indexing scope", + "reason": "data json outside structural indexing scope", + } walk_object(doc, file_nid, None, 0, [0]) else: # Top-level array or scalar => data JSON, never a config/manifest. - return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"} + return { + "nodes": [], + "edges": [], + "status": "skipped_intentional", + "skipped": "data json outside structural indexing scope", + "reason": "data json outside structural indexing scope", + } return {"nodes": nodes, "edges": edges} diff --git a/graphify/serve.py b/graphify/serve.py index 1e4ab4f6a..f2878c3a3 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -910,11 +910,29 @@ def _adj(n): occurrence_suffix = ( f" occurrences={occurrence_count}" if occurrence_count != 1 else "" ) + occurrence_items = d.get("occurrences", []) + evidence_parts = [] + if isinstance(occurrence_items, list): + for occurrence in occurrence_items[:3]: + if not isinstance(occurrence, dict): + continue + evidence = str( + occurrence.get("source_span") + or occurrence.get("source_location") + or "unlocated" + ) + if occurrence.get("parameter"): + evidence += f"({occurrence['parameter']})" + evidence_parts.append(evidence) + evidence_suffix = ( + f" evidence={sanitize_label(','.join(evidence_parts))}" + if evidence_parts else "" + ) line = ( f"EDGE {sanitize_label(G.nodes[src].get('label', src))} " f"--{sanitize_label(str(d.get('relation', '')))} " f"[{sanitize_label(str(d.get('confidence', '')))}" - f"{context_suffix}{occurrence_suffix}]--> " + f"{context_suffix}{occurrence_suffix}{evidence_suffix}]--> " f"{sanitize_label(G.nodes[tgt].get('label', tgt))}{at_suffix}" ) lines.append(line) diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 7352492e4..1ba6422a1 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 7be15f626..e6761a79c 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 7352492e4..1ba6422a1 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 21742630f..cf26dc02e 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index bdc603b2c..244e507d4 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 21742630f..cf26dc02e 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index 22e87e13e..4df2eaead 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -439,6 +439,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index d22d36045..16f81cf72 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 2ff196fd4..2a2a1eed8 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 21742630f..cf26dc02e 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 7ea78e4d4..2b2f45ad2 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -370,6 +370,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -463,6 +465,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -471,7 +477,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 21742630f..cf26dc02e 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index ab5d1928c..def4e65a8 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -376,6 +376,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -469,6 +471,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -477,7 +483,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index d87691eae..de4043980 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -374,6 +374,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -467,6 +469,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -475,7 +481,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index e638a685e..27b0337a4 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -400,6 +400,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -493,6 +495,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -501,7 +507,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/graphify/skill.md b/graphify/skill.md index 21742630f..cf26dc02e 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tests/test_edge_occurrences.py b/tests/test_edge_occurrences.py new file mode 100644 index 000000000..949bf0064 --- /dev/null +++ b/tests/test_edge_occurrences.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import _normalize_edge_occurrences, extract +from graphify.build import build_from_json, canonical_edge_key, edge_datas +from graphify.diagnostics import diagnose_extraction + + +def test_python_type_occurrences_use_real_spans_and_parameters(tmp_path: Path): + source = tmp_path / "sample.py" + source.write_text( + "class Thing: pass\n" + "def same(a: Thing, b: Thing, c: list[Thing]): pass\n" + "def split(\n" + " a: Thing,\n" + " b: Thing,\n" + "): pass\n" + ) + + result = extract([source], cache_root=tmp_path / "cache", parallel=False) + refs = [ + edge for edge in result["edges"] + if edge.get("relation") == "references" + and edge.get("target", "").endswith("thing") + ] + same = [edge for edge in refs if edge["source"].endswith("_same")] + split = [edge for edge in refs if edge["source"].endswith("_split")] + + parameter_edge = next(edge for edge in same if edge["context"] == "parameter_type") + assert parameter_edge["source_location"] == "L2" + assert parameter_edge["occurrence_count"] == 2 + assert {item["parameter"] for item in parameter_edge["occurrences"]} == {"a", "b"} + assert all(":C" in item["source_span"] for item in parameter_edge["occurrences"]) + assert {edge["source_location"] for edge in split} == {"L4", "L5"} + + +def test_post_resolver_normalization_suppresses_same_span_only(): + base = { + "source": "a", + "target": "b", + "relation": "calls", + "context": "call", + "confidence": "EXTRACTED", + "source_file": "a.py", + "source_location": "L10", + "source_span": "L10:C5-L10:C8", + } + edges = [dict(base), dict(base)] + diagnostics = _normalize_edge_occurrences(edges) + + assert len(edges) == 1 + assert edges[0]["occurrence_count"] == 1 + assert diagnostics["suppressed_producer_duplicate_occurrences"] == 1 + assert diagnostics["post_normalization_unclassified_duplicates"] == 0 + + +def test_occurrence_evidence_does_not_change_stable_edge_key(): + attrs = { + "relation": "references", + "context": "parameter_type", + "source_file": "a.py", + "source_location": "L2", + } + first = { + **attrs, + "occurrences": [{"source_span": "L2:C10-L2:C15", "parameter": "a"}], + "occurrence_count": 1, + } + second = { + **attrs, + "occurrences": [{"source_span": "L2:C20-L2:C25", "parameter": "b"}], + "occurrence_count": 1, + } + assert canonical_edge_key("a", "b", first) == canonical_edge_key("a", "b", second) + + graph = build_from_json( + { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [ + {"source": "a", "target": "b", **first}, + {"source": "a", "target": "b", **second}, + ], + }, + multigraph=True, + ) + edge = edge_datas(graph, "a", "b")[0] + assert edge["occurrence_count"] == 2 + assert len(edge["occurrences"]) == 2 + + +def test_diagnostic_classifies_occurrences_and_zero_node_outcomes(): + extraction = { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [{ + "source": "a", + "target": "b", + "relation": "calls", + "occurrence_count": 2, + "occurrences": [ + {"source_span": "L1:C1-L1:C2"}, + {"source_span": "L1:C4-L1:C5"}, + ], + }], + "extraction_diagnostics": { + "legitimate_repeated_source_occurrences": 1, + "suppressed_producer_duplicate_occurrences": 1, + "unlocated_duplicate_occurrences": 0, + "post_normalization_exact_duplicate_edges": 0, + "post_normalization_unclassified_duplicates": 0, + }, + "file_outcomes": [ + {"source_file": "data.json", "status": "skipped_intentional", "reason": "data"}, + {"source_file": "bad.py", "status": "failed", "reason": "parse"}, + ], + } + summary = diagnose_extraction(extraction, multigraph=True) + assert summary["exact_duplicate_occurrences"] == 2 + assert summary["legitimate_repeated_source_occurrences"] == 1 + assert summary["suppressed_producer_duplicate_occurrences"] == 1 + assert summary["intentionally_skipped_files"] == 1 + assert summary["failed_extraction_files"] == 1 diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 79ce9189b..4e5910058 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -167,6 +167,12 @@ def test_explain_lists_every_parallel_edge(monkeypatch, tmp_path, capsys): "relation": "references", "confidence": "EXTRACTED", "context": "parameter_type", "source_file": "stage_track_paths.py", "source_location": "L982", "occurrence_count": 6, + "occurrences": [ + { + "source_span": "L982:C12-L982:C16", + "parameter": "path", + } + ], }, { "source": "stage", "target": "path", "key": "calls", @@ -183,6 +189,7 @@ def test_explain_lists_every_parallel_edge(monkeypatch, tmp_path, capsys): assert "[references]" in out and "context=parameter_type x6" in out assert "stage_track_paths.py:L982" in out + assert "L982:C12-L982:C16(path)" in out assert "[calls]" in out and "context=call" in out assert "stage_track_paths.py:L1002" in out diff --git a/tests/test_export.py b/tests/test_export.py index 28a4707a7..1aa447379 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -55,6 +55,20 @@ def test_to_cypher_contains_merge_statements(): content = out.read_text() assert "MERGE" in content +def test_to_cypher_preserves_parallel_edge_identity_and_occurrences(tmp_path): + import networkx as nx + G = nx.MultiDiGraph() + G.add_edge("a", "b", key="first", relation="calls", occurrence_count=2, + occurrences=[{"source_span": "L1:C1-L1:C2"}]) + G.add_edge("a", "b", key="second", relation="calls", occurrence_count=1, + occurrences=[{"source_span": "L2:C1-L2:C2"}]) + out = tmp_path / "cypher.txt" + to_cypher(G, str(out)) + content = out.read_text() + assert "graphify_key: 'first'" in content + assert "graphify_key: 'second'" in content + assert "occurrences_json" in content + def test_to_graphml_creates_file(): G = make_graph() communities = cluster(G) @@ -141,6 +155,37 @@ def test_to_graphml_preserves_native_scalar_types(): assert H.nodes["a"]["name"] == "x" +def test_to_graphml_preserves_multigraph_occurrence_evidence(tmp_path): + import networkx as nx + G = nx.MultiDiGraph() + G.add_edge( + "a", + "b", + key="first", + relation="calls", + occurrences=[{"source_span": "L1:C1-L1:C2"}], + ) + G.add_edge( + "a", + "b", + key="second", + relation="references", + occurrences=[{"source_span": "L2:C1-L2:C2"}], + ) + out = tmp_path / "graph.graphml" + to_graphml(G, {0: ["a", "b"]}, str(out)) + H = nx.read_graphml(str(out), force_multigraph=True) + assert H.number_of_edges() == 2 + evidence = [ + json.loads(attrs["occurrences"]) + for _, _, _, attrs in H.edges(keys=True, data=True) + ] + assert {item[0]["source_span"] for item in evidence} == { + "L1:C1-L1:C2", + "L2:C1-L2:C2", + } + + def test_to_html_creates_file(): G = make_graph() communities = cluster(G) diff --git a/tests/test_extract.py b/tests/test_extract.py index be633c0fd..4aceb708a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2151,7 +2151,7 @@ def test_extract_json_large_file_skipped(tmp_path): # Write a JSON file just over 1 MiB big.write_bytes(b'{"x": "' + b"a" * (1_048_576) + b'"}') result = extract_json(big) - assert "error" in result + assert result["status"] == "skipped_intentional" assert result["nodes"] == [] @@ -2186,6 +2186,15 @@ def test_extract_json_data_file_skipped(tmp_path): assert result["nodes"] == [] assert result["edges"] == [] assert "skipped" in result + assert result["status"] == "skipped_intentional" + + +def test_extract_recognized_large_config_json_fails(tmp_path): + config = tmp_path / "tsconfig.json" + config.write_bytes(b'{"compilerOptions": {"x": "' + b"a" * 1_048_576 + b'"}}') + result = extract_json(config) + assert result["status"] == "failed" + assert "error" in result def test_extract_json_top_level_array_skipped(tmp_path): diff --git a/tests/test_path_cli.py b/tests/test_path_cli.py index 63b4fa58d..3adecd53b 100644 --- a/tests/test_path_cli.py +++ b/tests/test_path_cli.py @@ -197,6 +197,9 @@ def test_path_shows_parallel_relation_context_location_and_occurrences( "relation": "references", "confidence": "EXTRACTED", "context": "parameter_type", "source_file": "a.py", "source_location": "L10", "occurrence_count": 6, + "occurrences": [ + {"source_span": "L10:C5-L10:C9", "parameter": "value"}, + ], }, ], } @@ -207,6 +210,7 @@ def test_path_shows_parallel_relation_context_location_and_occurrences( assert "calls:call@a.py:L20" in out assert "references:parameter_type@a.py:L10x6" in out + assert "L10:C5-L10:C9(value)" in out def test_path_relation_fallback_related_when_missing(monkeypatch, tmp_path, capsys): diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index 8366b2995..8dd3fdb9a 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -121,6 +121,10 @@ def test_query_cli_traverses_directed_graph_both_ways_and_lists_parallel_edges( "caller", "callee", key="references", relation="references", confidence="EXTRACTED", context="parameter_type", source_file="a.py", source_location="L10", occurrence_count=2, + occurrences=[ + {"source_span": "L10:C5-L10:C9", "parameter": "first"}, + {"source_span": "L10:C12-L10:C16", "parameter": "second"}, + ], ) graph_path = tmp_path / "graph.json" graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links"))) @@ -137,6 +141,7 @@ def test_query_cli_traverses_directed_graph_both_ways_and_lists_parallel_edges( assert "caller_fn --calls" in out assert "caller_fn --references" in out assert "occurrences=2" in out + assert "L10:C5-L10:C9(first)" in out def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): diff --git a/tests/test_zero_node_no_cache.py b/tests/test_zero_node_no_cache.py index 5e62547ba..3001dc356 100644 --- a/tests/test_zero_node_no_cache.py +++ b/tests/test_zero_node_no_cache.py @@ -52,3 +52,15 @@ def test_no_warning_when_all_files_produce_nodes(tmp_path, capsys): ex.extract([f], cache_root=tmp_path / "out", parallel=False) err = capsys.readouterr().err assert "zero nodes" not in err + + +def test_intentional_json_skip_is_cached_and_not_warned(tmp_path, capsys): + f = tmp_path / "dataset.json" + f.write_text('{"records": [{"id": 1}]}') + result = ex.extract([f], cache_root=tmp_path / "out", parallel=False) + err = capsys.readouterr().err + + assert "zero nodes" not in err + assert result["file_outcomes"][0]["status"] == "skipped_intentional" + from graphify.cache import load_cached + assert load_cached(f, tmp_path / "out")["status"] == "skipped_intentional" diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 7352492e4..1ba6422a1 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 7be15f626..e6761a79c 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 7352492e4..1ba6422a1 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 21742630f..cf26dc02e 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index bdc603b2c..244e507d4 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 21742630f..cf26dc02e 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index 22e87e13e..4df2eaead 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -439,6 +439,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index d22d36045..16f81cf72 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -468,6 +470,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -476,7 +482,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 2ff196fd4..2a2a1eed8 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 21742630f..cf26dc02e 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 7ea78e4d4..2b2f45ad2 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -370,6 +370,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -463,6 +465,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -471,7 +477,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 21742630f..cf26dc02e 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index ab5d1928c..def4e65a8 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -376,6 +376,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -469,6 +471,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -477,7 +483,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index d87691eae..de4043980 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -374,6 +374,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -467,6 +469,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -475,7 +481,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index e638a685e..27b0337a4 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -400,6 +400,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -493,6 +495,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -501,7 +507,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 21742630f..cf26dc02e 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -378,6 +378,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -471,6 +473,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -479,7 +485,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 7be15f626..e6761a79c 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -375,6 +375,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 753a36bf6..f99f60bf6 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -313,6 +313,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } @@ -406,6 +408,10 @@ flags = [f'{summary[k]} {label}' for k, label in ( ('missing_endpoint_edges', 'missing-endpoint edges'), ('self_loop_edges', 'self-loop edges'), ('post_build_lost_distinct_edges', 'distinct relations lost after build'), + ('post_normalization_unclassified_duplicates', 'unclassified duplicate occurrences'), + ('failed_extraction_files', 'failed extraction files'), + ('unexpected_empty_files', 'unexpected empty files'), + ('unsupported_extraction_files', 'unsupported extraction files'), ) if summary.get(k, 0)] if not IS_MULTIGRAPH: flags.extend( @@ -414,7 +420,7 @@ if not IS_MULTIGRAPH: ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), ) if summary.get(k, 0) ) -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (external imports excluded; no unresolved/missing/collapsed edges).') +print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else f'Graph health: OK ({summary.get(\"intentionally_skipped_files\", 0)} intentional skips; no unresolved/missing/collapsed edges).') " ``` diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index 22e87e13e..4df2eaead 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -439,6 +439,8 @@ merged = { 'nodes': merged_nodes, 'edges': merged_edges, 'hyperedges': merged_hyperedges, + 'extraction_diagnostics': ast.get('extraction_diagnostics', {}), + 'file_outcomes': ast.get('file_outcomes', []), 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index e25098bca..b7ac47bf8 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -955,6 +955,13 @@ def _is_semantic_cache_scope_fix_line(line: str) -> bool: ) or stripped.startswith("saved = save_semantic_cache(") +def _is_extraction_outcome_line(line: str) -> bool: + stripped = line.strip() + return stripped.startswith( + ("'extraction_diagnostics': ast.get(", "'file_outcomes': ast.get(") + ) + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -975,6 +982,7 @@ def _is_semantic_cache_scope_fix_line(line: str) -> bool: _is_obsidian_usage_comment_line, _is_uv_from_interpreter_fix_line, _is_semantic_cache_scope_fix_line, + _is_extraction_outcome_line, ) From 5430a550cdd3b3c107f789fc13d9ace9923fcd33 Mon Sep 17 00:00:00 2001 From: Henrique Silveira Neves Date: Fri, 24 Jul 2026 22:58:41 -0300 Subject: [PATCH 5/5] fix: install tomli for Python 3.10 manifests --- pyproject.toml | 2 +- tests/test_manifest_ingest.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1c46f149d..a5eebc9ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "tree-sitter-fortran>=0.6,<0.8", "tree-sitter-bash>=0.23,<0.27", "tree-sitter-json>=0.23,<0.26", + "tomli>=2.0 ; python_version < '3.11'", ] [project.urls] @@ -102,7 +103,6 @@ dev = [ "ruff>=0.15.13", "setuptools>=82.0.1", "wheel>=0.47.0", - "tomli>=2.0 ; python_version < '3.11'", "tree-sitter-hcl>=1.2.0", ] diff --git a/tests/test_manifest_ingest.py b/tests/test_manifest_ingest.py index 2b97f5fb9..2cdee396f 100644 --- a/tests/test_manifest_ingest.py +++ b/tests/test_manifest_ingest.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +import tomllib from graphify.build import build_from_json from graphify.detect import FileType, classify_file @@ -11,6 +12,14 @@ ) +def test_tomli_is_a_runtime_dependency_for_python_310(): + pyproject = tomllib.loads( + (Path(__file__).resolve().parent.parent / "pyproject.toml").read_text(encoding="utf-8") + ) + dependencies = pyproject["project"]["dependencies"] + assert "tomli>=2.0 ; python_version < '3.11'" in dependencies + + def _write(p: Path, text: str) -> Path: p.parent.mkdir(parents=True, exist_ok=True) p.write_text(text, encoding="utf-8")