From e1db7e1920c72893782acf01860b4e742b4f724e Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:32:46 +0200 Subject: [PATCH] chore(index): remove the unused vector index, the ousast index subcommand and the [embeddings] section Legacy inventory 2, item R1. The vector half of index.py (VectorIndex, VectorRecord, RetrievalHit, RetrievalPackage, the EmbeddingClient protocol, index_reuse_key, build/write/read/query_vector_index, build_retrieval_package and their helpers) had no importer in src; `ousast index` wrote index/chunks.json that nothing read, and was the only reader of `[embeddings]`. index.py keeps CodeChunk and chunk_text_namespace, which skills.py imports; chunk_file/build_code_chunks go with their only caller. `[embeddings] model` and `[embeddings] store` join RETIRED_KEYS (RetiredConfigError naming the reason), the same way the [variants] keys were retired; OPENROUTER_EMBEDDING_MODEL is no longer read, so the README and .env.example drop it (`ousast learn memory embed --model` names the model). test_index.py now tests the chunker only; test_config.py covers the two retired keys and no longer asserts the removed field. The module audit record is regenerated (counts unchanged: 151 modules, 0 orphaned; the index module's importer list shrinks to skills). RELEASE_NOTES Unreleased carries the line. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .env.example | 3 +- README.md | 6 +- RELEASE_NOTES.md | 7 +- .../measurements/2026-09-08-module-audit.json | 3 +- src/openultrasast/cli.py | 33 --- src/openultrasast/config.py | 24 +- src/openultrasast/index.py | 215 +----------------- tests/test_config.py | 17 +- tests/test_index.py | 97 ++------ 9 files changed, 43 insertions(+), 362 deletions(-) diff --git a/.env.example b/.env.example index 118e634..6274c0e 100644 --- a/.env.example +++ b/.env.example @@ -6,9 +6,8 @@ DEEPSEEK_API_KEY= # DEEPSEEK_BASE_URL=https://api.deepseek.com -# Embeddings only: OpenAI text-embedding-3-small through OpenRouter. +# Embeddings only (`ousast learn memory embed`, default openai/text-embedding-3-small through OpenRouter). OPENROUTER_API_KEY= -OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # Plane memory store. Default is a local FileStore under ~/ousast-results/plane/memory. # For an S3-compatible server (RustFS is the tested one): OUSAST_MEMORY=s3://[/], or diff --git a/README.md b/README.md index 19d8d9c..3d6c74a 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,7 @@ Keys and store settings live in `.env` in the working directory (gitignored, nev ```bash DEEPSEEK_API_KEY=... # every LLM call: model layer, tool hunter, plane tasks, decision engine -OPENROUTER_API_KEY=... # embeddings only -OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small +OPENROUTER_API_KEY=... # embeddings only (`ousast learn memory embed`) # Plane memory store (maintainers). Unset: a local file store under ~/ousast-results/plane/memory. OUSAST_MEMORY=s3:// # or s3:///; needs the s3 extra @@ -276,6 +275,5 @@ uv run python dagger/ci.py # containerized CI pipeline ``` Maintainer commands not covered above: `ousast repos` (pinned known-vulnerable checkouts), -`ousast index` (chunk a repository for embedding-index construction), `ousast model candidates` -(what the candidate enumerator can reach) and `ousast learn` (the decision engine's data; see +`ousast model candidates` (what the candidate enumerator can reach) and `ousast learn` (the decision engine's data; see [docs/decision-engine.md](docs/decision-engine.md)). diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 338ef76..5a47886 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -8,8 +8,11 @@ within-pair 0.72-0.93, BLOCK offered nowhere, canary below 0.9 for three families) and that the engine is not adopted. The model layer's residual question is documented at `model/pipeline.py` (`model/judge.py` was deleted); a missing provider key is described as it behaves (no question asked, entailed findings reported), - not as a recorded degradation. The `.env` reference lists `OPENROUTER_EMBEDDING_MODEL` and the memory-store - variables; `ousast plane memory-normalise` and `ousast index` are listed. + not as a recorded degradation. The `.env` reference lists the memory-store variables; `ousast plane memory-normalise` is listed. +- **Removed: the unused vector index, `ousast index` and `[embeddings]`** (legacy cleanup, 2026-10-02). `index.py` + keeps only the text chunker the skill router uses; the `VectorIndex` half had no caller, and nothing read the + `chunks.json` the subcommand wrote. `[embeddings] model`/`store` fail to load with the reason, like the other + retired keys; `OPENROUTER_EMBEDDING_MODEL` is no longer read (`ousast learn memory embed --model` names the model). # v2.0.0 (2026-10-02) diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 56cfd90..8ce6a1d 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -338,7 +338,6 @@ "classification": "load_bearing", "reason": "reachable from gate, map_gate, pair_gate, cli", "importers": [ - "cli", "skills" ] }, @@ -530,6 +529,7 @@ "importers": [ "learn.compile", "learn.evaluate", + "learn.examples", "learn.features", "learn.leaks", "learn.program", @@ -1119,7 +1119,6 @@ "hunter", "hunter_tools", "improve.evolve", - "index", "learn.features", "learn.labels", "learn.roles", diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index bdf70bc..294b9db 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -43,7 +43,6 @@ from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config from .hunter import run_hunter_pool, write_hunter_trajectories from .improve import RoundOutcome, run_improvement -from .index import build_code_chunks from .learn.cli import add_commands as add_learn_commands from .learn.cli import run as run_learn from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints @@ -173,11 +172,6 @@ def _main(argv: list[str] | None) -> int: scan.add_argument("--config", type=Path, default=Path("openultrasast.toml")) scan.add_argument("--fail-on", choices=("never", "findings", "verified", "worth-fixing"), default="never") - index = subparsers.add_parser("index", help="chunk a local repository for embedding index construction") - index.add_argument("path", type=Path) - index.add_argument("--config", type=Path, default=Path("openultrasast.toml")) - index.add_argument("--chunk-lines", type=int, default=80) - benchmark = subparsers.add_parser("benchmark", help="run a benchmark manifest and write scoreboard artifacts") benchmark.add_argument("manifest", type=Path) benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick") @@ -426,8 +420,6 @@ def _main(argv: list[str] | None) -> int: return delivery.exit_code if args.command == "scan": return _scan(args.path, args.config, args.mode, args.fail_on) - if args.command == "index": - return _index(args.path, args.config, args.chunk_lines) if args.command == "benchmark": return _benchmark(args.manifest, args.config, args.mode) if args.command == "improve": @@ -1564,30 +1556,5 @@ def _model_candidates(args: argparse.Namespace) -> int: return 0 -def _index(path: Path, config_path: Path, chunk_lines: int) -> int: - if not path.exists() or not path.is_dir(): - raise SystemExit(f"index path is not a directory: {path}") - config = load_config(config_path if config_path.exists() else None) - run = create_scan_run(path, config) - snapshot, targets = preprocess_repository(run.target, run.root / "preprocess" / "file_targets.json") - chunks = build_code_chunks(run.target, targets, max_lines=chunk_lines) - payload = { - "store": config.embeddings.store or "json-local", - "embedding_model": config.embeddings.model, - "repo_root": snapshot.root, - "repo_commit": snapshot.commit, - "chunk_count": len(chunks), - "chunks": [chunk.__dict__ for chunk in chunks], - } - path_out = run.root / "index" / "chunks.json" - path_out.parent.mkdir(parents=True, exist_ok=True) - path_out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - print(f"scan_id={run.scan_id}") - print(f"run_dir={run.root}") - print(f"chunks={len(chunks)}") - print(f"index_artifact={path_out}") - return 0 - - if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index db06bde..0aa6bec 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -10,8 +10,6 @@ from typing import Literal from openultrasast.contracts import Contract -DEFAULT_VECTOR_STORE = "json-local" - # Retirement notes (harnessx-removal Req 1.3, 3.3). A key that asked for a retired LLM capability fails and names # the replacement, never a silent downgrade; the old budget section only warns. Every line below that names the # removed plane carries its retirement date, which the reference-search test holds the rest of the tree to. @@ -26,6 +24,10 @@ _UNREAD_ON = "retired 2026-10-02 because nothing ever read it" _UNREAD_DYNAMIC = "no dynamic or deep tier exists; deep mode will define its own keys when it is built" _UNREAD_EVIDENCE = "evidence tiers come from the program model's ladder (model/ladder.py), not from configuration" _UNREAD_VARIANTS = "the structural variant search it configured was deleted with semantic/variants.py" +_UNREAD_EMBEDDINGS = ( + "the json-local vector index it configured and `ousast index` were deleted; the decision engine embeds through " + "`ousast learn memory embed --model` (learn/embeddings.py)" +) RETIRED_KEYS: dict[tuple[str, str], str] = { ("models", "verifier"): f"{_RETIRED_ON}: LLM verification runs on the plane (`verify` + `agree`, {_PLANE_RUN})", ("fusion", "panel_model"): f"{_RETIRED_ON}: {_RETIRED_PANELS}", @@ -38,6 +40,8 @@ RETIRED_KEYS: dict[tuple[str, str], str] = { ("evidence", "minimum_patch"): f"{_UNREAD_ON}: {_UNREAD_EVIDENCE}", ("variants", "enabled"): f"{_UNREAD_ON}: {_UNREAD_VARIANTS}", ("variants", "max_mechanisms"): f"{_UNREAD_ON}: {_UNREAD_VARIANTS}", + ("embeddings", "model"): f"{_UNREAD_ON}: {_UNREAD_EMBEDDINGS}", + ("embeddings", "store"): f"{_UNREAD_ON}: {_UNREAD_EMBEDDINGS}", } RETIRED_SECTION = "harnessx" # retired 2026-09-30: loads with one warning and is otherwise ignored (Req 3.3) RETIRED_SECTION_WARNING = ( @@ -94,12 +98,6 @@ class ModelConfig: thinking: bool = False -@dataclass(frozen=True) -class EmbeddingConfig: - model: str | None = None - store: str = DEFAULT_VECTOR_STORE - - @dataclass(frozen=True) class SandboxConfig: network: bool = False @@ -205,7 +203,6 @@ class PushConfig(Contract): @dataclass(frozen=True) class ResolvedConfig: models: ModelConfig = ModelConfig() - embeddings: EmbeddingConfig = EmbeddingConfig() sandbox: SandboxConfig = SandboxConfig() static_analysis: StaticAnalysisConfig = StaticAnalysisConfig() score: ScoreConfig = ScoreConfig() @@ -231,7 +228,6 @@ def load_config(config_path: Path | None = None, *, dotenv: bool = True) -> Reso return ResolvedConfig( models=_load_models(data.get("models", {})), - embeddings=_load_embeddings(data.get("embeddings", {})), sandbox=_load_sandbox(data.get("sandbox", {})), static_analysis=_load_static_analysis(data.get("static_analysis", {})), score=_load_score(data.get("score", {})), @@ -277,14 +273,6 @@ def _load_models(value: object) -> ModelConfig: ) -def _load_embeddings(value: object) -> EmbeddingConfig: - data = _section(value) - return EmbeddingConfig( - model=_string(data.get("model")) or _string(os.environ.get("OPENROUTER_EMBEDDING_MODEL")), - store=_string(data.get("store")) or DEFAULT_VECTOR_STORE, - ) - - def _load_sandbox(value: object) -> SandboxConfig: data = _section(value) return SandboxConfig( diff --git a/src/openultrasast/index.py b/src/openultrasast/index.py index 3e179a1..01eb221 100644 --- a/src/openultrasast/index.py +++ b/src/openultrasast/index.py @@ -1,13 +1,14 @@ +"""Text chunking for the skill router (`skills.py`). + +The vector index that once lived beside the chunker (`VectorIndex`, `build_vector_index`, `query_vector_index`, +`build_retrieval_package`) and the `ousast index` subcommand that fed it were removed on 2026-10-02: nothing ever +read the `chunks.json` they wrote. The decision engine's embeddings live in `learn/embeddings.py`. +""" + from __future__ import annotations import hashlib -import json -import math -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Protocol - -from .preprocess import FileTarget, RepoSnapshot +from dataclasses import dataclass VALID_NAMESPACES = {"repo_code", "docs", "static_findings", "mechanisms", "skills", "traces"} @@ -24,82 +25,6 @@ class CodeChunk: metadata: dict[str, str | int | bool] -@dataclass(frozen=True) -class VectorRecord: - chunk: CodeChunk - embedding: list[float] - - -@dataclass(frozen=True) -class VectorIndex: - store: str - embedding_model: str - repo_root: str - repo_commit: str | None - records: list[VectorRecord] - - -@dataclass(frozen=True) -class RetrievalHit: - score: float - chunk: CodeChunk - - -@dataclass(frozen=True) -class RetrievalPackage: - role: str - hits: list[RetrievalHit] - text: str - truncated: bool - - -class EmbeddingClient(Protocol): - def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]: - raise NotImplementedError - - -def chunk_file(root: Path, target: FileTarget, *, max_lines: int = 80) -> list[CodeChunk]: - if max_lines < 1: - raise ValueError("max_lines must be at least 1") - text = (root / target.path).read_text(errors="ignore") - lines = text.splitlines() - chunks: list[CodeChunk] = [] - for start in range(0, len(lines), max_lines): - chunk_lines = lines[start : start + max_lines] - if not any(line.strip() for line in chunk_lines): - continue - start_line = start + 1 - end_line = start + len(chunk_lines) - chunk_text = "\n".join(chunk_lines) - chunks.append( - CodeChunk( - chunk_id=_chunk_id(target.path, start_line, end_line, chunk_text), - namespace="repo_code", - path=target.path, - language=target.language, - start_line=start_line, - end_line=end_line, - text=chunk_text, - metadata={ - "path": target.path, - "language": target.language, - "start_line": start_line, - "end_line": end_line, - "has_fuzz_entry_point": target.has_fuzz_entry_point, - "tags": ",".join(target.tags), - }, - ) - ) - return chunks - - -def build_code_chunks(root: Path, targets: list[FileTarget], *, max_lines: int = 80) -> list[CodeChunk]: - chunks: list[CodeChunk] = [] - for target in targets: - chunks.extend(chunk_file(root, target, max_lines=max_lines)) - return chunks - - def chunk_text_namespace( *, namespace: str, @@ -144,130 +69,6 @@ def chunk_text_namespace( return chunks -def index_reuse_key(snapshot: RepoSnapshot, embedding_model: str) -> str: - source = { - "root": snapshot.root, - "commit": snapshot.commit, - "file_count": snapshot.file_count, - "languages": snapshot.languages, - "embedding_model": embedding_model, - } - return hashlib.sha256(json.dumps(source, sort_keys=True).encode()).hexdigest()[:24] - - -def build_vector_index( - *, - snapshot: RepoSnapshot, - chunks: list[CodeChunk], - embedding_model: str, - client: EmbeddingClient, - store: str = "json-local", - batch_size: int = 32, -) -> VectorIndex: - if batch_size < 1: - raise ValueError("batch_size must be at least 1") - records: list[VectorRecord] = [] - for start in range(0, len(chunks), batch_size): - batch = chunks[start : start + batch_size] - vectors = client.embed(model=embedding_model, inputs=[chunk.text for chunk in batch]) - if len(vectors) != len(batch): - raise ValueError("embedding response count did not match chunk count") - records.extend(VectorRecord(chunk=chunk, embedding=vector) for chunk, vector in zip(batch, vectors, strict=True)) - return VectorIndex( - store=store, - embedding_model=embedding_model, - repo_root=snapshot.root, - repo_commit=snapshot.commit, - records=records, - ) - - -def write_vector_index(index: VectorIndex, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(_index_to_payload(index), indent=2, sort_keys=True) + "\n") - - -def read_vector_index(path: Path) -> VectorIndex: - payload = json.loads(path.read_text()) - return VectorIndex( - store=str(payload["store"]), - embedding_model=str(payload["embedding_model"]), - repo_root=str(payload["repo_root"]), - repo_commit=payload.get("repo_commit"), - records=[ - VectorRecord( - chunk=CodeChunk(**item["chunk"]), - embedding=[float(value) for value in item["embedding"]], - ) - for item in payload.get("records", []) - ], - ) - - -def query_vector_index( - index: VectorIndex, - query_embedding: list[float], - *, - metadata_filter: dict[str, str | int | bool] | None = None, - limit: int = 5, -) -> list[tuple[float, CodeChunk]]: - matches: list[tuple[float, CodeChunk]] = [] - for record in index.records: - if metadata_filter and not _metadata_matches(record.chunk.metadata, metadata_filter): - continue - matches.append((_cosine_similarity(query_embedding, record.embedding), record.chunk)) - return sorted(matches, key=lambda item: item[0], reverse=True)[:limit] - - -def build_retrieval_package( - *, - role: str, - index: VectorIndex, - query_embedding: list[float], - metadata_filter: dict[str, str | int | bool] | None = None, - limit: int = 5, - max_chars: int = 4000, -) -> RetrievalPackage: - hits: list[RetrievalHit] = [] - parts: list[str] = [] - total = 0 - truncated = False - for score, chunk in query_vector_index(index, query_embedding, metadata_filter=metadata_filter, limit=limit): - block = f"[{chunk.namespace}] {chunk.path}:{chunk.start_line}-{chunk.end_line}\n{chunk.text}" - if total + len(block) > max_chars: - truncated = True - break - hits.append(RetrievalHit(score=score, chunk=chunk)) - parts.append(block) - total += len(block) - return RetrievalPackage(role=role, hits=hits, text="\n\n".join(parts), truncated=truncated) - - -def _index_to_payload(index: VectorIndex) -> dict[str, object]: - return { - "store": index.store, - "embedding_model": index.embedding_model, - "repo_root": index.repo_root, - "repo_commit": index.repo_commit, - "records": [{"chunk": asdict(record.chunk), "embedding": record.embedding} for record in index.records], - } - - -def _metadata_matches(metadata: dict[str, str | int | bool], expected: dict[str, str | int | bool]) -> bool: - return all(metadata.get(key) == value for key, value in expected.items()) - - -def _cosine_similarity(left: list[float], right: list[float]) -> float: - if len(left) != len(right): - return 0.0 - numerator = sum(a * b for a, b in zip(left, right, strict=True)) - left_norm = math.sqrt(sum(a * a for a in left)) - right_norm = math.sqrt(sum(b * b for b in right)) - if left_norm == 0 or right_norm == 0: - return 0.0 - return numerator / (left_norm * right_norm) - - def _chunk_id(path: str, start_line: int, end_line: int, text: str) -> str: digest = hashlib.sha256(f"{path}:{start_line}:{end_line}:{text}".encode()).hexdigest()[:16] return f"{path}:{start_line}-{end_line}:{digest}" diff --git a/tests/test_config.py b/tests/test_config.py index 0e7b8e1..e23bdf8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -25,7 +25,6 @@ def test_load_config_reads_toml(tmp_path: Path) -> None: config = load_config(config_path) assert config.models.ranker == "openrouter/test-ranker" - assert config.embeddings.store == "json-local" assert config.sandbox.memory_mb == 1024 @@ -74,6 +73,8 @@ def test_retired_llm_keys_fail_naming_the_replacement(tmp_path: Path, body: str, ('[evidence]\nminimum_patch = "root_cause_explained"\n', "`[evidence] minimum_patch`"), ("[variants]\nenabled = false\n", "`[variants] enabled`"), ("[variants]\nmax_mechanisms = 10\n", "`[variants] max_mechanisms`"), + ('[embeddings]\nmodel = "openai/text-embedding-3-small"\n', "`[embeddings] model`"), + ('[embeddings]\nstore = "json-local"\n', "`[embeddings] store`"), ], ) def test_unread_keys_fail_naming_why(tmp_path: Path, body: str, key: str) -> None: @@ -140,7 +141,7 @@ def test_write_resolved_config_creates_json(tmp_path: Path) -> None: payload = json.loads(output.read_text()) assert payload["sandbox"]["memory_mb"] == 2048 # retired 2026-10-02 (never read): the unread sections are not echoed as if they configured something - assert not {"dynamic", "evidence", "variants"} & set(payload) and "patcher" not in payload["models"] + assert not {"dynamic", "evidence", "variants", "embeddings"} & set(payload) and "patcher" not in payload["models"] def test_complexity_and_regress_defaults_when_section_absent() -> None: @@ -208,17 +209,11 @@ def test_load_config_reads_regress_image_pins(tmp_path: Path) -> None: def test_load_dotenv_sets_missing_keys_only(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] env_file = tmp_path / ".env" - env_file.write_text("OPENROUTER_API_KEY=from-file\nOPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small\n") + env_file.write_text("OPENROUTER_API_KEY=from-file\nS3_REGION=us-east-1\n") monkeypatch.setenv("OPENROUTER_API_KEY", "already-set") - monkeypatch.delenv("OPENROUTER_EMBEDDING_MODEL", raising=False) + monkeypatch.delenv("S3_REGION", raising=False) load_dotenv(env_file, force=True) assert os.environ["OPENROUTER_API_KEY"] == "already-set" - assert os.environ["OPENROUTER_EMBEDDING_MODEL"] == "openai/text-embedding-3-small" - - -def test_embeddings_model_from_env(monkeypatch) -> None: # type: ignore[no-untyped-def] - monkeypatch.setenv("OPENROUTER_EMBEDDING_MODEL", "openai/text-embedding-3-small") - config = load_config(None) - assert config.embeddings.model == "openai/text-embedding-3-small" + assert os.environ["S3_REGION"] == "us-east-1" diff --git a/tests/test_index.py b/tests/test_index.py index 6cbc0ab..a2179ee 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1,80 +1,9 @@ -import json -from pathlib import Path - import pytest -from openultrasast.index import ( - CodeChunk, - build_code_chunks, - build_retrieval_package, - build_vector_index, - chunk_text_namespace, - index_reuse_key, - query_vector_index, - read_vector_index, - write_vector_index, -) -from openultrasast.preprocess import RepoSnapshot, preprocess_repository +from openultrasast.index import chunk_text_namespace -class FakeEmbeddingClient: - def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]: - return [[float(len(text)), 1.0] for text in inputs] - - -def test_build_code_chunks_preserves_metadata_filters(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def login(user):\n return user\n") - _, targets = preprocess_repository(repo) - - chunks = build_code_chunks(repo, targets, max_lines=1) - - assert len(chunks) == 2 - assert chunks[0].metadata["path"] == "app.py" - assert chunks[0].metadata["language"] == "python" - assert chunks[0].start_line == 1 - - -def test_json_vector_index_round_trips_and_filters(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def login(user):\n return user\n") - snapshot, targets = preprocess_repository(repo) - chunks = build_code_chunks(repo, targets) - index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="openrouter/embed", client=FakeEmbeddingClient()) - output = tmp_path / "index.json" - - write_vector_index(index, output) - loaded = read_vector_index(output) - matches = query_vector_index(loaded, [20.0, 1.0], metadata_filter={"language": "python"}) - - assert loaded.embedding_model == "openrouter/embed" - assert matches[0][1].path == "app.py" - assert json.loads(output.read_text())["store"] == "json-local" - - -def test_vector_index_rejects_embedding_count_mismatch() -> None: - class EmptyEmbeddingClient: - def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]: - return [] - - chunk = CodeChunk( - chunk_id="chunk-1", - namespace="repo_code", - path="app.py", - language="python", - start_line=1, - end_line=1, - text="print('x')", - metadata={"path": "app.py", "language": "python"}, - ) - snapshot = RepoSnapshot(root="/repo", commit=None, file_count=0, languages={}) - with pytest.raises(ValueError): - build_vector_index(snapshot=snapshot, chunks=[chunk], embedding_model="model", client=EmptyEmbeddingClient()) - - -def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None: +def test_namespace_chunks_are_metadata_scoped() -> None: chunks = chunk_text_namespace( namespace="skills", path="semgrep.md", @@ -82,21 +11,23 @@ def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None: metadata={"language": "markdown", "vulnerability_class": "mapping"}, max_lines=1, ) - snapshot = RepoSnapshot(root="/repo", commit="abc", file_count=1, languages={"python": 1}) assert len(chunks) == 2 assert chunks[0].namespace == "skills" + assert chunks[0].language == "markdown" assert chunks[0].metadata["vulnerability_class"] == "mapping" - assert index_reuse_key(snapshot, "embed-a") != index_reuse_key(snapshot, "embed-b") + assert (chunks[0].start_line, chunks[0].end_line) == (1, 1) + assert chunks[0].chunk_id != chunks[1].chunk_id -def test_retrieval_package_is_bounded_by_character_budget() -> None: - chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n" + "beta" * 100, max_lines=1) - snapshot = RepoSnapshot(root="/repo", commit=None, file_count=1, languages={"markdown": 1}) - index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient()) +def test_blank_blocks_are_skipped_and_the_language_defaults_to_text() -> None: + chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n\n\nbeta\n", max_lines=1) - package = build_retrieval_package(role="verifier", index=index, query_embedding=[4.0, 1.0], max_chars=80) + assert [chunk.text for chunk in chunks] == ["alpha", "beta"] + assert chunks[0].language == "text" - assert package.role == "verifier" - assert package.hits - assert package.truncated is True + +@pytest.mark.parametrize(("namespace", "max_lines"), [("unknown", 80), ("docs", 0)]) +def test_invalid_namespace_or_window_is_rejected(namespace: str, max_lines: int) -> None: + with pytest.raises(ValueError): + chunk_text_namespace(namespace=namespace, path="x", text="y", max_lines=max_lines)