mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
chore(index): remove the unused vector index, the ousast index subcommand and the [embeddings] section
Legacy inventory 2, item R1. The vector half of index.py (VectorIndex, VectorRecord, RetrievalHit, RetrievalPackage, the EmbeddingClient protocol, index_reuse_key, build/write/read/query_vector_index, build_retrieval_package and their helpers) had no importer in src; `ousast index` wrote index/chunks.json that nothing read, and was the only reader of `[embeddings]`. index.py keeps CodeChunk and chunk_text_namespace, which skills.py imports; chunk_file/build_code_chunks go with their only caller. `[embeddings] model` and `[embeddings] store` join RETIRED_KEYS (RetiredConfigError naming the reason), the same way the [variants] keys were retired; OPENROUTER_EMBEDDING_MODEL is no longer read, so the README and .env.example drop it (`ousast learn memory embed --model` names the model). test_index.py now tests the chunker only; test_config.py covers the two retired keys and no longer asserts the removed field. The module audit record is regenerated (counts unchanged: 151 modules, 0 orphaned; the index module's importer list shrinks to skills). RELEASE_NOTES Unreleased carries the line. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
69b5e0efcc
commit
e1db7e1920
+6
-11
@@ -25,7 +25,6 @@ def test_load_config_reads_toml(tmp_path: Path) -> None:
|
||||
config = load_config(config_path)
|
||||
|
||||
assert config.models.ranker == "openrouter/test-ranker"
|
||||
assert config.embeddings.store == "json-local"
|
||||
assert config.sandbox.memory_mb == 1024
|
||||
|
||||
|
||||
@@ -74,6 +73,8 @@ def test_retired_llm_keys_fail_naming_the_replacement(tmp_path: Path, body: str,
|
||||
('[evidence]\nminimum_patch = "root_cause_explained"\n', "`[evidence] minimum_patch`"),
|
||||
("[variants]\nenabled = false\n", "`[variants] enabled`"),
|
||||
("[variants]\nmax_mechanisms = 10\n", "`[variants] max_mechanisms`"),
|
||||
('[embeddings]\nmodel = "openai/text-embedding-3-small"\n', "`[embeddings] model`"),
|
||||
('[embeddings]\nstore = "json-local"\n', "`[embeddings] store`"),
|
||||
],
|
||||
)
|
||||
def test_unread_keys_fail_naming_why(tmp_path: Path, body: str, key: str) -> None:
|
||||
@@ -140,7 +141,7 @@ def test_write_resolved_config_creates_json(tmp_path: Path) -> None:
|
||||
payload = json.loads(output.read_text())
|
||||
assert payload["sandbox"]["memory_mb"] == 2048
|
||||
# retired 2026-10-02 (never read): the unread sections are not echoed as if they configured something
|
||||
assert not {"dynamic", "evidence", "variants"} & set(payload) and "patcher" not in payload["models"]
|
||||
assert not {"dynamic", "evidence", "variants", "embeddings"} & set(payload) and "patcher" not in payload["models"]
|
||||
|
||||
|
||||
def test_complexity_and_regress_defaults_when_section_absent() -> None:
|
||||
@@ -208,17 +209,11 @@ def test_load_config_reads_regress_image_pins(tmp_path: Path) -> None:
|
||||
|
||||
def test_load_dotenv_sets_missing_keys_only(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
env_file = tmp_path / ".env"
|
||||
env_file.write_text("OPENROUTER_API_KEY=from-file\nOPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small\n")
|
||||
env_file.write_text("OPENROUTER_API_KEY=from-file\nS3_REGION=us-east-1\n")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "already-set")
|
||||
monkeypatch.delenv("OPENROUTER_EMBEDDING_MODEL", raising=False)
|
||||
monkeypatch.delenv("S3_REGION", raising=False)
|
||||
|
||||
load_dotenv(env_file, force=True)
|
||||
|
||||
assert os.environ["OPENROUTER_API_KEY"] == "already-set"
|
||||
assert os.environ["OPENROUTER_EMBEDDING_MODEL"] == "openai/text-embedding-3-small"
|
||||
|
||||
|
||||
def test_embeddings_model_from_env(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
monkeypatch.setenv("OPENROUTER_EMBEDDING_MODEL", "openai/text-embedding-3-small")
|
||||
config = load_config(None)
|
||||
assert config.embeddings.model == "openai/text-embedding-3-small"
|
||||
assert os.environ["S3_REGION"] == "us-east-1"
|
||||
|
||||
+14
-83
@@ -1,80 +1,9 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.index import (
|
||||
CodeChunk,
|
||||
build_code_chunks,
|
||||
build_retrieval_package,
|
||||
build_vector_index,
|
||||
chunk_text_namespace,
|
||||
index_reuse_key,
|
||||
query_vector_index,
|
||||
read_vector_index,
|
||||
write_vector_index,
|
||||
)
|
||||
from openultrasast.preprocess import RepoSnapshot, preprocess_repository
|
||||
from openultrasast.index import chunk_text_namespace
|
||||
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
||||
return [[float(len(text)), 1.0] for text in inputs]
|
||||
|
||||
|
||||
def test_build_code_chunks_preserves_metadata_filters(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def login(user):\n return user\n")
|
||||
_, targets = preprocess_repository(repo)
|
||||
|
||||
chunks = build_code_chunks(repo, targets, max_lines=1)
|
||||
|
||||
assert len(chunks) == 2
|
||||
assert chunks[0].metadata["path"] == "app.py"
|
||||
assert chunks[0].metadata["language"] == "python"
|
||||
assert chunks[0].start_line == 1
|
||||
|
||||
|
||||
def test_json_vector_index_round_trips_and_filters(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def login(user):\n return user\n")
|
||||
snapshot, targets = preprocess_repository(repo)
|
||||
chunks = build_code_chunks(repo, targets)
|
||||
index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="openrouter/embed", client=FakeEmbeddingClient())
|
||||
output = tmp_path / "index.json"
|
||||
|
||||
write_vector_index(index, output)
|
||||
loaded = read_vector_index(output)
|
||||
matches = query_vector_index(loaded, [20.0, 1.0], metadata_filter={"language": "python"})
|
||||
|
||||
assert loaded.embedding_model == "openrouter/embed"
|
||||
assert matches[0][1].path == "app.py"
|
||||
assert json.loads(output.read_text())["store"] == "json-local"
|
||||
|
||||
|
||||
def test_vector_index_rejects_embedding_count_mismatch() -> None:
|
||||
class EmptyEmbeddingClient:
|
||||
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
||||
return []
|
||||
|
||||
chunk = CodeChunk(
|
||||
chunk_id="chunk-1",
|
||||
namespace="repo_code",
|
||||
path="app.py",
|
||||
language="python",
|
||||
start_line=1,
|
||||
end_line=1,
|
||||
text="print('x')",
|
||||
metadata={"path": "app.py", "language": "python"},
|
||||
)
|
||||
snapshot = RepoSnapshot(root="/repo", commit=None, file_count=0, languages={})
|
||||
with pytest.raises(ValueError):
|
||||
build_vector_index(snapshot=snapshot, chunks=[chunk], embedding_model="model", client=EmptyEmbeddingClient())
|
||||
|
||||
|
||||
def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None:
|
||||
def test_namespace_chunks_are_metadata_scoped() -> None:
|
||||
chunks = chunk_text_namespace(
|
||||
namespace="skills",
|
||||
path="semgrep.md",
|
||||
@@ -82,21 +11,23 @@ def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None:
|
||||
metadata={"language": "markdown", "vulnerability_class": "mapping"},
|
||||
max_lines=1,
|
||||
)
|
||||
snapshot = RepoSnapshot(root="/repo", commit="abc", file_count=1, languages={"python": 1})
|
||||
|
||||
assert len(chunks) == 2
|
||||
assert chunks[0].namespace == "skills"
|
||||
assert chunks[0].language == "markdown"
|
||||
assert chunks[0].metadata["vulnerability_class"] == "mapping"
|
||||
assert index_reuse_key(snapshot, "embed-a") != index_reuse_key(snapshot, "embed-b")
|
||||
assert (chunks[0].start_line, chunks[0].end_line) == (1, 1)
|
||||
assert chunks[0].chunk_id != chunks[1].chunk_id
|
||||
|
||||
|
||||
def test_retrieval_package_is_bounded_by_character_budget() -> None:
|
||||
chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n" + "beta" * 100, max_lines=1)
|
||||
snapshot = RepoSnapshot(root="/repo", commit=None, file_count=1, languages={"markdown": 1})
|
||||
index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient())
|
||||
def test_blank_blocks_are_skipped_and_the_language_defaults_to_text() -> None:
|
||||
chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n\n\nbeta\n", max_lines=1)
|
||||
|
||||
package = build_retrieval_package(role="verifier", index=index, query_embedding=[4.0, 1.0], max_chars=80)
|
||||
assert [chunk.text for chunk in chunks] == ["alpha", "beta"]
|
||||
assert chunks[0].language == "text"
|
||||
|
||||
assert package.role == "verifier"
|
||||
assert package.hits
|
||||
assert package.truncated is True
|
||||
|
||||
@pytest.mark.parametrize(("namespace", "max_lines"), [("unknown", 80), ("docs", 0)])
|
||||
def test_invalid_namespace_or_window_is_rejected(namespace: str, max_lines: int) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
chunk_text_namespace(namespace=namespace, path="x", text="y", max_lines=max_lines)
|
||||
|
||||
Reference in New Issue
Block a user