mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
fe45081b43
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
103 lines
3.7 KiB
Python
103 lines
3.7 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openultrasast.index import (
|
|
CodeChunk,
|
|
build_code_chunks,
|
|
build_retrieval_package,
|
|
build_vector_index,
|
|
chunk_text_namespace,
|
|
index_reuse_key,
|
|
query_vector_index,
|
|
read_vector_index,
|
|
write_vector_index,
|
|
)
|
|
from openultrasast.preprocess import RepoSnapshot, preprocess_repository
|
|
|
|
|
|
class FakeEmbeddingClient:
|
|
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
|
return [[float(len(text)), 1.0] for text in inputs]
|
|
|
|
|
|
def test_build_code_chunks_preserves_metadata_filters(tmp_path: Path) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text("def login(user):\n return user\n")
|
|
_, targets = preprocess_repository(repo)
|
|
|
|
chunks = build_code_chunks(repo, targets, max_lines=1)
|
|
|
|
assert len(chunks) == 2
|
|
assert chunks[0].metadata["path"] == "app.py"
|
|
assert chunks[0].metadata["language"] == "python"
|
|
assert chunks[0].start_line == 1
|
|
|
|
|
|
def test_json_vector_index_round_trips_and_filters(tmp_path: Path) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text("def login(user):\n return user\n")
|
|
snapshot, targets = preprocess_repository(repo)
|
|
chunks = build_code_chunks(repo, targets)
|
|
index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="openrouter/embed", client=FakeEmbeddingClient())
|
|
output = tmp_path / "index.json"
|
|
|
|
write_vector_index(index, output)
|
|
loaded = read_vector_index(output)
|
|
matches = query_vector_index(loaded, [20.0, 1.0], metadata_filter={"language": "python"})
|
|
|
|
assert loaded.embedding_model == "openrouter/embed"
|
|
assert matches[0][1].path == "app.py"
|
|
assert json.loads(output.read_text())["store"] == "json-local"
|
|
|
|
|
|
def test_vector_index_rejects_embedding_count_mismatch() -> None:
|
|
class EmptyEmbeddingClient:
|
|
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
|
return []
|
|
|
|
chunk = CodeChunk(
|
|
chunk_id="chunk-1",
|
|
namespace="repo_code",
|
|
path="app.py",
|
|
language="python",
|
|
start_line=1,
|
|
end_line=1,
|
|
text="print('x')",
|
|
metadata={"path": "app.py", "language": "python"},
|
|
)
|
|
snapshot = RepoSnapshot(root="/repo", commit=None, file_count=0, languages={})
|
|
with pytest.raises(ValueError):
|
|
build_vector_index(snapshot=snapshot, chunks=[chunk], embedding_model="model", client=EmptyEmbeddingClient())
|
|
|
|
|
|
def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None:
|
|
chunks = chunk_text_namespace(
|
|
namespace="skills",
|
|
path="semgrep.md",
|
|
text="Use Semgrep for variant mapping.\nKeep provenance.",
|
|
metadata={"language": "markdown", "vulnerability_class": "mapping"},
|
|
max_lines=1,
|
|
)
|
|
snapshot = RepoSnapshot(root="/repo", commit="abc", file_count=1, languages={"python": 1})
|
|
|
|
assert len(chunks) == 2
|
|
assert chunks[0].namespace == "skills"
|
|
assert chunks[0].metadata["vulnerability_class"] == "mapping"
|
|
assert index_reuse_key(snapshot, "embed-a") != index_reuse_key(snapshot, "embed-b")
|
|
|
|
|
|
def test_retrieval_package_is_bounded_by_character_budget() -> None:
|
|
chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n" + "beta" * 100, max_lines=1)
|
|
snapshot = RepoSnapshot(root="/repo", commit=None, file_count=1, languages={"markdown": 1})
|
|
index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient())
|
|
|
|
package = build_retrieval_package(role="verifier", index=index, query_embedding=[4.0, 1.0], max_chars=80)
|
|
|
|
assert package.role == "verifier"
|
|
assert package.hits
|
|
assert package.truncated is True
|