mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
e1db7e1920
Legacy inventory 2, item R1. The vector half of index.py (VectorIndex, VectorRecord, RetrievalHit, RetrievalPackage, the EmbeddingClient protocol, index_reuse_key, build/write/read/query_vector_index, build_retrieval_package and their helpers) had no importer in src; `ousast index` wrote index/chunks.json that nothing read, and was the only reader of `[embeddings]`. index.py keeps CodeChunk and chunk_text_namespace, which skills.py imports; chunk_file/build_code_chunks go with their only caller. `[embeddings] model` and `[embeddings] store` join RETIRED_KEYS (RetiredConfigError naming the reason), the same way the [variants] keys were retired; OPENROUTER_EMBEDDING_MODEL is no longer read, so the README and .env.example drop it (`ousast learn memory embed --model` names the model). test_index.py now tests the chunker only; test_config.py covers the two retired keys and no longer asserts the removed field. The module audit record is regenerated (counts unchanged: 151 modules, 0 orphaned; the index module's importer list shrinks to skills). RELEASE_NOTES Unreleased carries the line. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
75 lines
2.5 KiB
Python
75 lines
2.5 KiB
Python
"""Text chunking for the skill router (`skills.py`).
|
|
|
|
The vector index that once lived beside the chunker (`VectorIndex`, `build_vector_index`, `query_vector_index`,
|
|
`build_retrieval_package`) and the `ousast index` subcommand that fed it were removed on 2026-10-02: nothing ever
|
|
read the `chunks.json` they wrote. The decision engine's embeddings live in `learn/embeddings.py`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from dataclasses import dataclass
|
|
|
|
VALID_NAMESPACES = {"repo_code", "docs", "static_findings", "mechanisms", "skills", "traces"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodeChunk:
|
|
chunk_id: str
|
|
namespace: str
|
|
path: str
|
|
language: str
|
|
start_line: int
|
|
end_line: int
|
|
text: str
|
|
metadata: dict[str, str | int | bool]
|
|
|
|
|
|
def chunk_text_namespace(
|
|
*,
|
|
namespace: str,
|
|
path: str,
|
|
text: str,
|
|
metadata: dict[str, str | int | bool] | None = None,
|
|
max_lines: int = 80,
|
|
) -> list[CodeChunk]:
|
|
if namespace not in VALID_NAMESPACES:
|
|
raise ValueError(f"unsupported namespace: {namespace}")
|
|
if max_lines < 1:
|
|
raise ValueError("max_lines must be at least 1")
|
|
lines = text.splitlines()
|
|
chunks: list[CodeChunk] = []
|
|
for start in range(0, len(lines), max_lines):
|
|
chunk_lines = lines[start : start + max_lines]
|
|
if not any(line.strip() for line in chunk_lines):
|
|
continue
|
|
start_line = start + 1
|
|
end_line = start + len(chunk_lines)
|
|
chunk_text = "\n".join(chunk_lines)
|
|
chunk_metadata: dict[str, str | int | bool] = {
|
|
"path": path,
|
|
"namespace": namespace,
|
|
"start_line": start_line,
|
|
"end_line": end_line,
|
|
}
|
|
if metadata:
|
|
chunk_metadata.update({key: value for key, value in metadata.items() if isinstance(value, str | int | bool)})
|
|
chunks.append(
|
|
CodeChunk(
|
|
chunk_id=_chunk_id(f"{namespace}:{path}", start_line, end_line, chunk_text),
|
|
namespace=namespace,
|
|
path=path,
|
|
language=str(chunk_metadata.get("language", "text")),
|
|
start_line=start_line,
|
|
end_line=end_line,
|
|
text=chunk_text,
|
|
metadata=chunk_metadata,
|
|
)
|
|
)
|
|
return chunks
|
|
|
|
|
|
def _chunk_id(path: str, start_line: int, end_line: int, text: str) -> str:
|
|
digest = hashlib.sha256(f"{path}:{start_line}:{end_line}:{text}".encode()).hexdigest()[:16]
|
|
return f"{path}:{start_line}-{end_line}:{digest}"
|