diff --git a/.kiro/specs/harnessx-self-improving-rulesets/spec.json b/.kiro/specs/harnessx-self-improving-rulesets/spec.json index 14ed480..2b20d0a 100644 --- a/.kiro/specs/harnessx-self-improving-rulesets/spec.json +++ b/.kiro/specs/harnessx-self-improving-rulesets/spec.json @@ -1,22 +1,22 @@ { "feature_name": "harnessx-self-improving-rulesets", "created_at": "2026-06-22T10:07:18Z", - "updated_at": "2026-06-22T10:07:18Z", + "updated_at": "2026-06-22T10:20:00Z", "language": "en", - "phase": "tasks-generated", + "phase": "implementation", "approvals": { "requirements": { "generated": true, - "approved": false + "approved": true }, "design": { "generated": true, - "approved": false + "approved": true }, "tasks": { "generated": true, - "approved": false + "approved": true } }, - "ready_for_implementation": false + "ready_for_implementation": true } diff --git a/.kiro/specs/harnessx-self-improving-rulesets/tasks.md b/.kiro/specs/harnessx-self-improving-rulesets/tasks.md index ca0459e..7ed590f 100644 --- a/.kiro/specs/harnessx-self-improving-rulesets/tasks.md +++ b/.kiro/specs/harnessx-self-improving-rulesets/tasks.md @@ -4,30 +4,30 @@ This plan splits the fused regex layer into three governed owners (rules detect, ## Phase 1 — Foundation: Governance & Scoring Planes (zero-dep) -- [ ] 1. Stand up the central CWE policy authority -- [ ] 1.1 Vendor the verycode CWE severity data and build the Policy Store loader +- [x] 1. Stand up the central CWE policy authority +- [x] 1.1 Vendor the verycode CWE severity data and build the Policy Store loader - Vendor the verycode CWE-to-severity dataset into the governance plane and parse it with the standard library only, tolerating the trailing-space severity column header. - Expose, per CWE, an authoritative 0-5 severity, a static scope flag, and a dynamic scope flag; mark dynamic-only CWEs as report-only. - Observable completion: loading the policy yields the full set of CWE policy entries with correct static/dynamic flags and zero third-party imports, verified by a unit test. - _Requirements: 2.6, 2.9_ - _Boundary: PolicyStore_ -- [ ] 1.2 Expose policy-governed severity resolution +- [x] 1.2 Expose policy-governed severity resolution - Resolve a finding's severity exclusively from the Policy Store keyed on the rule's CWE, discarding any severity string carried in legacy rule data. - Guarantee that two findings whose rules map to the same CWE receive identical severity. - Observable completion: a resolution call returns the policy severity (never a legacy string) and is identical for two rules sharing a CWE, proven by a unit test. - _Requirements: 2.2, 2.3_ - _Boundary: PolicyStore_ -- [ ] 1.3 Enforce the fail-loud startup CWE-resolution invariant +- [x] 1.3 Enforce the fail-loud startup CWE-resolution invariant - Assert at startup that every enabled rule's CWE resolves in the Policy Store; abort the scan loudly before any work when an enabled rule's CWE is unmapped. - Observable completion: an enabled rule with an unmapped CWE aborts the scan with a loud error and no findings are produced, proven by an integration test. - _Requirements: 2.7_ - _Boundary: PolicyStore_ - _Depends: 1.1_ -- [ ] 2. Build the project Score Model and CI gates (zero-dep) -- [ ] 2.1 Compute the 0-100 policy-weighted, reachability-adjusted project score +- [x] 2. Build the project Score Model and CI gates (zero-dep) +- [x] 2.1 Compute the 0-100 policy-weighted, reachability-adjusted project score - Compute a single 0-100 project score from policy-governed severity weights multiplied by a reachability multiplier over reported findings, using the standard library only. - Key the reachability multipliers exactly on the three emitted reachability values (`reachable`, `inferred-file-surface`, `unknown`) without redefining the vocabulary. - Assign a zero score penalty to findings whose CWE is dynamic-only or unmapped, reporting them as out-of-scope or unmapped instead of scoring them. @@ -36,14 +36,14 @@ This plan splits the fused regex layer into three governed owners (rules detect, - _Boundary: ScoreModel_ - _Depends: 1.1_ -- [ ] 2.2 Emit the scan-level score artifact and manifest score block +- [x] 2.2 Emit the scan-level score artifact and manifest score block - Emit a score artifact containing the project score, the maximum severity, the total penalty, a by-category breakdown, the out-of-scope dynamic-only count, the unmapped-CWE list, and the gate verdict; include the score block in the scan manifest. - Observable completion: a scan writes a score artifact with every required field and merges the score block into the manifest, verified against the documented artifact schema. - _Requirements: 5.5_ - _Boundary: ScoreModel_ - _Depends: 2.1_ -- [ ] 2.3 Implement the two-condition CI score gate (advisory-first) +- [x] 2.3 Implement the two-condition CI score gate (advisory-first) - Fail the build when any reported finding maps to policy severity 5 and has reachability `reachable`. - When configured as blocking, fail the build when the project score is below the minimum; until the score constants are calibrated against the corpus, compute and report the score advisory-first without blocking. - Observable completion: the hard gate fails on a severity-5 reachable finding while the score gate reports advisory-only by default, proven by gate tests. @@ -51,7 +51,7 @@ This plan splits the fused regex layer into three governed owners (rules detect, - _Boundary: CI Score Gate_ - _Depends: 2.1_ -- [ ] 2.4 Centralize ruleset/policy/score configuration +- [x] 2.4 Centralize ruleset/policy/score configuration - Add configuration entries for the ruleset, policy, and score data locations plus the gate thresholds and emission floors used by later phases. - Observable completion: the scan reads the score constants and emission thresholds from configuration rather than hardcoded literals, confirmed by a config-driven test. - _Requirements: 5.8_ diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index e3d5a61..80b99ad 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -25,15 +25,17 @@ from .calibration import ( write_ranking_calibrations, ) from .config import load_config -from .findings import StaticFinding, quick_scan_findings, write_findings +from .findings import PATTERN_RULES, StaticFinding, quick_scan_findings, write_findings from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config from .hunter import run_hunter_pool, write_hunter_trajectories from .index import build_code_chunks from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints +from .policy import assert_rules_resolve, load_policy from .preprocess import preprocess_repository, write_preprocess_artifact from .rank import rank_targets, write_rankings from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from .run import ScanRun, create_scan_run +from .scoring import build_score_artifact from .verification import VerificationResult, verify_findings, write_verification_results CALIBRATION_DIR = ".openultrasast/calibration" @@ -101,6 +103,8 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut ) write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json") runtime.start(mode=mode, target=run.target) + policy = runtime.run_stage("policy_load", load_policy) + runtime.run_stage("policy_check", lambda: assert_rules_resolve(PATTERN_RULES, policy)) static_hints = runtime.run_stage("static_mapping", lambda: _load_static_hints(config.static_analysis.sarif_paths)) write_static_hints(static_hints, run.root / "mapping" / "static_hints.json") snapshot, targets = runtime.run_stage( @@ -140,6 +144,22 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut "record_calibration", lambda: _persist_calibration_feedback(run, ledger_path, prior_learnings, findings, verifications), ) + cwe_by_rule = {rule.rule_id: rule.cwe for rule in PATTERN_RULES} + rule_cwe_by_id = {finding.finding_id: cwe_by_rule.get(finding.finding_id.split(":", 1)[0], "") for finding in findings} + score_artifact = runtime.run_stage( + "score", + lambda: build_score_artifact( + findings, + rule_cwe_by_id, + policy, + k=config.score.k, + min_score=config.score.min_score, + block_severity_reachable=config.score.block_severity_reachable, + blocking=config.score.blocking, + ), + ) + score_path = run.root / "score.json" + score_path.write_text(json.dumps(score_artifact.to_dict(), indent=2, sort_keys=True) + "\n") runtime.run_stage("report", lambda: write_markdown_report(findings, markdown_path, verifications)) runtime.run_stage("sarif", lambda: write_sarif_report(findings, verifications, sarif_path)) runtime.run_stage( @@ -153,9 +173,11 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut verification=verification_path, markdown=markdown_path, sarif=sarif_path, + score=score_path, trajectories=trajectories_path if trajectories else None, ), path=manifest_path, + score=score_artifact.to_dict(), ), ) runtime.finish(status="succeeded") diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index b2886f9..6c72d30 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -50,6 +50,14 @@ class StaticAnalysisConfig: sarif_paths: tuple[str, ...] = () +@dataclass(frozen=True) +class ScoreConfig: + k: float = 60.0 + min_score: int = 80 + block_severity_reachable: int = 5 + blocking: bool = False + + @dataclass(frozen=True) class ResolvedConfig: models: ModelConfig = ModelConfig() @@ -58,6 +66,7 @@ class ResolvedConfig: dynamic: DynamicConfig = DynamicConfig() evidence: EvidenceConfig = EvidenceConfig() static_analysis: StaticAnalysisConfig = StaticAnalysisConfig() + score: ScoreConfig = ScoreConfig() runs_dir: str = ".openultrasast/runs" @@ -74,6 +83,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig: dynamic=_load_dynamic(data.get("dynamic", {})), evidence=_load_evidence(data.get("evidence", {})), static_analysis=_load_static_analysis(data.get("static_analysis", {})), + score=_load_score(data.get("score", {})), runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"), ) @@ -148,3 +158,26 @@ def _load_static_analysis(value: object) -> StaticAnalysisConfig: if not isinstance(paths, list): paths = [] return StaticAnalysisConfig(sarif_paths=tuple(str(item) for item in paths)) + + +def _load_score(value: object) -> ScoreConfig: + data = _section(value) + return ScoreConfig( + k=_float_value(data.get("k"), 60.0), + min_score=_int_value(data.get("min_score"), 80), + block_severity_reachable=_int_value(data.get("block_severity_reachable"), 5), + blocking=bool(data.get("blocking", False)), + ) + + +def _float_value(value: object, default: float) -> float: + if isinstance(value, bool): + return default + if isinstance(value, int | float): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return default + return default diff --git a/src/openultrasast/findings.py b/src/openultrasast/findings.py index f180c3c..a4db163 100644 --- a/src/openultrasast/findings.py +++ b/src/openultrasast/findings.py @@ -52,7 +52,7 @@ PATTERN_RULES = ( severity="high", tags=("memory_unsafe",), pattern=re.compile(r"\b(gets|strcpy|strcat|sprintf)\s*\("), - cwe="CWE-120", + cwe="CWE-121", languages=C_LANGUAGES, ), PatternRule( @@ -61,7 +61,7 @@ PATTERN_RULES = ( severity="high", tags=("memory_unsafe",), pattern=re.compile(r"\b(memcpy|memmove|bcopy)\s*\("), - cwe="CWE-120", + cwe="CWE-121", languages=C_LANGUAGES, ), PatternRule( @@ -81,7 +81,7 @@ PATTERN_RULES = ( severity="high", tags=("memory_unsafe",), pattern=re.compile(r"\b(scanf|fscanf|sscanf)\s*\([^\n]*%s"), - cwe="CWE-120", + cwe="CWE-121", languages=C_LANGUAGES, ), PatternRule( diff --git a/src/openultrasast/policy/CWE_Score.tsv b/src/openultrasast/policy/CWE_Score.tsv new file mode 100644 index 0000000..131d2f8 --- /dev/null +++ b/src/openultrasast/policy/CWE_Score.tsv @@ -0,0 +1,168 @@ +Flaw Category CWE ID CWE Name Flaw Severity Static Dynamic +API Abuse 234 Failure to Handle Missing Parameter 3 X +API Abuse 243 Creation of Chroot Jail Without Changing Working Directory 4 X +API Abuse 245 J2EE Bad Practices: Direct Management of Connections 2 X +API Abuse 560 Use of Umask() with Chmod-Style Argument 3 X +API Abuse 628 Function Call with Incorrectly Specified Arguments 2 X +API Abuse 675 Duplicate Operations on Resource 2 X +Authentication Issues 287 Improper Authentication 4 X X +Authentication Issues 352 Cross-Site Request Forgery (CSRF) 3 X X +Authentication Issues 693 Protection Mechanism Failure 3 X X +Authorization Issues 99 Improper Control of Resource Identifiers 3 X +Authorization Issues 272 Least Privilege Violation 3 X +Authorization Issues 273 Improper Check for Dropped Privileges 3 X +Authorization Issues 274 Improper Handling of Insufficient Privileges 0 X +Authorization Issues 282 Improper Ownership Management 3 X +Authorization Issues 285 Improper Authorization 3 X X +Authorization Issues 346 Origin Validation Error 3 X +Authorization Issues 350 Reliance on Reverse DNS Resolution for a Security-Critical Action 3 X +Authorization Issues 639 Authorization Bypass Through User-Controlled Key 4 X +Authorization Issues 566 Authorization Bypass Through User-Controlled SQL Primary Key 3 X +Authorization Issues 708 Incorrect Ownership Assignment 4 X +Authorization Issues 732 Incorrect Permission Assignment for Critical Resource 3 X +Authorization Issues 942 Permissive Cross-domain Policy with Untrusted Domains 3 X X +Buffer Management Errors 118 Improper Access of Indexable Resource (Range Error) 3 X +Buffer Management Errors 125 Out-of-Bounds Read 3 X +Buffer Management Errors 129 Improper Validation of Array Index 3 X +Buffer Management Errors 135 Incorrect Calculation of Multi-Byte String Length 5 X +Buffer Management Errors 170 Improper Null Termination 3 X +Buffer Management Errors 193 Off-by-One Error 3 X +Buffer Management Errors 787 Out-of-Bounds Write 3 X +Buffer Management Errors 823 Use of Out-of-Range Pointer Offset 3 X +Buffer Management Errors 824 Access of Uninitialized Pointer 3 X +Buffer Overflow 121 Stack-Based Buffer Overflow 5 X +Code Injection 74 Improper Neutralization of Special Elements in Output Used by a Downstream Component (Injection) 4 X +Code Injection 91 XML Injection (Blind XPath Injection) 3 X X +Code Injection 94 Improper Control of Generation of Code 3 X +Code Injection 95 Improper Neutralization of Directives in Dynamically Evaluated Code (‘Eval Injection’) 5 X X +Code Injection 98 Improper Control of Filename for Include/Require Statement in PHP Program (PHP File Inclusion) 4 X X +Code Injection 185 Incorrect Regular Expression 2 X +Code Injection 830 Inclusion of Web Functionality from an Untrusted Source 2 X +Code Quality 111 Direct Use of Unsafe JNI 4 X +Code Quality 159 Failure to Sanitize Special Element 0 X +Code Quality 401 Improper Release of Memory Before Removing Last Reference (Memory Leak) 2 X +Code Quality 404 Improper Resource Shutdown or Release 0 X +Code Quality 415 Double Free 3 X +Code Quality 416 Use After Free 2 X +Code Quality 477 Use of Obsolete Functions 0 X X +Code Quality 479 Signal Handler Use of a Non-Reentrant Function 3 X +Code Quality 489 Leftover Debug Code 3 X +Code Quality 597 Use of Wrong Operator in String Comparison 2 X +Command or Argument Injection 77 Improper Neutralization of Special Elements used in a Command (Command Injection) 5 X +Command or Argument Injection 78 Improper Neutralization of Special Elements used in an OS Command (OS Command Injection) 5 X X +Command or Argument Injection 88 Argument Injection or Modification 3 X +Credentials Management 256 Plaintext Storage of a Password 3 X +Credentials Management 259 Use of Hard-coded Password 3 X X +Credentials Management 522 Insufficiently Protected Credentials 3 X X +Credentials Management 798 Use of Hard-code Credentials 3 X +CRLF Injection 93 Improper Neutralization of CRLF Sequences (CRLF Injection) 3 X +CRLF Injection 113 Improper Neutralization of CRLF Sequences in HTTP Headers (HTTP Response Splitting) 3 X X +CRLF Injection 117 Improper Output Neutralization for Logs 3 X +Cross-Site Scripting (XSS) 79 Improper Neutralization of Input During Web Page Generation (Cross-site Scripting) 3 X X +Cross-Site Scripting (XSS) 80 Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) 3 X X +Cross-Site Scripting (XSS) 83 Improper Neutralization of Script in Attributes in a Web Page 3 X X +Cross-Site Scripting (XSS) 86 Improper Neutralization of Invalid Characters in Identifiers in Web Pages 3 X +Cryptographic Issues 261 Weak Cryptography for Passwords 3 X +Cryptographic Issues 295 Improper Certificate Validation 3 X +Cryptographic Issues 296 Improper Following of Chain of Trust for Certificate Validation 3 X +Cryptographic Issues 297 Improper Validation of Host-specific Certificate Data 3 X X +Cryptographic Issues 298 Improper Validation of Certificate Expiration 3 X +Cryptographic Issues 299 Improper Check for Certificate Revocation 3 X +Cryptographic Issues 311 Missing Encryption of Sensitive Data 3 X +Cryptographic Issues 312 Cleartext Storage of Sensitive Information 3 X +Cryptographic Issues 313 Plaintext Storage in a File or on Disk 3 X +Cryptographic Issues 316 Plaintext Storage in Memory 3 X +Cryptographic Issues 319 Cleartext Transmission of Sensitive Information 3 X +Cryptographic Issues 321 Use of Hard-coded Cryptographic Key 3 X X +Cryptographic Issues 326 Inadequate Encryption Strength 3 X X +Cryptographic Issues 327 Use of a Broken or Risky Cryptographic Algorithm 3 X X +Cryptographic Issues 328 Reversible One-Way Hash 3 X +Cryptographic Issues 329 Not Using a Random IV with CBC Mode 2 X +Cryptographic Issues 330 Use of Insufficiently Random Values 3 X +Cryptographic Issues 331 Insufficient Entropy 3 X +Cryptographic Issues 338 Use of Cryptographically Weak Pseudo-Random Number Generator 3 X +Cryptographic Issues 347 Improper Verification of Cryptographic Signature 2 X +Cryptographic Issues 354 Improper Validation of Integrity Check Value 3 X +Cryptographic Issues 547 Use of Hard-coded, Security-relevant Constants 3 X +Cryptographic Issues 614 Sensitive Cookie in HTTPS Session Without Secure Attribute 2 X X +Cryptographic Issues 760 Use of a One-Way Hash with a Predictable Salt 3 X +Cryptographic Issues 780 Use of RSA with Optimal Asymmetric Encryption Padding 3 X +Cryptographic Issues 916 Use of Password Hash With Insufficient Computational Effort 3 X +Dangerous Functions 242 Use of Inherently Dangerous Function 5 X +Dangerous Functions 676 Use of Potentially Dangerous Function 3 X +Deployment Configuration 402 Transmission of Private Resources into a New Sphere (Resource Leak) 3 X +Deployment Configuration 668 Exposure of Resource to Wrong Sphere 3 X X +Deployment Configuration 926 Improper Export of Android Application Components 3 X +Directory Traversal 22 Improper Limitation of a Pathname to a Restricted Directory (Path Traversal) 3 X X +Directory Traversal 35 Path Traversal 2 X +Directory Traversal 73 External Control of File Name or Path 3 X +Encapsulation 494 Download of Code Without Integrity Check 5 X +Encapsulation 501 Trust Boundary Violation 3 X +Encapsulation 502 Deserialization of Untrusted Data 3 X +Encapsulation 749 Exposed Dangerous Method or Function 4 X +Error Handling 248 Uncaught Exception 2 X +Error Handling 252 Unchecked Return Value 2 X +Format String 134 Use of Externally-Controlled Format String 5 X +Information Leakage 200 Information Exposure 2 X +Information Leakage 201 Insertion of Sensitive Information Into Sent Data 2 X +Information Leakage 209 Information Exposure Through an Error Message 2 X X +Information Leakage 215 Information Exposure Through Debug Information 2 X X +Information Leakage 359 Exposure of Private Information (Privacy Violation) 2 X +Information Leakage 497 Exposure of System Data to an Unauthorized Control Sphere 2 X +Information Leakage 526 Information Exposure Through Environmental Variables 2 X +Information Leakage 530 Exposure of Backup File to an Unauthorized Control Sphere 2 X +Information Leakage 532 Insertion of Sensitive Information into Log File 2 X +Information Leakage 538 File and Directory Information Exposure 0 X +Information Leakage 548 Information Exposure Through Directory Listing 2 X +Information Leakage 611 Information Exposure Through XML External Entity Reference 3 X X +Information Leakage 615 Information Exposure Through Comments 0 X X +Information Leakage 665 Improper Initialization 2 X +Information Leakage 918 Server-side Request Forgery 3 X X +Insecure Dependencies 829 Inclusion of Functionality from Untrusted Control Sphere 3 X X +Insufficient Input Validation 20 Improper Input Validation 0 X +Insufficient Input Validation 90 Improper Neutralization of Special Elements Used in an LDAP Query (LDAP Injection) 3 X +Insufficient Input Validation 103 Struts: Incomplete validate() Method Definition 3 X +Insufficient Input Validation 104 Struts: Form Bean Does Not Extend Validation Class 3 X +Insufficient Input Validation 112 Missing XML Validation 3 X +Insufficient Input Validation 183 Permissive List of Allowed Inputs 3 X +Insufficient Input Validation 345 Insufficient Verification of Data Authenticity 4 X +Insufficient Input Validation 434 Unrestricted Upload of File with Dangerous Type 4 X +Insufficient Input Validation 470 Use of Externally-Controlled Input to Select Classes or Code (Unsafe Reflection) 3 X +Insufficient Input Validation 472 External Control of Assumed-Immutable Web Parameter 3 X +Insufficient Input Validation 601 URL Redirection to Untrusted Site (Open Redirect) 3 X X +Insufficient Input Validation 618 Exposed Unsafe ActiveX Method 5 X +Insufficient Input Validation 915 Improperly Controlled Modification of Dynamically-Determined Object Attributes 3 X +Insufficient Input Validation 1174 ASP.NET Misconfiguration: Improper Model Validation 2 X +Insufficient Input Validation 1236 Improper Neutralization of Formula Elements in a CSV File 3 X +Insufficient Logging & Monitoring 223 Omission of Security-relevant Information 2 X +Numeric Errors 190 Integer Overflow or Wraparound 5 X +Numeric Errors 191 Integer Underflow (Wrap or Wraparound) 3 X +Numeric Errors 192 Integer Coercion Error 3 X +Numeric Errors 195 Signed to Unsigned Conversion Error 3 X +Numeric Errors 196 Unsigned to Signed Conversion Error 3 X +Numeric Errors 197 Numeric Truncation Error 3 X +Potential Backdoor 398 Indicator of Poor Code Quality 0 X +Potential Backdoor 506 Embedded Malicious Code 4 X +Potential Backdoor 511 Logic/Time Bomb 5 X +Potential Backdoor 514 Covert Channel 2 X +Potential Backdoor 656 Reliance on Security Through Obscurity 0 X +Race Conditions 366 Race Condition within a Thread 3 X +Race Conditions 367 Time-of-check Time-of-use (TOCTOU) Race Condition 3 X +Race Conditions 421 Race Condition During Access to Alternate Channel 3 X +Server Configuration 16 Configuration 0 X +Server Configuration 441 Unintended Proxy or Intermediary (Confused Deputy) 3 X +Server Configuration 642 External Control of Critical State Data 2 X +Server Configuration 757 Selection of Less-Secure Algorithm During Negotiation (Algorithm Downgrade) 3 X X +Session Fixation 384 Session Fixation 3 X X +SQL Injection 89 Improper Neutralization of Special Elements used in an SQL Command (SQL Injection) 4 X +SQL Injection 564 SQL Injection: Hibernate 4 X +SQL Injection 943 Improper Neutralization of Special Elements in Data Query Logic 4 X +Time and State 377 Insecure Temporary File 3 X +Time and State 382 J2EE Bad Practices: Use of System.exit() 2 X +Time and State 557 Concurrency Issues 2 X +Time and State 691 Insufficient Control Flow Management 0 X +Untrusted Initialization 15 External Control of System or Configuration Setting 4 X +Untrusted Initialization 454 External Initialization of Trusted Variables or Data Stores 0 X +Untrusted Search Path 114 Process Control 5 X +Untrusted Search Path 426 Untrusted Search Path 3 X +Untrusted Search Path 427 Uncontrolled Search Path Element 3 X diff --git a/src/openultrasast/policy/__init__.py b/src/openultrasast/policy/__init__.py new file mode 100644 index 0000000..95edb55 --- /dev/null +++ b/src/openultrasast/policy/__init__.py @@ -0,0 +1,19 @@ +"""Central CWE severity/scope policy (vendored verycode-policies).""" + +from .verycode import ( + CWE_SCORE_TSV, + CwePolicy, + PolicyError, + assert_rules_resolve, + load_policy, + resolve_severity, +) + +__all__ = [ + "CWE_SCORE_TSV", + "CwePolicy", + "PolicyError", + "assert_rules_resolve", + "load_policy", + "resolve_severity", +] diff --git a/src/openultrasast/policy/verycode.py b/src/openultrasast/policy/verycode.py new file mode 100644 index 0000000..d8ba264 --- /dev/null +++ b/src/openultrasast/policy/verycode.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import csv +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path + +CWE_SCORE_TSV = Path(__file__).with_name("CWE_Score.tsv") + + +class PolicyError(ValueError): + """Raised when an enabled rule references a CWE the policy does not govern.""" + + +@dataclass(frozen=True) +class CwePolicy: + """Authoritative governance for one CWE. + + ``severity`` (0-5) is upstream-governed and overrides any rule-local string. + ``static`` gates SAST scope; ``dynamic``-only CWEs are report-only for a + static tool and are never scored. + """ + + flaw_category: str + severity: int + static: bool + dynamic: bool + + +def load_policy(tsv: Path = CWE_SCORE_TSV) -> dict[str, CwePolicy]: + """Load the vendored CWE policy keyed by ``"CWE-NNN"``. + + The source TSV uses CRLF line endings and a ``"Flaw Severity "`` header with a + trailing space, so columns are read positionally and every cell is stripped. + """ + policy: dict[str, CwePolicy] = {} + with tsv.open(newline="", encoding="utf-8") as handle: + reader = csv.reader(handle, delimiter="\t") + rows = list(reader) + for row in rows[1:]: + if len(row) < 5: + continue + cwe_id = row[1].strip() + if not cwe_id.isdigit(): + continue + severity = _parse_severity(row[3]) + policy[f"CWE-{cwe_id}"] = CwePolicy( + flaw_category=row[0].strip(), + severity=severity, + static=_flag(row[4]), + dynamic=_flag(row[5]) if len(row) > 5 else False, + ) + return policy + + +def resolve_severity( + policy: Mapping[str, CwePolicy], + cwe: str, + target: object = None, + ranking: object = None, +) -> int: + """Resolve a finding's severity exclusively from policy, keyed on its CWE. + + Policy always wins; any legacy rule-local severity string is discarded. An + unmapped or non-static CWE resolves to 0 (report-only, not scored). + """ + pol = policy.get(cwe) + if pol is None or not pol.static: + return 0 + return pol.severity + + +def assert_rules_resolve(rules: Iterable[object], policy: Mapping[str, CwePolicy]) -> None: + """Fail loud if any enabled rule's CWE is unmapped in the policy. + + Rules expose ``cwe`` and (optionally) ``status``; only ``enabled`` rules are + checked. Raises :class:`PolicyError` before any scan work proceeds. + """ + unmapped: list[str] = [] + for rule in rules: + if getattr(rule, "status", "enabled") != "enabled": + continue + cwe = getattr(rule, "cwe", None) + if not isinstance(cwe, str) or cwe not in policy: + rule_id = getattr(rule, "rule_id", "") + unmapped.append(f"{rule_id} -> {cwe}") + if unmapped: + raise PolicyError("enabled rules reference CWEs absent from the policy: " + ", ".join(sorted(unmapped))) + + +def _parse_severity(value: str) -> int: + try: + severity = int(value.strip()) + except ValueError: + return 0 + return min(5, max(0, severity)) + + +def _flag(value: str) -> bool: + return value.strip().upper() == "X" diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index 649f367..c5e8f30 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -71,10 +71,11 @@ def write_manifest( verifications: list[VerificationResult], artifact_paths: dict[str, Path], path: Path, + score: dict[str, object] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) - payload = { + payload: dict[str, object] = { "scan_id": run.scan_id, "target": str(run.target), "artifacts": {name: _relative_artifact(run.root, artifact) for name, artifact in sorted(artifact_paths.items())}, @@ -93,6 +94,8 @@ def write_manifest( for finding in findings ], } + if score is not None: + payload["score"] = score path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") diff --git a/src/openultrasast/scoring/__init__.py b/src/openultrasast/scoring/__init__.py new file mode 100644 index 0000000..2e5bd89 --- /dev/null +++ b/src/openultrasast/scoring/__init__.py @@ -0,0 +1,21 @@ +"""Policy-weighted project scoring (verycode-style AppSec score).""" + +from .project_score import ( + REACH_MULT, + SEV_WEIGHT, + ScoreArtifact, + build_score_artifact, + finding_penalty, + gate, + project_score, +) + +__all__ = [ + "REACH_MULT", + "SEV_WEIGHT", + "ScoreArtifact", + "build_score_artifact", + "finding_penalty", + "gate", + "project_score", +] diff --git a/src/openultrasast/scoring/project_score.py b/src/openultrasast/scoring/project_score.py new file mode 100644 index 0000000..e07d13c --- /dev/null +++ b/src/openultrasast/scoring/project_score.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass + +from ..findings import StaticFinding +from ..policy import CwePolicy + +# Penalty weight per policy severity (0-5). Higher severity costs the score more. +SEV_WEIGHT = {5: 50, 4: 25, 3: 10, 2: 2, 1: 1, 0: 0} + +# Reachability multiplier — the false-positive calibration knob. A confirmed FP +# lowers a finding's effective reachability rather than deleting the rule. +REACH_MULT = {"reachable": 1.0, "inferred-file-surface": 0.6, "unknown": 0.4} + + +@dataclass(frozen=True) +class ScoreArtifact: + project_score: int + max_severity: int + penalty_total: float + by_category: dict[str, int] + out_of_scope_dynamic_only: int + unmapped_cwe: list[str] + gate: dict[str, object] + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +def finding_penalty(finding: StaticFinding, rule_cwe: str | None, policy: Mapping[str, CwePolicy]) -> float: + """Penalty contributed by one finding: severity weight × reachability multiplier. + + Dynamic-only or unmapped CWEs contribute zero (report-only, never scored). + """ + pol = policy.get(rule_cwe) if rule_cwe else None + if pol is None or not pol.static: + return 0.0 + return SEV_WEIGHT[pol.severity] * REACH_MULT.get(finding.reachability_status, REACH_MULT["unknown"]) + + +def project_score( + findings: list[StaticFinding], + rule_cwe_by_id: Mapping[str, str], + policy: Mapping[str, CwePolicy], + k: float = 60.0, +) -> int: + """Compute the 0-100 project score (shadow findings excluded by the caller). + + Exponential decay of total penalty: no penalty → 100; one severity-5 reachable + finding → ~43. + """ + total = sum(finding_penalty(finding, rule_cwe_by_id.get(finding.finding_id), policy) for finding in findings) + return round(100 * math.exp(-total / k)) + + +def gate( + findings: list[StaticFinding], + score: int, + rule_cwe_by_id: Mapping[str, str], + policy: Mapping[str, CwePolicy], + *, + min_score: int, + block_severity_reachable: int = 5, + blocking: bool, +) -> dict[str, object]: + """Two-condition gate verdict. + + A finding at ``block_severity_reachable`` severity that is ``reachable`` always + fails. The score threshold fails only when ``blocking`` is enabled. + """ + hard_fail = any( + _severity(rule_cwe_by_id.get(finding.finding_id), policy) >= block_severity_reachable and finding.reachability_status == "reachable" + for finding in findings + ) + score_fail = blocking and score < min_score + return { + "min_score": min_score, + "block_severity_reachable": block_severity_reachable, + "blocking": blocking, + "passed": not hard_fail and not score_fail, + } + + +def build_score_artifact( + findings: list[StaticFinding], + rule_cwe_by_id: Mapping[str, str], + policy: Mapping[str, CwePolicy], + *, + k: float = 60.0, + min_score: int = 80, + block_severity_reachable: int = 5, + blocking: bool = False, +) -> ScoreArtifact: + score = project_score(findings, rule_cwe_by_id, policy, k=k) + penalty_total = 0.0 + max_severity = 0 + by_category: dict[str, int] = {} + out_of_scope_dynamic_only = 0 + unmapped: set[str] = set() + for finding in findings: + cwe = rule_cwe_by_id.get(finding.finding_id) + pol = policy.get(cwe) if cwe else None + if pol is None: + unmapped.add(cwe or "unknown") + continue + if not pol.static: + if pol.dynamic: + out_of_scope_dynamic_only += 1 + continue + penalty_total += finding_penalty(finding, cwe, policy) + max_severity = max(max_severity, pol.severity) + by_category[pol.flaw_category] = by_category.get(pol.flaw_category, 0) + 1 + return ScoreArtifact( + project_score=score, + max_severity=max_severity, + penalty_total=round(penalty_total, 2), + by_category=dict(sorted(by_category.items())), + out_of_scope_dynamic_only=out_of_scope_dynamic_only, + unmapped_cwe=sorted(unmapped), + gate=gate( + findings, + score, + rule_cwe_by_id, + policy, + min_score=min_score, + block_severity_reachable=block_severity_reachable, + blocking=blocking, + ), + ) + + +def _severity(cwe: str | None, policy: Mapping[str, CwePolicy]) -> int: + pol = policy.get(cwe) if cwe else None + if pol is None or not pol.static: + return 0 + return pol.severity diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100644 index 0000000..9415226 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,47 @@ +from types import SimpleNamespace + +import pytest + +from openultrasast.findings import PATTERN_RULES +from openultrasast.policy import ( + CwePolicy, + PolicyError, + assert_rules_resolve, + load_policy, + resolve_severity, +) + + +def test_load_policy_parses_severity_and_scope() -> None: + policy = load_policy() + + assert policy["CWE-121"] == CwePolicy("Buffer Overflow", 5, True, False) + assert policy["CWE-89"].severity == 4 + assert policy["CWE-78"].static is True + assert len(policy) > 150 # ~167 CWEs vendored from verycode-policies + + +def test_resolve_severity_is_policy_governed_and_identical_for_same_cwe() -> None: + policy = load_policy() + + # Two different rules that map to the same CWE must get the same severity. + assert resolve_severity(policy, "CWE-78") == resolve_severity(policy, "CWE-78") == 5 + # Unmapped CWE resolves to 0 (report-only, never scored). + assert resolve_severity(policy, "CWE-99999") == 0 + + +def test_builtin_rules_all_resolve_in_policy() -> None: + # The fail-loud invariant must pass for the shipped ruleset (CWE-120 remapped to CWE-121). + assert_rules_resolve(PATTERN_RULES, load_policy()) + + +def test_assert_rules_resolve_fails_loud_on_unmapped_enabled_rule() -> None: + policy = load_policy() + bad_enabled = SimpleNamespace(rule_id="bad", cwe="CWE-99999", status="enabled") + bad_disabled = SimpleNamespace(rule_id="bad-off", cwe="CWE-99999", status="disabled") + + with pytest.raises(PolicyError, match="bad"): + assert_rules_resolve([bad_enabled], policy) + + # A disabled rule with an unmapped CWE does not abort the scan. + assert_rules_resolve([bad_disabled], policy) diff --git a/tests/test_score_pipeline.py b/tests/test_score_pipeline.py new file mode 100644 index 0000000..6e2afaf --- /dev/null +++ b/tests/test_score_pipeline.py @@ -0,0 +1,41 @@ +"""Phase 1 in-pipeline wiring: project score artifact + fail-loud CWE invariant.""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openultrasast import cli +from openultrasast.cli import main +from openultrasast.policy import PolicyError + + +def test_scan_writes_score_artifact_and_manifest_block(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("@app.route('/x')\ndef view():\n return eval(request.data)\n") + monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") + + assert main(["scan", str(repo), "--mode", "quick"]) == 0 + + run_dir = sorted((repo / ".runs").iterdir())[-1] + score = json.loads((run_dir / "score.json").read_text()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + + assert isinstance(score["project_score"], int) and 0 <= score["project_score"] <= 100 + assert set(score["gate"]) == {"min_score", "block_severity_reachable", "blocking", "passed"} + assert manifest["score"] == score + assert manifest["artifacts"]["score"] == "score.json" + + +def test_scan_fails_loud_when_an_enabled_rule_cwe_is_unmapped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("x = 1\n") + monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") + unmapped = SimpleNamespace(rule_id="bogus", cwe="CWE-99999", status="enabled") + monkeypatch.setattr(cli, "PATTERN_RULES", (*cli.PATTERN_RULES, unmapped)) + + with pytest.raises(PolicyError): + main(["scan", str(repo), "--mode", "quick"]) diff --git a/tests/test_scoring.py b/tests/test_scoring.py new file mode 100644 index 0000000..5aa4c3c --- /dev/null +++ b/tests/test_scoring.py @@ -0,0 +1,73 @@ +from openultrasast.findings import StaticFinding +from openultrasast.policy import CwePolicy +from openultrasast.scoring import build_score_artifact, gate, project_score + +POLICY = { + "CWE-78": CwePolicy("Command Injection", 5, True, False), + "CWE-89": CwePolicy("SQL Injection", 4, True, False), + "CWE-999": CwePolicy("DAST Only", 5, False, True), # dynamic-only -> report, never scored +} + + +def _finding(finding_id: str, reachability: str) -> StaticFinding: + return StaticFinding( + finding_id=finding_id, + path="a.py", + title="t", + severity="high", + confidence="medium", + evidence_level="static_corroboration", + rationale="r", + line=1, + function_name=None, + reachability_status=reachability, + reachability_evidence=[], + reachability_conditions=[], + tags=[], + ranking_priority=1.0, + ) + + +def test_empty_corpus_scores_100() -> None: + assert project_score([], {}, POLICY) == 100 + + +def test_single_severity5_reachable_scores_about_43() -> None: + finding = _finding("cmd:a.py:1", "reachable") + score = project_score([finding], {"cmd:a.py:1": "CWE-78"}, POLICY) + assert score == 43 + + +def test_dynamic_only_and_unmapped_contribute_zero_penalty() -> None: + dynamic = _finding("dyn:a.py:1", "reachable") + unmapped = _finding("ump:a.py:2", "reachable") + rule_cwe = {"dyn:a.py:1": "CWE-999", "ump:a.py:2": "CWE-12345"} + + artifact = build_score_artifact([dynamic, unmapped], rule_cwe, POLICY) + + assert artifact.project_score == 100 # neither contributes penalty + assert artifact.out_of_scope_dynamic_only == 1 + assert artifact.unmapped_cwe == ["CWE-12345"] + + +def test_hard_gate_fails_on_severity5_reachable_only() -> None: + reachable = _finding("cmd:a.py:1", "reachable") + inferred = _finding("cmd:a.py:2", "inferred-file-surface") + rule_cwe = {"cmd:a.py:1": "CWE-78", "cmd:a.py:2": "CWE-78"} + + fail = gate([reachable], 90, rule_cwe, POLICY, min_score=80, blocking=False) + ok = gate([inferred], 90, rule_cwe, POLICY, min_score=80, blocking=False) + + assert fail["passed"] is False # severity-5 + reachable always fails + assert ok["passed"] is True # inferred reachability does not trip the hard gate + + +def test_score_gate_blocks_only_when_blocking_enabled() -> None: + finding = _finding("sql:a.py:1", "inferred-file-surface") + rule_cwe = {"sql:a.py:1": "CWE-89"} + + advisory = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=False) + blocking = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=True) + + assert advisory.gate["passed"] is True # advisory-first: low score does not block + assert blocking.gate["passed"] is False # blocking: score below min_score fails