mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-20 13:39:27 +00:00
157 lines
16 KiB
JSON
157 lines
16 KiB
JSON
|
{
|
||
|
|
"added": [
|
||
|
|
"tests.test_config::test_retired_agentic_section_loads_with_one_warning",
|
||
|
|
"tests.test_config::test_retired_llm_keys_fail_naming_the_replacement[[fusion]\\ndecider_model = \"gpt-4o\"\\n-`[fusion] decider_model`-`agree` task]",
|
||
|
|
"tests.test_config::test_retired_llm_keys_fail_naming_the_replacement[[fusion]\\npanel_model = \"gpt-4o\"\\n-`[fusion] panel_model`-`agree` task]",
|
||
|
|
"tests.test_config::test_retired_llm_keys_fail_naming_the_replacement[[models]\\nverifier = \"gpt-4o\"\\n-`[models] verifier`-`verify` + `agree`]",
|
||
|
|
"tests.test_config::test_the_cli_exits_2_on_a_retired_key",
|
||
|
|
"tests.test_gate::test_governance_planes_import_only_stdlib_and_declared_dependencies",
|
||
|
|
"tests.test_improve_memory::test_a_reverted_memory_proposal_is_not_retried_on_the_same_evidence",
|
||
|
|
"tests.test_improve_memory::test_default_improve_output_matches_the_recorded_baseline",
|
||
|
|
"tests.test_improve_memory::test_default_round_journal_has_no_evidence_and_no_sidecar",
|
||
|
|
"tests.test_improve_memory::test_disputed_families_are_advisory_and_never_reach_the_validator",
|
||
|
|
"tests.test_improve_memory::test_guard_drops_holdout_pair_repositories_and_gated_cases",
|
||
|
|
"tests.test_improve_memory::test_guard_drops_the_qualification_population_and_reports_it",
|
||
|
|
"tests.test_improve_memory::test_improve_memory_with_an_empty_store_has_no_proposals",
|
||
|
|
"tests.test_improve_memory::test_m1_counts_fixed_pin_alerts_in_the_fix_range_and_ignores_disputes",
|
||
|
|
"tests.test_improve_memory::test_m1_demotes_a_rule_with_false_alerts_across_repositories",
|
||
|
|
"tests.test_improve_memory::test_m1_never_demotes_a_rule_with_an_agreed_hit",
|
||
|
|
"tests.test_improve_memory::test_m1_skips_rules_the_ruleset_does_not_know_or_that_are_not_enabled",
|
||
|
|
"tests.test_improve_memory::test_m1_stays_silent_below_threshold[one-repository]",
|
||
|
|
"tests.test_improve_memory::test_m1_stays_silent_below_threshold[two-alerts]",
|
||
|
|
"tests.test_improve_memory::test_m2_promotes_a_shadow_rule_that_hits_declared_sites_nothing_else_catches",
|
||
|
|
"tests.test_improve_memory::test_m2_stays_silent_below_threshold_or_when_already_caught",
|
||
|
|
"tests.test_improve_memory::test_m2_stays_silent_on_a_fixed_pin_alert_or_rejected_candidates",
|
||
|
|
"tests.test_improve_memory::test_provenance_round_trips_journal_sidecar_store",
|
||
|
|
"tests.test_plane_loop::test_alerts_attribute_functions_and_mark_the_fix_on_both_pins",
|
||
|
|
"tests.test_plane_loop::test_every_case_ends_in_remember_and_the_loop_is_model_free",
|
||
|
|
"tests.test_plane_loop::test_fix_ranges_new_side_are_the_fixed_pins_lines",
|
||
|
|
"tests.test_plane_loop::test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_costs_recall",
|
||
|
|
"tests.test_plane_memory::test_a_failing_select_on_one_object_falls_back_for_that_object",
|
||
|
|
"tests.test_plane_memory::test_a_malformed_row_fails_naming_it_and_writes_nothing[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_a_malformed_row_fails_naming_it_and_writes_nothing[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_a_malformed_row_fails_naming_it_and_writes_nothing[file]",
|
||
|
|
"tests.test_plane_memory::test_a_malformed_row_fails_naming_it_and_writes_nothing[minio]",
|
||
|
|
"tests.test_plane_memory::test_a_repeated_id_replaces_the_stored_row[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_a_repeated_id_replaces_the_stored_row[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_a_repeated_id_replaces_the_stored_row[file]",
|
||
|
|
"tests.test_plane_memory::test_a_repeated_id_replaces_the_stored_row[minio]",
|
||
|
|
"tests.test_plane_memory::test_configure_turns_versioning_on_and_expires_runs_only",
|
||
|
|
"tests.test_plane_memory::test_facts_that_no_longer_match_their_hash_are_dropped[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_facts_that_no_longer_match_their_hash_are_dropped[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_facts_that_no_longer_match_their_hash_are_dropped[file]",
|
||
|
|
"tests.test_plane_memory::test_facts_that_no_longer_match_their_hash_are_dropped[minio]",
|
||
|
|
"tests.test_plane_memory::test_file_store_refuses_below_one_gib_free_naming_its_path",
|
||
|
|
"tests.test_plane_memory::test_identical_facts_are_stored_once_and_read_back_by_content[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_identical_facts_are_stored_once_and_read_back_by_content[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_identical_facts_are_stored_once_and_read_back_by_content[file]",
|
||
|
|
"tests.test_plane_memory::test_identical_facts_are_stored_once_and_read_back_by_content[minio]",
|
||
|
|
"tests.test_plane_memory::test_ingest_is_idempotent_by_the_index[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_ingest_is_idempotent_by_the_index[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_ingest_is_idempotent_by_the_index[file]",
|
||
|
|
"tests.test_plane_memory::test_ingest_is_idempotent_by_the_index[minio]",
|
||
|
|
"tests.test_plane_memory::test_minio_settings_name_missing_keys_never_values",
|
||
|
|
"tests.test_plane_memory::test_minio_without_the_sdk_names_the_extra",
|
||
|
|
"tests.test_plane_memory::test_open_store_selects_the_backend",
|
||
|
|
"tests.test_plane_memory::test_presigned_urls_name_the_object[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_presigned_urls_name_the_object[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_presigned_urls_name_the_object[file]",
|
||
|
|
"tests.test_plane_memory::test_presigned_urls_name_the_object[minio]",
|
||
|
|
"tests.test_plane_memory::test_row_objects_carry_tags_and_a_kind_tag_skips_objects_without_that_kind",
|
||
|
|
"tests.test_plane_memory::test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version[fake-minio-fallback]",
|
||
|
|
"tests.test_plane_memory::test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version[fake-minio-select]",
|
||
|
|
"tests.test_plane_memory::test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version[file]",
|
||
|
|
"tests.test_plane_memory::test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version[minio]",
|
||
|
|
"tests.test_plane_memory::test_seed_never_overwrites_a_task_directory_and_drops_tampered_facts",
|
||
|
|
"tests.test_plane_memory::test_seed_recomputes_on_a_changed_pin_candidate_set_or_image[change0]",
|
||
|
|
"tests.test_plane_memory::test_seed_recomputes_on_a_changed_pin_candidate_set_or_image[change1]",
|
||
|
|
"tests.test_plane_memory::test_seed_recomputes_on_a_changed_pin_candidate_set_or_image[change2]",
|
||
|
|
"tests.test_plane_memory::test_seed_reuses_facts_for_an_unchanged_key_and_marks_the_task_done",
|
||
|
|
"tests.test_plane_memory::test_select_is_pushed_down_when_the_probe_answers",
|
||
|
|
"tests.test_plane_memory::test_where_sql_quotes_values_and_refuses_odd_fields",
|
||
|
|
"tests.test_plane_memory::test_without_select_the_same_rows_come_from_a_local_filter_logged_once[raises]",
|
||
|
|
"tests.test_plane_memory::test_without_select_the_same_rows_come_from_a_local_filter_logged_once[wrong]",
|
||
|
|
"tests.test_plane_reconciler::test_mark_done_sets_the_state_under_the_run_lock",
|
||
|
|
"tests.test_plane_remember::test_a_malformed_delivered_row_fails_naming_file_and_line",
|
||
|
|
"tests.test_plane_remember::test_ingest_without_remember_outputs_is_a_no_op",
|
||
|
|
"tests.test_plane_remember::test_plane_remember_cli_reads_population_from_the_run_and_reports_counts",
|
||
|
|
"tests.test_plane_remember::test_plane_run_seeds_before_and_ingests_after",
|
||
|
|
"tests.test_plane_remember::test_re_ingest_of_the_same_run_is_a_no_op",
|
||
|
|
"tests.test_plane_remember::test_remember_turns_the_recorded_case_into_rows",
|
||
|
|
"tests.test_plane_remember::test_rows_carry_every_final_the_tiebreak_and_alerts",
|
||
|
|
"tests.test_plane_remember::test_the_task_entrypoint_writes_memory_and_passes_facts_through",
|
||
|
|
"tests.test_removed_plane_references::test_no_harnessx_outside_retirement_notes",
|
||
|
|
"tests.test_removed_plane_references::test_the_readme_names_the_removed_plane_only_in_its_retirement_note",
|
||
|
|
"tests.test_removed_plane_references::test_the_retirement_notes_exist"
|
||
|
|
],
|
||
|
|
"added_from": "tasks 2-6 of harnessx-removal: test_plane_memory, test_plane_remember, test_plane_loop, test_improve_memory, test_plane_reconciler (mark_done), test_removed_plane_references, and the replacements named above",
|
||
|
|
"deleted": [
|
||
|
|
"tests.test_cli_hx_dispatch::test_degradation_recorded_when_models_set_but_extra_absent",
|
||
|
|
"tests.test_cli_hx_dispatch::test_manifest_has_no_harnessx_degradations_without_models",
|
||
|
|
"tests.test_cli_hx_dispatch::test_standard_scan_falls_back_to_run_hunter_pool",
|
||
|
|
"tests.test_config::test_harnessx_defaults_when_section_absent",
|
||
|
|
"tests.test_config::test_load_config_reads_harnessx_section",
|
||
|
|
"tests.test_fusion::test_dispatch_records_degradation_when_llm_requested_but_extra_absent",
|
||
|
|
"tests.test_gate::test_governance_scoring_benchmark_planes_never_import_harnessx",
|
||
|
|
"tests.test_gate::test_harnessx_stage_stays_within_tolerance_of_baseline",
|
||
|
|
"tests.test_gate::test_planes_have_no_module_level_harnessx_import",
|
||
|
|
"tests.test_harness_ext::test_build_provider_constructs_selected_provider[anthropic-AnthropicProvider]",
|
||
|
|
"tests.test_harness_ext::test_build_provider_constructs_selected_provider[openai-OpenAIProvider]",
|
||
|
|
"tests.test_harness_ext::test_build_provider_rejects_unknown_provider_before_importing",
|
||
|
|
"tests.test_harness_ext::test_core_dependencies_are_pyyaml_only",
|
||
|
|
"tests.test_harness_ext::test_harnessx_is_a_sha_pinned_optional_extra",
|
||
|
|
"tests.test_harness_ext::test_has_harnessx_probes_without_importing",
|
||
|
|
"tests.test_harness_ext::test_importing_openultrasast_does_not_import_harnessx",
|
||
|
|
"tests.test_harness_ext::test_require_harnessx_raises_with_install_guidance_when_absent",
|
||
|
|
"tests.test_harness_ext::test_supported_providers_cover_anthropic_and_openai",
|
||
|
|
"tests.test_hunter_harness::test_build_sast_raises_without_extra",
|
||
|
|
"tests.test_hunter_harness::test_build_sast_returns_harness_config_offline",
|
||
|
|
"tests.test_hunter_harness::test_extract_findings_parses_json_array_and_tolerates_garbage",
|
||
|
|
"tests.test_hunter_harness::test_module_import_does_not_pull_harnessx",
|
||
|
|
"tests.test_hunter_harness::test_orchestrator_builds_agent_offline",
|
||
|
|
"tests.test_hunter_harness::test_orchestrator_construction_raises_without_extra",
|
||
|
|
"tests.test_slot_contract::test_host_under_harnessx_disables_the_model",
|
||
|
|
"tests.test_verify_judge::test_dispatch_does_not_import_harnessx_on_fallback",
|
||
|
|
"tests.test_verify_judge::test_dispatch_falls_back_to_structural_by_default",
|
||
|
|
"tests.test_verify_judge::test_dispatch_stays_structural_when_model_unset_or_extra_absent",
|
||
|
|
"tests.test_verify_judge::test_verdict_mapping_is_evidence_gated"
|
||
|
|
],
|
||
|
|
"justification": {
|
||
|
|
"tests.test_cli_hx_dispatch::test_degradation_recorded_when_models_set_but_extra_absent": "file deleted: exercised only the HarnessX dispatch (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_cli_hx_dispatch::test_manifest_has_no_harnessx_degradations_without_models": "file deleted: exercised only the HarnessX dispatch (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_cli_hx_dispatch::test_standard_scan_falls_back_to_run_hunter_pool": "file deleted: exercised only the HarnessX dispatch (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_config::test_harnessx_defaults_when_section_absent": "replaced by test_retired_agentic_section_loads_with_one_warning and test_retired_llm_keys_fail_naming_the_replacement (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_config::test_load_config_reads_harnessx_section": "replaced by test_retired_agentic_section_loads_with_one_warning and test_retired_llm_keys_fail_naming_the_replacement (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_fusion::test_dispatch_records_degradation_when_llm_requested_but_extra_absent": "deleted: asserted the extra-absent degradation of the retired LLM panels (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_gate::test_governance_scoring_benchmark_planes_never_import_harnessx": "the live-extra tolerance test is deleted and the two never-import tests are replaced by test_governance_planes_import_only_stdlib_and_declared_dependencies (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_gate::test_harnessx_stage_stays_within_tolerance_of_baseline": "the live-extra tolerance test is deleted and the two never-import tests are replaced by test_governance_planes_import_only_stdlib_and_declared_dependencies (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_gate::test_planes_have_no_module_level_harnessx_import": "the live-extra tolerance test is deleted and the two never-import tests are replaced by test_governance_planes_import_only_stdlib_and_declared_dependencies (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_build_provider_constructs_selected_provider[anthropic-AnthropicProvider]": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_build_provider_constructs_selected_provider[openai-OpenAIProvider]": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_build_provider_rejects_unknown_provider_before_importing": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_core_dependencies_are_pyyaml_only": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_harnessx_is_a_sha_pinned_optional_extra": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_has_harnessx_probes_without_importing": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_importing_openultrasast_does_not_import_harnessx": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_require_harnessx_raises_with_install_guidance_when_absent": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_harness_ext::test_supported_providers_cover_anthropic_and_openai": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_build_sast_raises_without_extra": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_build_sast_returns_harness_config_offline": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_extract_findings_parses_json_array_and_tolerates_garbage": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_module_import_does_not_pull_harnessx": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_orchestrator_builds_agent_offline": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_hunter_harness::test_orchestrator_construction_raises_without_extra": "file deleted: exercised only HarnessX (design Tests table; commit b9afb66)",
|
||
|
|
"tests.test_scan_pipeline_integrity::test_quick_scan_is_unaffected_by_the_extra": "renamed to the behaviour it asserts; has_harnessx monkeypatch dropped (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_scan_pipeline_integrity::test_standard_scan_extra_absent_matches_deterministic_baseline": "renamed to the behaviour it asserts; has_harnessx monkeypatch dropped (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_slot_contract::test_host_under_harnessx_disables_the_model": "deleted with host_under_harnessx (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_verify_judge::test_dispatch_does_not_import_harnessx_on_fallback": "file deleted with verify_judge.py (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_verify_judge::test_dispatch_falls_back_to_structural_by_default": "file deleted with verify_judge.py (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_verify_judge::test_dispatch_stays_structural_when_model_unset_or_extra_absent": "file deleted with verify_judge.py (design Tests table; commit cbadc38)",
|
||
|
|
"tests.test_verify_judge::test_verdict_mapping_is_evidence_gated": "file deleted with verify_judge.py (design Tests table; commit cbadc38)"
|
||
|
|
},
|
||
|
|
"renamed": {
|
||
|
|
"tests.test_scan_pipeline_integrity::test_quick_scan_is_unaffected_by_the_extra": "tests.test_scan_pipeline_integrity::test_quick_scan_records_no_degradation",
|
||
|
|
"tests.test_scan_pipeline_integrity::test_standard_scan_extra_absent_matches_deterministic_baseline": "tests.test_scan_pipeline_integrity::test_standard_scan_with_a_hunter_model_keeps_deterministic_findings"
|
||
|
|
}
|
||
|
|
}
|