70 lines
2.5 KiB
Python
70 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
from src.services.agent_nemotron_replay_preflight import (
|
|
evaluate_nemotron_external_runner_preflight,
|
|
)
|
|
from src.services.agent_nemotron_replay_sanitizer import (
|
|
contains_sensitive_context_marker,
|
|
sanitize_nemotron_request_pack_from_fixtures,
|
|
)
|
|
|
|
|
|
def test_sanitizer_removes_sensitive_context_markers_and_preflight_passes():
|
|
sanitized_fixtures, candidate_inputs, requests, report = (
|
|
sanitize_nemotron_request_pack_from_fixtures([_fixture_with_sensitive_context()])
|
|
)
|
|
|
|
assert report.valid is True
|
|
assert report.sensitive_marker_records_before == 1
|
|
assert report.sensitive_marker_records_after == 0
|
|
assert report.changed_fixture_records == 1
|
|
assert not contains_sensitive_context_marker(sanitized_fixtures[0]["incident_context"])
|
|
assert not contains_sensitive_context_marker(candidate_inputs[0]["incident_context"])
|
|
assert not contains_sensitive_context_marker(requests[0]["incident_context"])
|
|
|
|
preflight = evaluate_nemotron_external_runner_preflight(
|
|
fixtures=sanitized_fixtures,
|
|
candidate_inputs=candidate_inputs,
|
|
requests=requests,
|
|
).to_dict()
|
|
assert preflight["valid"] is True
|
|
assert preflight["sensitive_marker_records"] == 0
|
|
|
|
|
|
def test_sanitizer_preserves_evaluation_labels_for_local_grading():
|
|
sanitized_fixtures, _, _, _ = sanitize_nemotron_request_pack_from_fixtures(
|
|
[_fixture_with_sensitive_context()]
|
|
)
|
|
|
|
assert sanitized_fixtures[0]["evaluation_labels"]["verification_result"] == "success"
|
|
assert sanitized_fixtures[0]["evaluation_labels"]["expected_action_markers"] == [
|
|
"rollout restart",
|
|
"checkout",
|
|
]
|
|
|
|
|
|
def _fixture_with_sensitive_context() -> dict:
|
|
return {
|
|
"schema_version": "agent_replay_fixture_v1",
|
|
"run_id": "run",
|
|
"incident_id": "INC-1",
|
|
"incident_context": {
|
|
"alertname": "DockerContainerUnhealthy",
|
|
"severity": "P2",
|
|
"affected_services": ["checkout"],
|
|
"evidence_summary": (
|
|
"/srv/app/.secrets/admin.htpasswd=***REDACTED*** "
|
|
"PGPASSFILE=\"$pgpass\" pg_dump --no-password"
|
|
),
|
|
"metadata": {
|
|
"secret_path": "/k8s/08-google-drive-secret.yaml",
|
|
},
|
|
},
|
|
"evaluation_labels": {
|
|
"verification_result": "success",
|
|
"execution_success": True,
|
|
"expected_action_markers": ["rollout restart", "checkout"],
|
|
},
|
|
"source_metadata": {"source": "test"},
|
|
}
|