83 lines
2.8 KiB
Python
83 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Sanitize and regenerate a NeMo/Nemotron external replay request pack.
|
|
|
|
Input is the internal fixture JSONL. Output is a sanitized fixture JSONL,
|
|
candidate input JSONL, request JSONL, and sanitize report. This command is local
|
|
and does not call external APIs, production tools, or LLMs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
API_SRC = ROOT / "apps" / "api"
|
|
sys.path.insert(0, str(API_SRC))
|
|
|
|
from src.services.agent_nemotron_replay_sanitizer import ( # noqa: E402
|
|
sanitize_nemotron_request_pack_from_fixtures,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Sanitize and regenerate NeMo external replay request pack."
|
|
)
|
|
parser.add_argument("--fixtures", required=True, help="source fixture JSONL")
|
|
parser.add_argument("--output-fixtures", required=True, help="sanitized fixture JSONL")
|
|
parser.add_argument("--output-inputs", required=True, help="candidate input JSONL")
|
|
parser.add_argument("--output-requests", required=True, help="NeMo request JSONL")
|
|
parser.add_argument("--report", required=True, help="sanitize report JSON")
|
|
args = parser.parse_args()
|
|
|
|
sanitized_fixtures, candidate_inputs, requests, report = (
|
|
sanitize_nemotron_request_pack_from_fixtures(
|
|
_read_jsonl(Path(args.fixtures)),
|
|
)
|
|
)
|
|
_write_jsonl(Path(args.output_fixtures), sanitized_fixtures)
|
|
_write_jsonl(Path(args.output_inputs), candidate_inputs)
|
|
_write_jsonl(Path(args.output_requests), requests)
|
|
report_payload = report.to_dict()
|
|
_write_json(Path(args.report), report_payload)
|
|
print(json.dumps(report_payload, ensure_ascii=False, sort_keys=True))
|
|
return 0 if report.valid else 2
|
|
|
|
|
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
with path.open(encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
try:
|
|
records.append(json.loads(line))
|
|
except Exception as exc:
|
|
raise SystemExit(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
|
|
return records
|
|
|
|
|
|
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
for record in records:
|
|
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True))
|
|
handle.write("\n")
|
|
|
|
|
|
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.write_text(
|
|
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|