96 lines
3.0 KiB
Python
96 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Import externally produced NeMo/Nemotron replay results.
|
|
|
|
Input records must use agent_nemotron_external_result_v1. The output is
|
|
agent_candidate_replay_result_v1 JSONL ready for validate -> normalize -> grade
|
|
-> score. When a request pack is provided, the importer also proves one-to-one
|
|
alignment before writing raw candidate output.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
API_SRC = ROOT / "apps" / "api"
|
|
sys.path.insert(0, str(API_SRC))
|
|
|
|
from src.services.agent_nemotron_replay_adapter import ( # noqa: E402
|
|
import_nemotron_external_results_with_report,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Import NeMo/Nemotron external replay results."
|
|
)
|
|
parser.add_argument("--external-results", required=True, help="external result JSONL")
|
|
parser.add_argument("--requests", help="original NeMo/Nemotron request JSONL")
|
|
parser.add_argument("--output", required=True, help="candidate raw result JSONL")
|
|
parser.add_argument("--report", help="import report JSON path")
|
|
args = parser.parse_args()
|
|
|
|
results, report = import_nemotron_external_results_with_report(
|
|
_read_jsonl(Path(args.external_results)),
|
|
requests=_read_jsonl(Path(args.requests)) if args.requests else None,
|
|
)
|
|
report_payload = report.to_dict()
|
|
rendered_report = json.dumps(
|
|
report_payload,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
if args.report:
|
|
Path(args.report).write_text(rendered_report + "\n", encoding="utf-8")
|
|
if not report.valid:
|
|
if not args.report:
|
|
print(rendered_report, file=sys.stderr)
|
|
return 2
|
|
|
|
with Path(args.output).open("w", encoding="utf-8") as handle:
|
|
for result in results:
|
|
handle.write(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
handle.write("\n")
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"external_results": args.external_results,
|
|
"output": args.output,
|
|
"records": len(results),
|
|
"report": args.report,
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"adapter_mode": "real_offline_replay",
|
|
"valid": report.valid,
|
|
},
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
with path.open(encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
try:
|
|
records.append(json.loads(line))
|
|
except Exception as exc:
|
|
raise SystemExit(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
|
|
return records
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|