210577de28
Some checks failed
Code Review / ai-code-review (push) Successful in 13s
CD Pipeline / tests (push) Successful in 1m39s
CD Pipeline / build-and-deploy (push) Successful in 4m35s
CD Pipeline / post-deploy-checks (push) Successful in 1m51s
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled
336 lines
11 KiB
Python
336 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
from email.message import Message
|
|
from urllib.error import HTTPError
|
|
|
|
from src.services import agent_market_watch
|
|
from src.services.agent_market_watch import (
|
|
FetchedSource,
|
|
fetch_url,
|
|
run_agent_market_watch,
|
|
)
|
|
|
|
|
|
def test_market_watch_detects_version_change_without_approving_replacement():
|
|
registry = {
|
|
"schema_version": "agent_market_watch_sources_v1",
|
|
"updated_at": "2026-06-02",
|
|
"cadence": {
|
|
"weekly_market_watch": "weekly",
|
|
"monthly_integration_review": "monthly",
|
|
"trigger_on_major_version": True,
|
|
},
|
|
"policy": {
|
|
"replacement_decision_allowed": False,
|
|
"integration_requires_replay": True,
|
|
"paid_provider_requires_approval": True,
|
|
"new_dependency_requires_approval": True,
|
|
},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "langgraph_incident_kernel",
|
|
"display_name": "LangGraph",
|
|
"evaluation_priority": "must_test",
|
|
"recommended_role": "workflow kernel",
|
|
"requires_cost_approval": False,
|
|
"requires_dependency_approval": True,
|
|
"sources": [
|
|
{
|
|
"source_id": "langgraph_pypi",
|
|
"type": "pypi",
|
|
"url": "https://pypi.org/pypi/langgraph/json",
|
|
"reference_version": "1.0.0",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
def fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
payload = {
|
|
"info": {"version": "1.1.0"},
|
|
"releases": {
|
|
"1.1.0": [{"upload_time_iso_8601": "2026-06-02T01:02:03Z"}]
|
|
},
|
|
}
|
|
return FetchedSource(status="ok", http_status=200, body=json.dumps(payload).encode())
|
|
|
|
report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="live",
|
|
fetcher=fetcher,
|
|
generated_at="2026-06-02T00:00:00+00:00",
|
|
)
|
|
|
|
assert report["summary"]["changed_candidates"] == 1
|
|
assert report["summary"]["integration_queue_count"] == 1
|
|
assert report["policy"]["replacement_decision_allowed"] is False
|
|
candidate = report["candidates"][0]
|
|
assert candidate["changed"] is True
|
|
assert candidate["decision"] == "changed_requires_replay_readiness_review"
|
|
assert "run_offline_replay_before_shadow" in candidate["recommended_actions"]
|
|
assert report["integration_queue"][0]["required_next_gate"] == (
|
|
"refresh_market_scorecard_then_offline_replay"
|
|
)
|
|
assert report["integration_queue"][0]["requires_dependency_approval"] is True
|
|
|
|
|
|
def test_market_watch_offline_mode_skips_network():
|
|
registry = {
|
|
"schema_version": "agent_market_watch_sources_v1",
|
|
"cadence": {
|
|
"weekly_market_watch": "weekly",
|
|
"monthly_integration_review": "monthly",
|
|
"trigger_on_major_version": True,
|
|
},
|
|
"policy": {
|
|
"replacement_decision_allowed": False,
|
|
"integration_requires_replay": True,
|
|
"paid_provider_requires_approval": True,
|
|
"new_dependency_requires_approval": True,
|
|
},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "openai_agents_sdk_coordinator",
|
|
"display_name": "OpenAI",
|
|
"evaluation_priority": "must_test",
|
|
"recommended_role": "coordinator",
|
|
"sources": [
|
|
{
|
|
"source_id": "openai_docs",
|
|
"type": "docs",
|
|
"url": "https://example.invalid",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
def fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
raise AssertionError("offline mode must not fetch")
|
|
|
|
report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="offline",
|
|
fetcher=fetcher,
|
|
generated_at="2026-06-02T00:00:00+00:00",
|
|
)
|
|
|
|
assert report["summary"]["changed_candidates"] == 0
|
|
assert report["summary"]["integration_queue_count"] == 0
|
|
assert report["candidates"][0]["sources"][0]["status"] == "skipped_offline"
|
|
|
|
|
|
def test_fetch_url_follows_permanent_redirect(monkeypatch):
|
|
class Response:
|
|
status = 200
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self):
|
|
return b'{"ok": true}'
|
|
|
|
calls: list[str] = []
|
|
|
|
def fake_urlopen(request, timeout: int):
|
|
calls.append(request.full_url)
|
|
if request.full_url == "https://example.com/start":
|
|
headers = Message()
|
|
headers["Location"] = "/final"
|
|
raise HTTPError(
|
|
request.full_url,
|
|
308,
|
|
"Permanent Redirect",
|
|
headers,
|
|
io.BytesIO(b"redirect"),
|
|
)
|
|
assert timeout == 12
|
|
return Response()
|
|
|
|
monkeypatch.setattr(agent_market_watch, "urlopen", fake_urlopen)
|
|
|
|
fetched = fetch_url("https://example.com/start", 12)
|
|
|
|
assert fetched.status == "ok"
|
|
assert fetched.http_status == 200
|
|
assert fetched.body == b'{"ok": true}'
|
|
assert calls == ["https://example.com/start", "https://example.com/final"]
|
|
|
|
|
|
def test_docs_hash_ignores_dynamic_script_noise():
|
|
registry = {
|
|
"schema_version": "agent_market_watch_sources_v1",
|
|
"cadence": {
|
|
"weekly_market_watch": "weekly",
|
|
"monthly_integration_review": "monthly",
|
|
"trigger_on_major_version": True,
|
|
},
|
|
"policy": {
|
|
"replacement_decision_allowed": False,
|
|
"integration_requires_replay": True,
|
|
"paid_provider_requires_approval": True,
|
|
"new_dependency_requires_approval": True,
|
|
},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "docs_candidate",
|
|
"display_name": "Docs Candidate",
|
|
"sources": [
|
|
{
|
|
"source_id": "docs",
|
|
"type": "docs",
|
|
"url": "https://example.com/docs",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
bodies = [
|
|
b"<html><title>Agent Docs</title><script>nonce='one'</script><main>Stable contract text</main></html>",
|
|
b"<html><title>Agent Docs</title><script>nonce='two'</script><main>Stable contract text</main></html>",
|
|
]
|
|
|
|
def first_fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
return FetchedSource(status="ok", http_status=200, body=bodies[0])
|
|
|
|
first_report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="live",
|
|
fetcher=first_fetcher,
|
|
generated_at="2026-06-02T00:00:00+00:00",
|
|
)
|
|
|
|
def second_fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
return FetchedSource(status="ok", http_status=200, body=bodies[1])
|
|
|
|
second_report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="live",
|
|
previous_report=first_report,
|
|
fetcher=second_fetcher,
|
|
generated_at="2026-06-02T00:00:00+00:00",
|
|
)
|
|
|
|
assert second_report["summary"]["changed_candidates"] == 0
|
|
assert second_report["candidates"][0]["sources"][0]["changed_since_reference"] is False
|
|
|
|
|
|
def test_market_watch_parses_github_tags_for_projects_without_releases():
|
|
registry = {
|
|
"schema_version": "agent_market_watch_sources_v1",
|
|
"policy": {"replacement_decision_allowed": False},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "pgvector",
|
|
"display_name": "pgvector",
|
|
"evaluation_priority": "watch",
|
|
"recommended_role": "vector store",
|
|
"sources": [
|
|
{
|
|
"source_id": "pgvector_tags",
|
|
"type": "github_tags",
|
|
"url": "https://api.github.com/repos/pgvector/pgvector/tags",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
def fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
return FetchedSource(
|
|
status="ok",
|
|
http_status=200,
|
|
body=json.dumps([{"name": "v0.8.0"}]).encode(),
|
|
)
|
|
|
|
report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="live",
|
|
fetcher=fetcher,
|
|
generated_at="2026-06-25T00:00:00+00:00",
|
|
)
|
|
|
|
source = report["candidates"][0]["sources"][0]
|
|
assert source["status"] == "ok"
|
|
assert source["version"] == "v0.8.0"
|
|
assert report["summary"]["failure_count"] == 0
|
|
|
|
|
|
def test_versioned_source_ignores_metadata_hash_noise_when_version_is_unchanged():
|
|
registry = {
|
|
"schema_version": "agent_market_watch_sources_v1",
|
|
"cadence": {
|
|
"weekly_market_watch": "weekly",
|
|
"monthly_integration_review": "monthly",
|
|
"trigger_on_major_version": True,
|
|
},
|
|
"policy": {
|
|
"replacement_decision_allowed": False,
|
|
"integration_requires_replay": True,
|
|
"paid_provider_requires_approval": True,
|
|
"new_dependency_requires_approval": True,
|
|
},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "versioned_candidate",
|
|
"display_name": "Versioned Candidate",
|
|
"sources": [
|
|
{
|
|
"source_id": "pypi",
|
|
"type": "pypi",
|
|
"url": "https://example.com/pypi.json",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
previous_report = {
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "versioned_candidate",
|
|
"sources": [
|
|
{
|
|
"source_id": "pypi",
|
|
"version": "1.2.3",
|
|
"content_hash": "old-hash",
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
|
|
def fetcher(_url: str, _timeout: int) -> FetchedSource:
|
|
payload = {
|
|
"info": {"version": "1.2.3"},
|
|
"releases": {
|
|
"1.2.3": [{"upload_time_iso_8601": "2026-06-02T01:02:03Z"}],
|
|
"0.0.1": [{"upload_time_iso_8601": "2025-01-01T00:00:00Z"}],
|
|
},
|
|
"volatile_metadata": "changed package json body",
|
|
}
|
|
return FetchedSource(status="ok", http_status=200, body=json.dumps(payload).encode())
|
|
|
|
report = run_agent_market_watch(
|
|
registry,
|
|
registry_path="registry.json",
|
|
mode="live",
|
|
previous_report=previous_report,
|
|
fetcher=fetcher,
|
|
generated_at="2026-06-04T00:00:00+00:00",
|
|
)
|
|
|
|
assert report["summary"]["changed_candidates"] == 0
|
|
assert report["candidates"][0]["sources"][0]["version"] == "1.2.3"
|
|
assert report["candidates"][0]["sources"][0]["changed_since_reference"] is False
|