"""市場情報專業來源治理 gate。

本模組把主流市場資料採集做法轉成可審核合約：
robots/REP、sitemap、structured data、canonical URL、rate limit、
公開資料邊界、provenance、snapshot hash 與 idempotency。

API/UI 只審核操作員提供的治理摘要；不抓外站、不讀 robots/sitemap、
不開 DB、不寫檔、不掛 scheduler。
"""

from urllib.parse import urlparse

from services.market_intel.mcp_fetch_candidate_queue_writer_post_closeout_inventory_review import (
    FORBIDDEN_SECRET_KEYS,
    SAFE_SECRET_METADATA_KEYS,
    _as_dict,
    _as_list,
    _blocked_side_effects,
    _contains_forbidden_key,
    _safe_int,
    _safe_path,
    _safe_text,
)
from services.market_intel.mcp_fetch_candidate_queue_writer_run_readiness import (
    ARTIFACT_PREFIX,
)
from services.market_intel.mcp_professional_source_governance_gates import (
    CONTRACT_SCOPE,
    SOURCE_POLICY_VERSION,
    _is_public_http_url,
    build_professional_source_governance_gates,
)
from services.market_intel.mcp_professional_source_governance_sample import (
    build_sample_professional_source_governance_package,
)


_BLOCKED_SOURCE_GOVERNANCE_SIDE_EFFECT_KEYS = (
    "allow_api_database_write",
    "allow_api_execution",
    "allow_api_file_write",
    "allow_api_network_fetch",
    "allow_database_write",
    "allow_external_network",
    "allow_scheduler_attach",
    "api_fetches_robots_txt",
    "api_fetches_sitemap",
    "api_fetches_source_url",
    "api_opens_database_connection",
    "api_uses_external_network",
    "api_writes_database",
    "api_writes_file",
    "database_commit_executed",
    "database_write_executed",
    "external_network_executed",
    "fetch_executed",
    "file_written",
    "network_request_allowed",
    "payload_persisted",
    "ready_for_api_database_write",
    "real_write_allowed_by_api",
    "scheduler_attached",
    "write_database",
    "writes_executed",
    "would_write_database",
)
_RAW_PAYLOAD_KEYS = (
    "body_html",
    "full_response_body",
    "html",
    "page_body",
    "page_html",
    "raw_html",
    "raw_page_html",
    "response_body",
)


def _safe_float(value):
    try:
        return float(value or 0)
    except (TypeError, ValueError):
        return 0.0


def _contains_raw_payload(value):
    if isinstance(value, dict):
        for key, nested in value.items():
            if str(key).lower() in _RAW_PAYLOAD_KEYS and bool(nested):
                return True
            if _contains_raw_payload(nested):
                return True
    if isinstance(value, list):
        return any(_contains_raw_payload(item) for item in value)
    return False


def _blocked_source_governance_side_effects(payload):
    found = list(_blocked_side_effects(payload))

    def visit(value, path):
        if isinstance(value, dict):
            for key, item in value.items():
                normalized_key = str(key).lower()
                key_path = f"{path}.{key}" if path else key
                if (
                    normalized_key in _BLOCKED_SOURCE_GOVERNANCE_SIDE_EFFECT_KEYS
                    and bool(item)
                ):
                    found.append(key_path)
                visit(item, key_path)
        elif isinstance(value, list):
            for index, item in enumerate(value):
                visit(item, f"{path}[{index}]")

    visit(payload, "")
    return sorted(set(found))


def _normalize_host(value):
    if not value:
        return None
    parsed = urlparse(value)
    return parsed.netloc.lower() or None


def _source_summary(source):
    source = _as_dict(source)
    source_url = _safe_text(source.get("source_url"), 500)
    canonical_url = _safe_text(source.get("canonical_url"), 500)
    robots_url = _safe_text(source.get("robots_url"), 500)
    sitemap_url = _safe_text(source.get("sitemap_url"), 500)
    structured_data_types = [
        _safe_text(item, 80)
        for item in _as_list(source.get("structured_data_types"))
        if _safe_text(item, 80)
    ]
    max_requests = _safe_int(source.get("max_requests_per_run"))
    crawl_delay_seconds = _safe_float(source.get("crawl_delay_seconds"))
    evidence_artifact_path = _safe_text(source.get("evidence_artifact_path"))
    source_host = _normalize_host(source_url)
    canonical_host = _normalize_host(canonical_url)
    return {
        "platform_code": _safe_text(source.get("platform_code"), 80),
        "source_key": _safe_text(source.get("source_key"), 160),
        "source_url": source_url,
        "canonical_url": canonical_url,
        "robots_url": robots_url,
        "sitemap_url": sitemap_url,
        "lastmod_source": _safe_text(source.get("lastmod_source"), 160),
        "source_url_safe": _is_public_http_url(source_url),
        "canonical_url_safe": _is_public_http_url(canonical_url),
        "robots_url_safe": _is_public_http_url(robots_url),
        "sitemap_url_safe": _is_public_http_url(sitemap_url),
        "canonical_host_matches_source": bool(
            source_host and canonical_host and source_host == canonical_host
        ),
        "robots_policy_checked": bool(source.get("robots_policy_checked")),
        "robots_allowed": bool(source.get("robots_allowed")),
        "tos_public_page_checked": bool(source.get("tos_public_page_checked")),
        "login_required": bool(source.get("login_required")),
        "member_or_order_data": bool(source.get("member_or_order_data")),
        "cart_order_or_pii": bool(source.get("cart_order_or_pii")),
        "anti_bot_bypass_required": bool(source.get("anti_bot_bypass_required")),
        "structured_data_preferred": bool(source.get("structured_data_preferred")),
        "json_ld_first": bool(source.get("json_ld_first")),
        "dom_selector_fallback_allowed": bool(
            source.get("dom_selector_fallback_allowed")
        ),
        "structured_data_types": structured_data_types,
        "selector_version": _safe_text(source.get("selector_version"), 120),
        "crawl_delay_seconds": crawl_delay_seconds,
        "max_requests_per_run": max_requests,
        "public_cache_ttl_hours": _safe_int(source.get("public_cache_ttl_hours")),
        "evidence_artifact_path": evidence_artifact_path,
        "evidence_artifact_path_safe": _safe_path(
            evidence_artifact_path,
            prefixes=(ARTIFACT_PREFIX,),
            suffixes=(".json",),
        ),
        "provenance_required": bool(source.get("provenance_required")),
        "snapshot_hash_required": bool(source.get("snapshot_hash_required")),
        "idempotency_key_strategy": _safe_text(
            source.get("idempotency_key_strategy"), 160
        ),
    }


def _operator_confirmations(payload):
    confirmations = _as_dict(payload.get("operator_confirmations"))
    return {
        "human_reviewed_source_policy": bool(
            confirmations.get("human_reviewed_source_policy")
        ),
        "robots_and_tos_checked_by_operator": bool(
            confirmations.get("robots_and_tos_checked_by_operator")
        ),
        "public_pages_only": bool(confirmations.get("public_pages_only")),
        "no_login_or_member_data": bool(
            confirmations.get("no_login_or_member_data")
        ),
        "no_cart_order_or_pii": bool(confirmations.get("no_cart_order_or_pii")),
        "no_antibot_bypass": bool(confirmations.get("no_antibot_bypass")),
        "structured_data_first": bool(confirmations.get("structured_data_first")),
        "provenance_required": bool(confirmations.get("provenance_required")),
        "no_api_network_fetch": bool(confirmations.get("no_api_network_fetch")),
        "no_database_write": bool(confirmations.get("no_database_write")),
        "no_scheduler_attach": bool(confirmations.get("no_scheduler_attach")),
        "no_secret_payload": bool(confirmations.get("no_secret_payload")),
    }


def _governance_summary(payload):
    payload = _as_dict(payload)
    sources = [_source_summary(source) for source in _as_list(payload.get("sources"))]
    blocked_side_effects = _blocked_source_governance_side_effects(payload)
    raw_payload_submitted = _contains_raw_payload(payload)
    secret_or_token_submitted = _contains_forbidden_key(
        payload,
        FORBIDDEN_SECRET_KEYS,
        safe_keys=SAFE_SECRET_METADATA_KEYS
        | {
            "no_api_network_fetch",
            "no_secret_payload",
            "source_contract_version",
        },
    )
    return {
        "governance_id": _safe_text(payload.get("governance_id"), 160),
        "governance_scope": _safe_text(payload.get("governance_scope"), 160),
        "policy_version": _safe_text(payload.get("policy_version"), 120),
        "source_contract_version": _safe_text(
            payload.get("source_contract_version"), 120
        ),
        "sources": sources,
        "source_count": len(sources),
        "platform_count": len(
            {source["platform_code"] for source in sources if source["platform_code"]}
        ),
        "robots_checked_count": len(
            [
                source
                for source in sources
                if source["robots_policy_checked"] and source["robots_allowed"]
            ]
        ),
        "structured_data_ready_count": len(
            [
                source
                for source in sources
                if source["structured_data_preferred"] and source["json_ld_first"]
            ]
        ),
        "min_crawl_delay_seconds": min(
            [source["crawl_delay_seconds"] for source in sources] or [0.0]
        ),
        "max_requests_per_run": max(
            [source["max_requests_per_run"] for source in sources] or [0]
        ),
        "operator_confirmations": _operator_confirmations(payload),
        "raw_payload_submitted_to_api": raw_payload_submitted,
        "secret_or_token_submitted_to_api": secret_or_token_submitted,
        "blocked_side_effects": blocked_side_effects,
        "api_uses_external_network": bool(payload.get("api_uses_external_network")),
        "api_fetches_robots_txt": bool(payload.get("api_fetches_robots_txt")),
        "api_fetches_sitemap": bool(payload.get("api_fetches_sitemap")),
        "api_fetches_source_url": bool(payload.get("api_fetches_source_url")),
        "api_opens_database_connection": bool(
            payload.get("api_opens_database_connection")
        ),
        "api_writes_database": bool(payload.get("api_writes_database")),
        "api_writes_file": bool(payload.get("api_writes_file")),
        "scheduler_attached": bool(payload.get("scheduler_attached")),
    }


def _source_contract():
    return {
        "contract_scope": CONTRACT_SCOPE,
        "policy_version": SOURCE_POLICY_VERSION,
        "mainstream_practices": [
            {
                "key": "robots_exclusion_protocol",
                "label": "先人工確認 robots.txt / REP，不由 API 自動抓取或繞過",
                "reference_url": "https://www.rfc-editor.org/rfc/rfc9309",
            },
            {
                "key": "sitemap_provenance",
                "label": "以 sitemap / lastmod 作為活動來源發現與更新依據之一",
                "reference_url": "https://www.sitemaps.org/protocol.html",
            },
            {
                "key": "structured_data_first",
                "label": "優先解析 JSON-LD / schema.org Product、Offer、ItemList",
                "reference_url": "https://developers.google.com/search/docs/appearance/structured-data/product-snippet",
            },
            {
                "key": "canonical_public_url",
                "label": "所有來源需保留 canonical URL、公開 URL 與 host provenance",
                "reference_url": "https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls",
            },
            {
                "key": "bronze_silver_gold",
                "label": "raw evidence、normalized source、reviewed product/match 分層保存",
                "reference_url": "internal:market_intel_lakehouse_contract",
            },
        ],
        "required_source_fields": [
            "platform_code",
            "source_key",
            "source_url",
            "canonical_url",
            "robots_url",
            "sitemap_url",
            "robots_policy_checked",
            "robots_allowed",
            "structured_data_types",
            "crawl_delay_seconds",
            "max_requests_per_run",
            "evidence_artifact_path",
            "snapshot_hash_required",
            "idempotency_key_strategy",
        ],
        "forbidden_data": [
            "login_page",
            "member_profile",
            "cart_or_checkout",
            "order_data",
            "personal_data",
            "cookie_or_token",
            "anti_bot_bypass",
        ],
        "next_gate": "mcp_fetch_target_review_with_source_governance",
    }


def build_mcp_professional_source_governance_preview(
    *, operator_source_governance=None, phase=None
):
    payload_received = operator_source_governance is not None
    governance_payload = _as_dict(operator_source_governance)
    governance = _governance_summary(governance_payload)
    gates = build_professional_source_governance_gates(
        package_received=payload_received,
        governance=governance,
    )
    blocked_reasons = [gate["key"] for gate in gates if not gate["passed"]]
    accepted = bool(payload_received and not blocked_reasons)
    return {
        "mode": (
            "mcp_professional_source_governance"
            if accepted
            else "mcp_professional_source_governance_preview"
        ),
        "phase": phase,
        "source_governance_payload_received": payload_received,
        "mcp_professional_source_governance_accepted": accepted,
        "ready_for_mcp_fetch_source_contract": accepted,
        "ready_for_api_database_write": False,
        "ready_for_scheduler_attach": False,
        "network_request_allowed": False,
        "external_network_executed": False,
        "api_uses_external_network": False,
        "api_fetches_robots_txt": False,
        "api_fetches_sitemap": False,
        "api_fetches_source_url": False,
        "api_opens_database_connection": False,
        "api_writes_database": False,
        "api_writes_file": False,
        "database_connection_opened": False,
        "database_write_executed": False,
        "database_commit_executed": False,
        "file_written": False,
        "payload_persisted": False,
        "scheduler_attached": False,
        "gate_count": len(gates),
        "passed_gate_count": len([gate for gate in gates if gate["passed"]]),
        "blocked_reasons": blocked_reasons,
        "gates": gates,
        "source_governance_summary": governance,
        "source_contract": _source_contract(),
        "sources": governance["sources"],
        "next_operator_steps": [
            "人工保留 robots / sitemap / ToS / public URL 審核證據。",
            "將通過治理的來源餵給後續 fetch target review，不由 API 直接抓外站。",
            "正式 fetch 前仍需 MCP readiness、外部 MCP health、manual run package 與 receipt gate。",
        ],
        "sample_professional_source_governance_package": (
            build_sample_professional_source_governance_package()
        ),
    }
