"""Bendras HTTP klientas su retry / backoff ir robots.txt tikrinimu.

Skirtas pagarbiai naudotis viešais šaltiniais:
- nustatomas User-Agent,
- automatiniai pakartojimai pagal status_code arba klaidą,
- pauzės tarp užklausų,
- robots.txt tikrinimas web scraping atvejais.
"""

from __future__ import annotations

import time
import urllib.robotparser as rp
from typing import Optional
from urllib.parse import urlparse

import requests
from tenacity import (
    retry, stop_after_attempt, wait_exponential, retry_if_exception_type, before_sleep_log,
)

from ..logger import get_logger

log = get_logger(__name__)

_robots_cache: dict[str, rp.RobotFileParser] = {}


class HttpClient:
    def __init__(self,
                 user_agent: str = "trends_collector/0.1",
                 timeout: int = 20,
                 retries: int = 3,
                 backoff_factor: float = 1.5,
                 default_sleep: float = 1.5):
        self.user_agent = user_agent
        self.timeout = timeout
        self.retries = retries
        self.backoff_factor = backoff_factor
        self.default_sleep = default_sleep
        self.session = requests.Session()
        self.session.headers.update({
            "User-Agent": user_agent,
            "Accept-Language": "en;q=0.9,*;q=0.5",
        })

    # ---- robots.txt -------------------------------------------------------
    def can_fetch(self, url: str) -> bool:
        try:
            parsed = urlparse(url)
            base = f"{parsed.scheme}://{parsed.netloc}"
            if base not in _robots_cache:
                parser = rp.RobotFileParser()
                parser.set_url(f"{base}/robots.txt")
                try:
                    parser.read()
                except Exception:
                    # jei robots nepavyko nuskaityti – nelaikom nuolaidžia,
                    # bet ir nedraudžiam, leidžiam
                    _robots_cache[base] = parser
                    return True
                _robots_cache[base] = parser
            return _robots_cache[base].can_fetch(self.user_agent, url)
        except Exception as exc:
            log.debug("robots.txt klaida (%s): %s", url, exc)
            return True

    # ---- pagrindinis GET --------------------------------------------------
    def get(self, url: str, *, params: Optional[dict] = None,
            headers: Optional[dict] = None,
            respect_robots: bool = True) -> Optional[requests.Response]:
        if respect_robots and not self.can_fetch(url):
            log.warning("robots.txt draudžia: %s", url)
            return None

        @retry(reraise=True,
               stop=stop_after_attempt(self.retries),
               wait=wait_exponential(multiplier=self.backoff_factor, min=1, max=20),
               retry=retry_if_exception_type((requests.RequestException,)),
               before_sleep=before_sleep_log(log, logging_level=20))
        def _do() -> requests.Response:
            resp = self.session.get(url, params=params, headers=headers, timeout=self.timeout)
            if resp.status_code in (429, 503):
                raise requests.RequestException(f"Throttled status={resp.status_code}")
            resp.raise_for_status()
            return resp

        try:
            resp = _do()
            time.sleep(self.default_sleep)
            return resp
        except Exception as exc:
            log.warning("GET %s nepavyko: %s", url, exc)
            return None

    def get_json(self, url: str, **kw) -> Optional[dict]:
        resp = self.get(url, **kw)
        if resp is None:
            return None
        try:
            return resp.json()
        except ValueError:
            log.warning("Negaliojas JSON iš %s", url)
            return None

    def get_text(self, url: str, **kw) -> Optional[str]:
        resp = self.get(url, **kw)
        return resp.text if resp is not None else None