#!/usr/bin/env python3
"""EOJN public RSS -> ScraperAPI residential proxies -> TenderScan ingestion.

Set SCRAPERAPI_KEY securely in the robot host environment (never in this file).
Set EOJN_FEED_URL to an accessible official public feed when available.
python eojn_scraper.py --output tenders.json
For Cloud sync set TENDERS_SYNC_URL and TENDERS_SYNC_TOKEN, then use --sync.
The token comes from the app's secure secret form, not a database service key.
Run on an external Python host; Lovable Cloud does not execute Python scripts.
Residential requests consume premium credits. Proxies do not grant login access.
"""
import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from urllib.parse import urlparse
import xml.etree.ElementTree as ET

import requests
from bs4 import BeautifulSoup

EOJN_RSS = 'https://eojn.nn.hr/SPIN/application/ipn/DocumentManagement/RssFeed.aspx'
SCRAPERAPI_URL = 'https://api.scraperapi.com'
KEYWORDS = re.compile(r'radov|građen|gradnj|cest|obnov|rekonstrukcij|sanacij|asfalt', re.I)


def official_url(value):
    url = urlparse(value)
    return (url.scheme == 'https' and not url.username and not url.password
            and bool(re.fullmatch(r'(?:[a-z0-9-]+\.)*eojn\.(?:nn\.hr|hr)', url.hostname or '', re.I)))


def iso_date(raw):
    if not raw:
        return None
    try:
        dt = datetime.fromisoformat(raw.replace('Z', '+00:00'))
    except ValueError:
        try:
            dt = parsedate_to_datetime(raw)
        except (ValueError, TypeError):
            return None
    if dt.tzinfo is None:
        # Do not guess the zone of local tender deadlines.
        return None
    return dt.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')


def parse_tenders(xml):
    if '<!DOCTYPE' in xml.upper() or '<!ENTITY' in xml.upper():
        raise ValueError('Unsupported XML declarations')
    root = ET.fromstring(xml)
    local = lambda tag: tag.split('}')[-1]
    if local(root.tag) not in ('rss', 'feed', 'RDF'):
        raise ValueError('EOJN returned a login page or non-feed response')
    rows = {}
    for item in root.iter():
        if local(item.tag) not in ('item', 'entry'):
            continue
        children = list(item.iter())
        def text(*names):
            return next((''.join(el.itertext()).strip() for el in children if local(el.tag) in names), '')
        title = text('title')
        description = BeautifulSoup(text('description', 'summary', 'content'), 'html.parser').get_text('\n')
        if not title or not KEYWORDS.search(title + ' ' + description):
            continue
        link = next((el for el in children if local(el.tag) == 'link' and el.get('rel', 'alternate') == 'alternate'), None)
        url = (link.get('href') or (link.text or '').strip()) if link is not None else ''
        if not official_url(url):
            continue
        deadline = iso_date(text('deadline', 'rok'))
        if deadline and datetime.fromisoformat(deadline.replace('Z', '+00:00')) < datetime.now(timezone.utc):
            continue
        buyer = text('narucitelj', 'buyer', 'investor')
        if not buyer:
            match = re.search(r'naručitelj\s*:\s*([^\n\r;]+)', description, re.I)
            buyer = match.group(1).strip() if match else ''
        cpv = text('cpv', 'CPV')
        if not cpv:
            match = re.search(r'\bCPV\s*:?\s*(\d{8}(?:-\d)?)', description, re.I)
            cpv = match.group(1) if match else ''
        raw_value = text('estimatedValue', 'vrijednost')
        value = None
        if re.fullmatch(r'\d+(?:\.\d+)?', raw_value):
            value = float(raw_value)
        rows[url] = dict(id=hashlib.sha256(url.encode()).hexdigest(), naziv=title,
                         narucitelj=buyer or None, url=url, cpv=cpv or None,
                         vrijednost=value, rok=deadline,
                         objavljeno=iso_date(text('pubDate', 'published', 'updated', 'date')))
    return list(rows.values())


def fetch_feed(session, api_key, feed_url):
    if not official_url(feed_url):
        raise ValueError('EOJN_FEED_URL must be an official HTTPS EOJN URL')
    # No session_number: use the provider's automatic rotation, not sticky IPs.
    try:
        response = session.get(SCRAPERAPI_URL, params={'api_key': api_key, 'url': feed_url,
                               'premium': 'true'}, timeout=90)
    except requests.RequestException:
        raise RuntimeError('ScraperAPI request failed; check provider settings') from None
    if not response.ok:
        raise RuntimeError(f'ScraperAPI HTTP {response.status_code}; no records saved')
    xml = response.text
    parse_tenders(xml)  # Reject login/challenge HTML before sync or output.
    return xml


def main():
    parser = argparse.ArgumentParser(description='EOJN public RSS synchronization')
    parser.add_argument('--output', default='tenders.json')
    parser.add_argument('--sync', action='store_true')
    args = parser.parse_args()
    key = os.environ.get('SCRAPERAPI_KEY')
    if not key:
        parser.error('Set SCRAPERAPI_KEY in the robot host secret environment')
    try:
        with requests.Session() as session:
            xml = fetch_feed(session, key, os.environ.get('EOJN_FEED_URL', EOJN_RSS))
            rows = parse_tenders(xml)
            if args.sync:
                url = os.environ.get('TENDERS_SYNC_URL', '')
                token = os.environ.get('TENDERS_SYNC_TOKEN', '')
                if not url.startswith('https://') or not token:
                    raise ValueError('Set HTTPS TENDERS_SYNC_URL and TENDERS_SYNC_TOKEN')
                # Send original XML so the server validates and maps records itself.
                try:
                    result = session.post(url, data=xml.encode(), headers={
                        'Authorization': 'Bearer ' + token, 'Content-Type': 'application/xml'},
                        timeout=60, allow_redirects=False)
                except requests.RequestException:
                    raise RuntimeError('Tender synchronization request failed') from None
                if not result.ok:
                    raise RuntimeError(f'Tender synchronization HTTP {result.status_code}')
            with open(args.output, 'w', encoding='utf-8') as handle:
                json.dump(rows, handle, ensure_ascii=False, indent=2)
            print(f'Saved {len(rows)} source records' + (' and synchronized database' if args.sync else ''))
    except (ValueError, ET.ParseError, RuntimeError):
        # Never log provider URLs, response bodies, exceptions containing credentials.
        print('Sync failed: check public feed access, provider account and secure configuration. No fallback rows generated.', file=sys.stderr)
        sys.exit(1)


if __name__ == '__main__':
    main()