#!/usr/bin/env python3
"""
Baut data/natives.db (SQLite + FTS5) aus:
  - https://runtime.fivem.net/doc/natives.json      (GTA5 natives)
  - https://runtime.fivem.net/doc/natives_cfx.json  (Cfx/FiveM natives)
  - https://github.com/citizenfx/fivem-docs          (docs.fivem.net Markdown)

Nur Python-Stdlib. Aufruf:  python build.py [--refresh]
"""
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
import urllib.request
from datetime import datetime, timezone
from html.parser import HTMLParser

ROOT = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(ROOT, "data")
DB = os.path.join(DATA, "natives.db")
CACHE = os.path.join(ROOT, ".cache")

NATIVE_SOURCES = [
    ("gta5", "https://runtime.fivem.net/doc/natives.json"),
    ("cfx", "https://runtime.fivem.net/doc/natives_cfx.json"),
]
DOCS_REPO = "https://github.com/citizenfx/fivem-docs.git"


# ---------------------------------------------------------------- helpers
def log(*a):
    print("[build]", *a, flush=True)


def fetch(url, dest):
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    log("fetch", url)
    req = urllib.request.Request(url, headers={"User-Agent": "fivem-natives-db/1.0"})
    with urllib.request.urlopen(req, timeout=180) as r, open(dest, "wb") as f:
        shutil.copyfileobj(r, f)
    return dest


def pascal(name):
    """SET_ENTITY_COORDS -> SetEntityCoords.

    Repliziert exakt die Regel aus FiveM ext/natives/codegen_out_lua.lua:
        name:lower():gsub('_(%a)', upper):gsub('^%l', upper)
    Wichtig: ein Unterstrich vor einer *Ziffer* bleibt erhalten, deshalb wird
    DRAW_SCALEFORM_MOVIE_3D zu DrawScaleformMovie_3d (nicht ...Movie3D).
    """
    if not name:
        return ""
    s = re.sub(r"_([a-z])", lambda m: m.group(1).upper(), name.lower())
    return s[:1].upper() + s[1:]


def lua_name(nat):
    n = nat.get("name")
    if n:
        return pascal(n)
    # Unbenannte Natives werden von den Runtimes als N_0x<hash> exportiert
    return "N_" + nat["hash"].lower()


def signature(nat, lname):
    ps = ", ".join(
        "%s %s" % (p.get("type", "Any"), p.get("name", "arg"))
        for p in nat.get("params", [])
    )
    return "%s %s(%s)" % (nat.get("results") or "void", lname, ps)


# ---------------------------------------------------------------- natives
def load_natives(refresh=False):
    rows = []
    for source, url in NATIVE_SOURCES:
        path = os.path.join(CACHE, os.path.basename(url))
        if refresh or not os.path.exists(path):
            fetch(url, path)
        with open(path, "r", encoding="utf-8") as f:
            doc = json.load(f)
        for ns, natives in doc.items():
            for h, nat in natives.items():
                nat.setdefault("hash", h)
                lname = lua_name(nat)
                params = nat.get("params") or []
                rows.append({
                    "hash": nat["hash"],
                    "jhash": nat.get("jhash") or "",
                    "name": nat.get("name") or "",
                    "lua_name": lname,
                    "ns": ns,
                    "source": source,
                    "game": nat.get("game") or ("gta5" if source == "gta5" else ""),
                    "apiset": nat.get("apiset") or ("client" if source == "gta5" else ""),
                    "results": nat.get("results") or "void",
                    "results_description": nat.get("resultsDescription") or "",
                    "description": (nat.get("description") or "").strip(),
                    "signature": signature(nat, lname),
                    "params_json": json.dumps(params, ensure_ascii=False),
                    "param_names": " ".join(p.get("name", "") for p in params),
                    "examples_json": json.dumps(nat.get("examples") or [], ensure_ascii=False),
                    "aliases": " ".join(nat.get("aliases") or []),
                    "url": "https://docs.fivem.net/natives/?_%s" % nat["hash"],
                })
    log("natives:", len(rows))
    return rows


# ---------------------------------------------------------------- html -> text
class HtmlToText(HTMLParser):
    """Minimaler Konverter fuer die Event-Referenz (TypeDoc-HTML)."""

    BLOCK = {"p", "div", "section", "li", "tr", "br", "ul", "ol", "table", "pre"}
    HEAD = {"h1": "#", "h2": "##", "h3": "###", "h4": "####", "h5": "#####"}

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.parts = []
        self._skip = 0
        self._pre = 0

    def handle_starttag(self, tag, attrs):
        if tag in ("script", "style"):
            self._skip += 1
        elif tag in self.HEAD:
            self.parts.append("\n\n%s " % self.HEAD[tag])
        elif tag == "li":
            self.parts.append("\n- ")
        elif tag in ("pre", "code"):
            self._pre += 1
            self.parts.append("`" if tag == "code" else "\n```\n")
        elif tag in self.BLOCK:
            self.parts.append("\n")

    def handle_endtag(self, tag):
        if tag in ("script", "style"):
            self._skip = max(0, self._skip - 1)
        elif tag in ("pre", "code"):
            self._pre = max(0, self._pre - 1)
            self.parts.append("`" if tag == "code" else "\n```\n")
        elif tag in self.HEAD or tag in self.BLOCK:
            self.parts.append("\n")

    def handle_data(self, data):
        if not self._skip:
            self.parts.append(data if self._pre else re.sub(r"\s+", " ", data))

    def text(self):
        out = "".join(self.parts)
        out = re.sub(r"``\s*``", "", out)
        out = re.sub(r"\n{3,}", "\n\n", out)
        return "\n".join(l.rstrip() for l in out.splitlines()).strip()


def html_to_text(html):
    p = HtmlToText()
    p.feed(html)
    p.close()
    return p.text()


# ---------------------------------------------------------------- shortcodes
FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.S)
PAIRED_SC = re.compile(
    r"\{\{[%<]\s*(alert|steps)\b(.*?)[%>]\}\}(.*?)\{\{[%<]\s*/\s*\1\s*[%>]\}\}", re.S)
SINGLE_SC = re.compile(r"\{\{[%<]\s*([A-Za-z0-9_-]+)((?:[^%>]|%(?!\}\})|>(?!\}))*?)\s*[%>]\}\}", re.S)
ATTR = re.compile(r'([A-Za-z0-9_-]+)\s*=\s*"([^"]*)"')
POSITIONAL = re.compile(r'"([^"]*)"')


def parse_attrs(s):
    named = dict(ATTR.findall(s))
    pos = [v for v in POSITIONAL.findall(ATTR.sub("", s))]
    return named, pos


class Shortcodes:
    """Loest die Hugo-Shortcodes der fivem-docs zu reinem Markdown auf."""

    def __init__(self, repo_root, native_hash_by_name):
        self.root = repo_root
        self.hashes = native_hash_by_name
        self._events = {}
        self._rmv = self._read_static("/static/resource_manifest_version.txt")
        self._rmv2 = self._read_static("/static/resource_manifest_version2.txt")

    def _read_static(self, rel):
        path = os.path.join(self.root, rel.lstrip("/").replace("/", os.sep))
        try:
            with open(path, "r", encoding="utf-8", errors="replace") as f:
                return f.read().strip()
        except OSError:
            return ""

    def _event_doc(self, kind):
        if kind not in self._events:
            cached = os.path.join(CACHE, "events_%s.json" % kind)
            if not os.path.exists(cached):
                try:
                    fetch("https://runtime.fivem.net/doc/events/%s.html.json" % kind, cached)
                except Exception as exc:  # Netzwerk optional
                    log("events %s nicht geladen: %s" % (kind, exc))
                    self._events[kind] = ""
                    return ""
            with open(cached, "r", encoding="utf-8") as f:
                self._events[kind] = html_to_text(json.load(f).get("content", ""))
        return self._events[kind]

    # -- einzelne Shortcodes ------------------------------------------------
    def _sc(self, name, attrs, pos):
        if name == "code":
            body = self._read_static(attrs.get("file", "")).replace("{{RMV}}", self._rmv)
            return "\n```%s\n%s\n```\n" % (attrs.get("language", ""), body)
        if name == "rmv":
            return self._rmv
        if name == "rmv2":
            return self._rmv2
        if name == "native_link":
            n = (pos[0] if pos else "").strip()
            h = self.hashes.get(n)
            return "[%s](https://docs.fivem.net/natives/?_%s)" % (n, h) if h else "`%s`" % n
        if name == "youtube":
            vid = attrs.get("id") or (pos[0] if pos else "")
            return "\n[Video: https://www.youtube.com/watch?v=%s]\n" % vid
        if name == "video":
            return "\n[Video: %s]\n" % attrs.get("src", "")
        if name == "events":
            return "\n" + self._event_doc((pos[0] if pos else "client").strip()) + "\n"
        if name == "forum_topic":
            return "\n[Community-Post: https://forum.cfx.re/t/%s]\n" % (pos[0] if pos else "")
        if name in ("article-nav", "steps"):
            return ""
        return ""

    def _paired(self, m):
        name, raw, inner = m.group(1), m.group(2), m.group(3)
        inner = self.render(inner).strip()
        if name == "steps":
            return "\n" + inner + "\n"
        attrs, _ = parse_attrs(raw)
        title = attrs.get("title", "")
        head = "> **%s**\n" % title if title else ""
        body = "\n".join("> " + l for l in inner.splitlines())
        return "\n%s%s\n" % (head, body)

    def render(self, text):
        prev = None
        while prev != text:
            prev = text
            text = PAIRED_SC.sub(self._paired, text)
        return SINGLE_SC.sub(
            lambda m: self._sc(m.group(1), *parse_attrs(m.group(2))), text)


def load_docs(natives, refresh=False):
    src = os.path.join(CACHE, "fivem-docs")
    if not os.path.isdir(src):
        os.makedirs(CACHE, exist_ok=True)
        log("clone", DOCS_REPO)
        subprocess.run(["git", "clone", "--depth", "1", "-q", DOCS_REPO, src], check=True)
    elif refresh:
        subprocess.run(["git", "-C", src, "pull", "-q"], check=False)

    # native_link braucht Name -> Hash (inkl. Aliases)
    hashes = {}
    for n in natives:
        if n["name"]:
            hashes.setdefault(n["name"], n["hash"])
        for a in n["aliases"].split():
            hashes.setdefault(a, n["hash"])
    shortcodes = Shortcodes(src, hashes)

    base = os.path.join(src, "content", "docs")
    rows = []
    for dirpath, _dirs, files in os.walk(base):
        for fn in sorted(files):
            if not fn.endswith(".md"):
                continue
            full = os.path.join(dirpath, fn)
            rel = os.path.relpath(full, base).replace("\\", "/")
            with open(full, "r", encoding="utf-8", errors="replace") as f:
                raw = f.read()

            title, weight = "", 0
            body = raw
            m = FRONTMATTER.match(raw)
            if m:
                body = raw[m.end():]
                for line in m.group(1).splitlines():
                    low = line.lower()
                    if low.startswith("title:"):
                        title = line.split(":", 1)[1].strip().strip('"').strip("'")
                    elif low.startswith("weight:"):
                        try:
                            weight = int(line.split(":", 1)[1].strip())
                        except ValueError:
                            pass
            if not title:
                title = os.path.splitext(fn)[0].replace("-", " ").title()

            slug = rel[:-3]
            if slug.endswith("/_index") or slug == "_index":
                slug = slug[:-len("_index")].rstrip("/")
            section = slug.split("/")[0] if slug else "root"
            url = "https://docs.fivem.net/docs/" + (slug + "/" if slug else "")

            rows.append({
                "path": rel,
                "slug": slug,
                "section": section,
                "title": title,
                "weight": weight,
                "url": url,
                "body": shortcodes.render(body).strip(),
                "raw": body.strip(),
            })
    log("docs:", len(rows))
    return rows


# ---------------------------------------------------------------- schema
SCHEMA = """
DROP TABLE IF EXISTS natives_fts;
DROP TABLE IF EXISTS natives;
DROP TABLE IF EXISTS docs_fts;
DROP TABLE IF EXISTS docs;
DROP TABLE IF EXISTS meta;

CREATE TABLE natives (
  id INTEGER PRIMARY KEY,
  hash TEXT NOT NULL,
  jhash TEXT,
  name TEXT,
  lua_name TEXT,
  ns TEXT,
  source TEXT,          -- gta5 | cfx
  game TEXT,            -- gta5 | rdr3 | ny | ''
  apiset TEXT,          -- client | server | shared
  results TEXT,
  results_description TEXT,
  description TEXT,
  signature TEXT,
  params_json TEXT,
  param_names TEXT,
  examples_json TEXT,
  aliases TEXT,
  url TEXT
);
CREATE INDEX idx_nat_hash ON natives(hash);
CREATE INDEX idx_nat_name ON natives(name);
CREATE INDEX idx_nat_lua  ON natives(lua_name);
CREATE INDEX idx_nat_ns   ON natives(ns);
CREATE INDEX idx_nat_api  ON natives(apiset);

CREATE VIRTUAL TABLE natives_fts USING fts5(
  name, lua_name, ns, description, param_names, aliases, hash,
  content='natives', content_rowid='id',
  tokenize="unicode61 tokenchars '_'"
);

CREATE TABLE docs (
  id INTEGER PRIMARY KEY,
  path TEXT, slug TEXT, section TEXT, title TEXT,
  weight INTEGER, url TEXT, body TEXT, raw TEXT
);
CREATE INDEX idx_docs_slug ON docs(slug);
CREATE INDEX idx_docs_sect ON docs(section);

CREATE VIRTUAL TABLE docs_fts USING fts5(
  title, section, slug, body,
  content='docs', content_rowid='id',
  tokenize="unicode61 tokenchars '_'"
);

CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
"""


def build(refresh=False):
    os.makedirs(DATA, exist_ok=True)
    natives = load_natives(refresh)
    docs = load_docs(natives, refresh)

    if os.path.exists(DB):
        os.remove(DB)
    con = sqlite3.connect(DB)
    con.executescript(SCHEMA)

    ncols = ["hash", "jhash", "name", "lua_name", "ns", "source", "game", "apiset",
             "results", "results_description", "description", "signature",
             "params_json", "param_names", "examples_json", "aliases", "url"]
    con.executemany(
        "INSERT INTO natives (%s) VALUES (%s)" % (",".join(ncols), ",".join("?" * len(ncols))),
        [[r[c] for c in ncols] for r in natives],
    )
    con.execute("INSERT INTO natives_fts(natives_fts) VALUES('rebuild')")

    dcols = ["path", "slug", "section", "title", "weight", "url", "body", "raw"]
    con.executemany(
        "INSERT INTO docs (%s) VALUES (%s)" % (",".join(dcols), ",".join("?" * len(dcols))),
        [[r[c] for c in dcols] for r in docs],
    )
    con.execute("INSERT INTO docs_fts(docs_fts) VALUES('rebuild')")

    con.executemany("INSERT INTO meta (key,value) VALUES (?,?)", [
        ("built_at", datetime.now(timezone.utc).isoformat(timespec="seconds")),
        ("native_count", str(len(natives))),
        ("doc_count", str(len(docs))),
        ("sources", json.dumps([u for _, u in NATIVE_SOURCES] + [DOCS_REPO])),
        ("schema_version", "1"),
    ])
    con.commit()
    con.execute("VACUUM")
    con.close()

    # Grep-freundlicher Export (eine Zeile pro Native)
    out = os.path.join(DATA, "natives.jsonl")
    with open(out, "w", encoding="utf-8", newline="\n") as f:
        for r in sorted(natives, key=lambda x: (x["ns"], x["name"] or x["hash"])):
            slim = {k: r[k] for k in ("hash", "name", "lua_name", "ns", "apiset", "game",
                                      "signature", "description", "aliases", "url")}
            slim["params"] = json.loads(r["params_json"])
            f.write(json.dumps(slim, ensure_ascii=False) + "\n")

    log("ok ->", DB, "%.1f MB" % (os.path.getsize(DB) / 1e6))


if __name__ == "__main__":
    build(refresh="--refresh" in sys.argv)