D4rkst3r/fivem-natives-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
fivem-natives-db/build.py
T
D4rkst3randClaude Opus 5 a33dba46a4 FiveM Natives + Docs Nachschlagewerk 
7359 Natives (GTA5 + Cfx) und 342 Doc-Seiten von docs.fivem.net in einer
SQLite-DB mit FTS5-Volltextsuche, dazu ein dependency-freies Python-CLI und
eine SKILL.md fuer Claude Code.

- build.py zieht runtime.fivem.net/doc/natives*.json und citizenfx/fivem-docs
  und loest die Hugo-Shortcodes der Docs auf (code, native_link, alert, events)
- lua_name folgt exakt der Regel aus FiveM ext/natives/codegen_out_lua.lua
- fivem.py: show / search / ns / docs / doc / stats, optional --json
- data/natives.jsonl als grep-barer Fallback

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-04 02:14:56 +02:00

435 lines
15 KiB
Python
Raw Permalink Blame History

#!/usr/bin/env python3
"""
Baut data/natives.db (SQLite + FTS5) aus:
- https://runtime.fivem.net/doc/natives.json (GTA5 natives)
- https://runtime.fivem.net/doc/natives_cfx.json (Cfx/FiveM natives)
- https://github.com/citizenfx/fivem-docs (docs.fivem.net Markdown)
Nur Python-Stdlib. Aufruf: python build.py [--refresh]
"""
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
import urllib.request
from datetime import datetime, timezone
from html.parser import HTMLParser
ROOT = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(ROOT, "data")
DB = os.path.join(DATA, "natives.db")
CACHE = os.path.join(ROOT, ".cache")
NATIVE_SOURCES = [
("gta5", "https://runtime.fivem.net/doc/natives.json"),
("cfx", "https://runtime.fivem.net/doc/natives_cfx.json"),
]
DOCS_REPO = "https://github.com/citizenfx/fivem-docs.git"
# ---------------------------------------------------------------- helpers
def log(*a):
print("[build]", *a, flush=True)
def fetch(url, dest):
os.makedirs(os.path.dirname(dest), exist_ok=True)
log("fetch", url)
req = urllib.request.Request(url, headers={"User-Agent": "fivem-natives-db/1.0"})
with urllib.request.urlopen(req, timeout=180) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f)
return dest
def pascal(name):
"""SET_ENTITY_COORDS -> SetEntityCoords.
Repliziert exakt die Regel aus FiveM ext/natives/codegen_out_lua.lua:
name:lower():gsub('_(%a)', upper):gsub('^%l', upper)
Wichtig: ein Unterstrich vor einer *Ziffer* bleibt erhalten, deshalb wird
DRAW_SCALEFORM_MOVIE_3D zu DrawScaleformMovie_3d (nicht ...Movie3D).
"""
if not name:
return ""
s = re.sub(r"_([a-z])", lambda m: m.group(1).upper(), name.lower())
return s[:1].upper() + s[1:]
def lua_name(nat):
n = nat.get("name")
if n:
return pascal(n)
# Unbenannte Natives werden von den Runtimes als N_0x<hash> exportiert
return "N_" + nat["hash"].lower()
def signature(nat, lname):
ps = ", ".join(
"%s %s" % (p.get("type", "Any"), p.get("name", "arg"))
for p in nat.get("params", [])
)
return "%s %s(%s)" % (nat.get("results") or "void", lname, ps)
# ---------------------------------------------------------------- natives
def load_natives(refresh=False):
rows = []
for source, url in NATIVE_SOURCES:
path = os.path.join(CACHE, os.path.basename(url))
if refresh or not os.path.exists(path):
fetch(url, path)
with open(path, "r", encoding="utf-8") as f:
doc = json.load(f)
for ns, natives in doc.items():
for h, nat in natives.items():
nat.setdefault("hash", h)
lname = lua_name(nat)
params = nat.get("params") or []
rows.append({
"hash": nat["hash"],
"jhash": nat.get("jhash") or "",
"name": nat.get("name") or "",
"lua_name": lname,
"ns": ns,
"source": source,
"game": nat.get("game") or ("gta5" if source == "gta5" else ""),
"apiset": nat.get("apiset") or ("client" if source == "gta5" else ""),
"results": nat.get("results") or "void",
"results_description": nat.get("resultsDescription") or "",
"description": (nat.get("description") or "").strip(),
"signature": signature(nat, lname),
"params_json": json.dumps(params, ensure_ascii=False),
"param_names": " ".join(p.get("name", "") for p in params),
"examples_json": json.dumps(nat.get("examples") or [], ensure_ascii=False),
"aliases": " ".join(nat.get("aliases") or []),
"url": "https://docs.fivem.net/natives/?_%s" % nat["hash"],
})
log("natives:", len(rows))
return rows
# ---------------------------------------------------------------- html -> text
class HtmlToText(HTMLParser):
"""Minimaler Konverter fuer die Event-Referenz (TypeDoc-HTML)."""
BLOCK = {"p", "div", "section", "li", "tr", "br", "ul", "ol", "table", "pre"}
HEAD = {"h1": "#", "h2": "##", "h3": "###", "h4": "####", "h5": "#####"}
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts = []
self._skip = 0
self._pre = 0
def handle_starttag(self, tag, attrs):
if tag in ("script", "style"):
self._skip += 1
elif tag in self.HEAD:
self.parts.append("\n\n%s " % self.HEAD[tag])
elif tag == "li":
self.parts.append("\n- ")
elif tag in ("pre", "code"):
self._pre += 1
self.parts.append("`" if tag == "code" else "\n```\n")
elif tag in self.BLOCK:
self.parts.append("\n")
def handle_endtag(self, tag):
if tag in ("script", "style"):
self._skip = max(0, self._skip - 1)
elif tag in ("pre", "code"):
self._pre = max(0, self._pre - 1)
self.parts.append("`" if tag == "code" else "\n```\n")
elif tag in self.HEAD or tag in self.BLOCK:
self.parts.append("\n")
def handle_data(self, data):
if not self._skip:
self.parts.append(data if self._pre else re.sub(r"\s+", " ", data))
def text(self):
out = "".join(self.parts)
out = re.sub(r"``\s*``", "", out)
out = re.sub(r"\n{3,}", "\n\n", out)
return "\n".join(l.rstrip() for l in out.splitlines()).strip()
def html_to_text(html):
p = HtmlToText()
p.feed(html)
p.close()
return p.text()
# ---------------------------------------------------------------- shortcodes
FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.S)
PAIRED_SC = re.compile(
r"\{\{[%<]\s*(alert|steps)\b(.*?)[%>]\}\}(.*?)\{\{[%<]\s*/\s*\1\s*[%>]\}\}", re.S)
SINGLE_SC = re.compile(r"\{\{[%<]\s*([A-Za-z0-9_-]+)((?:[^%>]|%(?!\}\})|>(?!\}))*?)\s*[%>]\}\}", re.S)
ATTR = re.compile(r'([A-Za-z0-9_-]+)\s*=\s*"([^"]*)"')
POSITIONAL = re.compile(r'"([^"]*)"')
def parse_attrs(s):
named = dict(ATTR.findall(s))
pos = [v for v in POSITIONAL.findall(ATTR.sub("", s))]
return named, pos
class Shortcodes:
"""Loest die Hugo-Shortcodes der fivem-docs zu reinem Markdown auf."""
def __init__(self, repo_root, native_hash_by_name):
self.root = repo_root
self.hashes = native_hash_by_name
self._events = {}
self._rmv = self._read_static("/static/resource_manifest_version.txt")
self._rmv2 = self._read_static("/static/resource_manifest_version2.txt")
def _read_static(self, rel):
path = os.path.join(self.root, rel.lstrip("/").replace("/", os.sep))
try:
with open(path, "r", encoding="utf-8", errors="replace") as f:
return f.read().strip()
except OSError:
return ""
def _event_doc(self, kind):
if kind not in self._events:
cached = os.path.join(CACHE, "events_%s.json" % kind)
if not os.path.exists(cached):
try:
fetch("https://runtime.fivem.net/doc/events/%s.html.json" % kind, cached)
except Exception as exc: # Netzwerk optional
log("events %s nicht geladen: %s" % (kind, exc))
self._events[kind] = ""
return ""
with open(cached, "r", encoding="utf-8") as f:
self._events[kind] = html_to_text(json.load(f).get("content", ""))
return self._events[kind]
# -- einzelne Shortcodes ------------------------------------------------
def _sc(self, name, attrs, pos):
if name == "code":
body = self._read_static(attrs.get("file", "")).replace("{{RMV}}", self._rmv)
return "\n```%s\n%s\n```\n" % (attrs.get("language", ""), body)
if name == "rmv":
return self._rmv
if name == "rmv2":
return self._rmv2
if name == "native_link":
n = (pos[0] if pos else "").strip()
h = self.hashes.get(n)
return "[%s](https://docs.fivem.net/natives/?_%s)" % (n, h) if h else "`%s`" % n
if name == "youtube":
vid = attrs.get("id") or (pos[0] if pos else "")
return "\n[Video: https://www.youtube.com/watch?v=%s]\n" % vid
if name == "video":
return "\n[Video: %s]\n" % attrs.get("src", "")
if name == "events":
return "\n" + self._event_doc((pos[0] if pos else "client").strip()) + "\n"
if name == "forum_topic":
return "\n[Community-Post: https://forum.cfx.re/t/%s]\n" % (pos[0] if pos else "")
if name in ("article-nav", "steps"):
return ""
return ""
def _paired(self, m):
name, raw, inner = m.group(1), m.group(2), m.group(3)
inner = self.render(inner).strip()
if name == "steps":
return "\n" + inner + "\n"
attrs, _ = parse_attrs(raw)
title = attrs.get("title", "")
head = "> **%s**\n" % title if title else ""
body = "\n".join("> " + l for l in inner.splitlines())
return "\n%s%s\n" % (head, body)
def render(self, text):
prev = None
while prev != text:
prev = text
text = PAIRED_SC.sub(self._paired, text)
return SINGLE_SC.sub(
lambda m: self._sc(m.group(1), *parse_attrs(m.group(2))), text)
def load_docs(natives, refresh=False):
src = os.path.join(CACHE, "fivem-docs")
if not os.path.isdir(src):
os.makedirs(CACHE, exist_ok=True)
log("clone", DOCS_REPO)
subprocess.run(["git", "clone", "--depth", "1", "-q", DOCS_REPO, src], check=True)
elif refresh:
subprocess.run(["git", "-C", src, "pull", "-q"], check=False)
# native_link braucht Name -> Hash (inkl. Aliases)
hashes = {}
for n in natives:
if n["name"]:
hashes.setdefault(n["name"], n["hash"])
for a in n["aliases"].split():
hashes.setdefault(a, n["hash"])
shortcodes = Shortcodes(src, hashes)
base = os.path.join(src, "content", "docs")
rows = []
for dirpath, _dirs, files in os.walk(base):
for fn in sorted(files):
if not fn.endswith(".md"):
continue
full = os.path.join(dirpath, fn)
rel = os.path.relpath(full, base).replace("\\", "/")
with open(full, "r", encoding="utf-8", errors="replace") as f:
raw = f.read()
title, weight = "", 0
body = raw
m = FRONTMATTER.match(raw)
if m:
body = raw[m.end():]
for line in m.group(1).splitlines():
low = line.lower()
if low.startswith("title:"):
title = line.split(":", 1)[1].strip().strip('"').strip("'")
elif low.startswith("weight:"):
try:
weight = int(line.split(":", 1)[1].strip())
except ValueError:
pass
if not title:
title = os.path.splitext(fn)[0].replace("-", " ").title()
slug = rel[:-3]
if slug.endswith("/_index") or slug == "_index":
slug = slug[:-len("_index")].rstrip("/")
section = slug.split("/")[0] if slug else "root"
url = "https://docs.fivem.net/docs/" + (slug + "/" if slug else "")
rows.append({
"path": rel,
"slug": slug,
"section": section,
"title": title,
"weight": weight,
"url": url,
"body": shortcodes.render(body).strip(),
"raw": body.strip(),
})
log("docs:", len(rows))
return rows
# ---------------------------------------------------------------- schema
SCHEMA = """
DROP TABLE IF EXISTS natives_fts;
DROP TABLE IF EXISTS natives;
DROP TABLE IF EXISTS docs_fts;
DROP TABLE IF EXISTS docs;
DROP TABLE IF EXISTS meta;
CREATE TABLE natives (
id INTEGER PRIMARY KEY,
hash TEXT NOT NULL,
jhash TEXT,
name TEXT,
lua_name TEXT,
ns TEXT,
source TEXT, -- gta5 | cfx
game TEXT, -- gta5 | rdr3 | ny | ''
apiset TEXT, -- client | server | shared
results TEXT,
results_description TEXT,
description TEXT,
signature TEXT,
params_json TEXT,
param_names TEXT,
examples_json TEXT,
aliases TEXT,
url TEXT
);
CREATE INDEX idx_nat_hash ON natives(hash);
CREATE INDEX idx_nat_name ON natives(name);
CREATE INDEX idx_nat_lua ON natives(lua_name);
CREATE INDEX idx_nat_ns ON natives(ns);
CREATE INDEX idx_nat_api ON natives(apiset);
CREATE VIRTUAL TABLE natives_fts USING fts5(
name, lua_name, ns, description, param_names, aliases, hash,
content='natives', content_rowid='id',
tokenize="unicode61 tokenchars '_'"
);
CREATE TABLE docs (
id INTEGER PRIMARY KEY,
path TEXT, slug TEXT, section TEXT, title TEXT,
weight INTEGER, url TEXT, body TEXT, raw TEXT
);
CREATE INDEX idx_docs_slug ON docs(slug);
CREATE INDEX idx_docs_sect ON docs(section);
CREATE VIRTUAL TABLE docs_fts USING fts5(
title, section, slug, body,
content='docs', content_rowid='id',
tokenize="unicode61 tokenchars '_'"
);
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
"""
def build(refresh=False):
os.makedirs(DATA, exist_ok=True)
natives = load_natives(refresh)
docs = load_docs(natives, refresh)
if os.path.exists(DB):
os.remove(DB)
con = sqlite3.connect(DB)
con.executescript(SCHEMA)
ncols = ["hash", "jhash", "name", "lua_name", "ns", "source", "game", "apiset",
"results", "results_description", "description", "signature",
"params_json", "param_names", "examples_json", "aliases", "url"]
con.executemany(
"INSERT INTO natives (%s) VALUES (%s)" % (",".join(ncols), ",".join("?" * len(ncols))),
[[r[c] for c in ncols] for r in natives],
)
con.execute("INSERT INTO natives_fts(natives_fts) VALUES('rebuild')")
dcols = ["path", "slug", "section", "title", "weight", "url", "body", "raw"]
con.executemany(
"INSERT INTO docs (%s) VALUES (%s)" % (",".join(dcols), ",".join("?" * len(dcols))),
[[r[c] for c in dcols] for r in docs],
)
con.execute("INSERT INTO docs_fts(docs_fts) VALUES('rebuild')")
con.executemany("INSERT INTO meta (key,value) VALUES (?,?)", [
("built_at", datetime.now(timezone.utc).isoformat(timespec="seconds")),
("native_count", str(len(natives))),
("doc_count", str(len(docs))),
("sources", json.dumps([u for _, u in NATIVE_SOURCES] + [DOCS_REPO])),
("schema_version", "1"),
])
con.commit()
con.execute("VACUUM")
con.close()
# Grep-freundlicher Export (eine Zeile pro Native)
out = os.path.join(DATA, "natives.jsonl")
with open(out, "w", encoding="utf-8", newline="\n") as f:
for r in sorted(natives, key=lambda x: (x["ns"], x["name"] or x["hash"])):
slim = {k: r[k] for k in ("hash", "name", "lua_name", "ns", "apiset", "game",
"signature", "description", "aliases", "url")}
slim["params"] = json.loads(r["params_json"])
f.write(json.dumps(slim, ensure_ascii=False) + "\n")
log("ok ->", DB, "%.1f MB" % (os.path.getsize(DB) / 1e6))
if __name__ == "__main__":
build(refresh="--refresh" in sys.argv)
Reference in New Issue View Git Blame Copy Permalink