"""Builds Soi/Resources/additives.json from the Open Food Facts additives taxonomy.

Run from the project folder:  python3 Tools/build_additives.py [--review]

Source: Tools/data/additives.txt (Open Food Facts, Open Database License).
The levels are not ours. They come from the EFSA exposure assessments recorded in the
taxonomy, plus two facts of EU law that are listed in EU_WARNING and EU_NOT_AUTHORISED.
"""
import json
import re
import sys
import unicodedata
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SOURCE = ROOT / "Tools/data/additives.txt"
OUTPUT = ROOT / "Soi/Resources/additives.json"
SOURCE_DATE = "2026-11-11"

# Regulation (EC) No 1333/2008, Annex V: foods with these colours must carry
# "may have an adverse effect on activity and attention in children".
EU_WARNING = {"e102", "e104", "e110", "e122", "e124", "e129"}
# Commission Regulation (EU) 2022/63: titanium dioxide is no longer authorised in food in the EU.
EU_NOT_AUTHORISED = {"e171"}

# Vietnamese names as they are printed on labels. The first one is shown in the app,
# all of them are matched. Only common additives are covered; the rest show the English name.
VI = {
    "e100": ["Curcumin"],
    "e101": ["Riboflavin"],
    "e102": ["Tartrazin", "vàng tartrazin"],
    "e104": ["Vàng quinolin", "quinolin vàng"],
    "e110": ["Sunset yellow FCF", "vàng sunset FCF"],
    "e120": ["Carmin"],
    "e122": ["Carmoisin", "azorubin"],
    "e123": ["Amaranth"],
    "e124": ["Ponceau 4R", "đỏ ponceau 4R"],
    "e127": ["Erythrosin"],
    "e129": ["Đỏ allura AC", "allura red AC"],
    "e132": ["Indigotin", "indigo carmin"],
    "e133": ["Xanh brilliant FCF", "brilliant blue FCF"],
    "e150": ["Màu caramen", "màu caramel"],
    "e150a": ["Caramen nhóm I", "caramel nhóm I"],
    "e150b": ["Caramen nhóm II", "caramel nhóm II"],
    "e150c": ["Caramen nhóm III", "caramel nhóm III"],
    "e150d": ["Caramen nhóm IV", "caramel nhóm IV"],
    "e160a": ["Beta-caroten", "beta caroten", "caroten"],
    "e160b": ["Màu điều (annatto)", "màu điều nhuộm", "màu điều", "cao annatto"],
    "e160c": ["Paprika oleoresin", "màu paprika", "chiết xuất ớt paprika"],
    "e162": ["Đỏ củ dền", "màu củ dền"],
    "e163": ["Anthocyanin"],
    "e170": ["Calci carbonat", "canxi cacbonat", "canxi carbonat"],
    "e171": ["Titan dioxyd", "titan dioxit"],
    "e200": ["Acid sorbic"],
    "e202": ["Kali sorbat"],
    "e210": ["Acid benzoic"],
    "e211": ["Natri benzoat"],
    "e220": ["Sulfua dioxyd", "lưu huỳnh dioxyd", "lưu huỳnh đioxit"],
    "e223": ["Natri metabisulfit"],
    "e224": ["Kali metabisulfit"],
    "e234": ["Nisin"],
    "e235": ["Natamycin"],
    "e249": ["Kali nitrit"],
    "e250": ["Natri nitrit"],
    "e251": ["Natri nitrat"],
    "e252": ["Kali nitrat"],
    "e260": ["Acid acetic"],
    "e262": ["Natri acetat"],
    "e270": ["Acid lactic"],
    "e280": ["Acid propionic"],
    "e281": ["Natri propionat"],
    "e282": ["Calci propionat", "canxi propionat"],
    "e290": ["Carbon dioxyd", "khí carbonic", "khí CO2"],
    "e296": ["Acid malic"],
    "e300": ["Acid ascorbic"],
    "e301": ["Natri ascorbat"],
    "e316": ["Natri erythorbat"],
    "e319": ["TBHQ"],
    "e320": ["BHA"],
    "e321": ["BHT"],
    "e322": ["Lecithin", "lecitin"],
    "e325": ["Natri lactat"],
    "e327": ["Calci lactat", "canxi lactat"],
    "e330": ["Acid citric"],
    "e331": ["Natri citrat"],
    "e331iii": ["Trinatri citrat"],
    "e332": ["Kali citrat"],
    "e333": ["Calci citrat", "canxi citrat"],
    "e334": ["Acid tartric"],
    "e338": ["Acid phosphoric"],
    "e339": ["Natri phosphat"],
    "e340": ["Kali phosphat"],
    "e341": ["Calci phosphat", "canxi phosphat"],
    "e341iii": ["Tricalci phosphat"],
    "e401": ["Natri alginat"],
    "e406": ["Agar"],
    "e407": ["Carrageenan"],
    "e410": ["Gôm đậu carob", "gum đậu carob"],
    "e412": ["Gôm guar", "gum guar"],
    "e414": ["Gôm arabic", "gum arabic"],
    "e415": ["Gôm xanthan", "gum xanthan"],
    "e418": ["Gôm gellan", "gum gellan"],
    "e420": ["Sorbitol"],
    "e421": ["Manitol"],
    "e422": ["Glycerol", "glycerin"],
    "e440": ["Pectin"],
    "e450": ["Diphosphat"],
    "e451": ["Triphosphat"],
    "e451i": ["Pentanatri triphosphat"],
    "e452": ["Polyphosphat"],
    "e452i": ["Natri polyphosphat"],
    "e466": ["Natri carboxymethyl cellulose", "CMC"],
    "e471": ["Mono và diglycerid của acid béo", "mono và diglycerid của các acid béo", "mono diglycerid"],
    "e472e": ["DATEM"],
    "e473": ["Este sucrose của acid béo"],
    "e476": ["Polyglycerol polyricinoleat", "PGPR"],
    "e481": ["Natri stearoyl lactylat"],
    "e500": ["Natri carbonat"],
    "e500ii": ["Natri hydro carbonat", "natri hydrocarbonat", "natri bicarbonat"],
    "e501": ["Kali carbonat"],
    "e503": ["Amoni carbonat"],
    "e503ii": ["Amoni hydro carbonat", "amoni hydrocarbonat", "amoni bicarbonat"],
    "e508": ["Kali clorid", "kali clorua"],
    "e509": ["Calci clorid", "canxi clorua"],
    "e551": ["Silic dioxyd", "dioxyd silic"],
    "e621": ["Bột ngọt (mononatri glutamat)", "mononatri glutamat", "mononatri L-glutamat", "bột ngọt"],
    "e627": ["Dinatri guanylat", "dinatri 5'-guanylat"],
    "e631": ["Dinatri inosinat", "dinatri 5'-inosinat"],
    "e635": ["Dinatri 5'-ribonucleotid", "dinatri ribonucleotid"],
    "e950": ["Acesulfam kali", "acesulfame kali", "acesulfam K"],
    "e951": ["Aspartam"],
    "e952": ["Cyclamat", "natri cyclamat"],
    "e954": ["Saccharin", "sacarin"],
    "e955": ["Sucralose", "sucraloza"],
    "e960": ["Steviol glycosid"],
    "e965": ["Maltitol"],
    "e967": ["Xylitol"],
    "e968": ["Erythritol"],
    "e1400": ["Dextrin"],
    "e1422": ["Acetylated distarch adipat"],
    "e1442": ["Hydroxypropyl distarch phosphat"],
}

# Words that are also everyday ingredients, flavours or brand words. Matching them as an additive
# would be wrong more often than right, so they are only found through their code or a longer name.
STOP = {
    "gold", "silver", "aluminium", "aluminum", "caramel", "caramen", "paprika", "turmeric", "saffron",
    "carbon", "oxygen", "nitrogen", "hydrogen", "helium", "argon", "air", "water", "salt", "sugar",
    "starch", "urea", "alcohol", "ethanol", "caffeine", "vang", "bac", "nhom", "do", "xanh", "nito",
    "oxy", "ure", "sap", "talc", "beetroot", "beet", "annatto extract", "vitamin c", "vitamin e",
    "vitamin b2", "vitamin b 2", "lactoflavin", "colour", "color", "wax", "shellac", "lutein",
    "lycopene", "chlorophyll", "carotene", "carotenes", "caroten", "tocopherol", "tocopherols",
    "glutamate", "glutamat", "phosphate", "phosphates", "citrate", "citrates", "lactate", "nitrate",
    "nitrite", "sulphite", "sulfite", "sulphites", "sulfites", "benzoate", "sorbate", "acetate",
    "propane", "butane", "isobutane", "gelatin", "gelatine", "glycine", "cysteine", "protease",
    "lipase", "invertase", "lysozyme", "polydextrose", "acid", "enzyme", "enzymes",
    "linear", "monobasic", "ceylon", "bengal", "chalk", "ammonia", "magnesia", "niacin", "nicotinamide", "beeswax", "wax",
    "fatty acids", "linear fatty acids", "caprylic acid", "capric acid", "lauric acid", "myristic acid", "palmitic acid",
    "stearic acid", "oleic acid", "linoleic acid",
    "riboflavin", "riboflavine", "riboflavins", "leucine", "alanin", "carbonates", "succinate", "amylase",
    "proteinase", "peptidase", "chlorine", "sandalwood",
}
SHORT_OK = {"msg", "bha", "bht", "cmc"}


def fold(text):
    """The same folding as LabelText.fold in Swift: lowercase ASCII letters and digits, everything else a space."""
    text = text.replace("đ", "d").replace("Đ", "d")
    text = unicodedata.normalize("NFKD", text)
    out = []
    for ch in text:
        if unicodedata.combining(ch):
            continue
        ch = ch.lower()
        out.append(ch if ("a" <= ch <= "z" or "0" <= ch <= "9") else " ")
    tokens = "".join(out).split()
    return " ".join("acid" if t == "axit" else t for t in tokens)


CODE_TOKEN = re.compile(r"^(?:e|ins|sin)?\d{3,4}[a-z]{0,5}$")


def usable_name(name):
    tokens = name.split()
    if not tokens or not any(c.isalpha() for c in name):
        return False
    if any(CODE_TOKEN.match(t) for t in tokens):
        return False
    if tokens[0] in ("cas", "ci", "einecs") or tokens[:2] == ["c", "i"]:
        return False
    if all(len(t) <= 1 or t.isdigit() for t in tokens):
        return False
    if len(tokens) == 1 and len(name) < 4 and name not in SHORT_OK:
        return False
    return name not in STOP


def split_values(text):
    return [v.replace("\\,", ",").strip() for v in re.split(r"(?<!\\),", text) if v.strip()]


def ident(name):
    return re.sub(r"[^a-z0-9]", "", name.lower())


PROPERTY = re.compile(r"^([a-z_0-9]+):([a-z]{2}): ?(.*)$")
LANGUAGE = re.compile(r"^([a-z]{2}): ?(.*)$")
PARENT = re.compile(r"^< ?en: ?(.*)$")
VARIANT = re.compile(r"^(e\d{3,4})([a-h])?([ivx]+)?$")


def parse():
    entries = {}
    for block in re.split(r"\n\s*\n", SOURCE.read_text(encoding="utf-8")):
        names, props, parents = {}, {}, []
        for line in block.splitlines():
            if not line or line.startswith("#"):
                continue
            if m := PARENT.match(line):
                parents.append(ident(m.group(1)))
            elif m := PROPERTY.match(line):
                if m.group(2) == "en":
                    props[m.group(1)] = m.group(3).strip()
            elif m := LANGUAGE.match(line):
                names[m.group(1)] = split_values(m.group(2))
        canonical = (names.get("en") or names.get("xx") or [None])[0]
        if not canonical or not re.match(r"^E\d{3,4}", canonical):
            continue
        entries[ident(canonical)] = {"canonical": canonical, "names": names, "props": props, "parents": parents}
    # The taxonomy doesn't always say that E150a belongs to E150, or E170(i) to E170.
    for key, e in entries.items():
        if m := VARIANT.match(key):
            base = m.group(1) + (m.group(2) or "") if m.group(3) else m.group(1)
            if base != key and base in entries and base not in e["parents"]:
                e["parents"].append(base)
    return entries


def groups(value):
    return [g.strip().removeprefix("en:") for g in (value or "").split(",") if g.strip() and "no-group" not in g]


def build():
    entries = parse()
    for key in VI:
        if key not in entries:
            print(f"warning: {key} in VI is not in the taxonomy", file=sys.stderr)

    # What EFSA said about each additive itself.
    own = {}
    for key, e in entries.items():
        p = e["props"]
        risk = p.get("efsa_evaluation_overexposure_risk", "").removeprefix("en:")
        level = {"high": "limit", "moderate": "caution", "no": "ok"}.get(risk)
        reason = f"efsa_{risk}" if level else None
        if key in EU_NOT_AUTHORISED:
            level, reason = "limit", "eu_not_authorised"
        elif key in EU_WARNING:
            level, reason = "limit", "eu_warning"
        own[key] = {
            "level": level,
            "reason": reason,
            "mean": groups(p.get("efsa_evaluation_exposure_mean_greater_than_adi")),
            "high": groups(p.get("efsa_evaluation_exposure_95th_greater_than_adi")),
            "title": p.get("efsa_evaluation"),
            "url": p.get("efsa_evaluation_url") or p.get("efsa"),
            "year": (re.search(r"\d{4}", p.get("efsa_evaluation_date", "")) or [None])[0],
        }

    def ancestors(key):
        seen, queue = [], list(entries[key]["parents"])
        while queue:
            parent = queue.pop(0)
            if parent in entries and parent not in seen:
                seen.append(parent)
                queue += entries[parent]["parents"]
        return seen

    # Names: English, the language-neutral ones and Vietnamese. A name shared by several
    # additives goes to the most general one (the parent), or to nobody if they are unrelated.
    wanted = {}
    for key, e in entries.items():
        raw = e["names"].get("en", [])[1:] + e["names"].get("xx", []) + e["names"].get("vi", [])[1:] + VI.get(key, [])
        wanted[key] = list(dict.fromkeys(n for n in (fold(r) for r in raw) if usable_name(n)))
    owners = {}
    for key, names in wanted.items():
        for name in names:
            owners.setdefault(name, []).append(key)
    dropped = []
    for name, keys in owners.items():
        if len(keys) == 1:
            continue
        top = [k for k in keys if all(k == other or k in ancestors(other) for other in keys)]
        ours = [k for k in keys if name in (fold(v) for v in VI.get(k, []))]
        if len(ours) == 1:
            owners[name] = ours
        elif top:
            owners[name] = [top[0]]
        else:
            owners[name] = []
            dropped.append((name, keys))

    additives = []
    for key in sorted(entries, key=lambda k: (int(re.search(r"\d+", k).group()), k)):
        e = entries[key]
        p = e["props"]
        info, inherited = own[key], None
        if info["level"] is None:
            for parent in ancestors(key):
                if own[parent]["level"] is not None:
                    info, inherited = own[parent], parent
                    break
        english = e["names"].get("en", [])
        off_vi = e["names"].get("vi", [])
        item = {
            "id": key,
            "code": e["canonical"][1:],
            "en": english[1] if len(english) > 1 else None,
            "vi": VI[key][0] if key in VI else (off_vi[1] if len(off_vi) > 1 else None),
            "classes": [c.strip().removeprefix("en:") for c in p.get("additives_classes", "").split(",") if c.strip()],
            "level": info["level"] or "unknown",
            "reason": info["reason"],
            "inheritedFrom": inherited,
            "exceedsMean": info["mean"] if info["reason"] and info["reason"].startswith("efsa") else [],
            "exceedsHigh": info["high"] if info["reason"] and info["reason"].startswith("efsa") else [],
            "efsaTitle": info["title"] if info["reason"] else None,
            "efsaURL": info["url"] if info["reason"] else None,
            "efsaYear": info["year"] if info["reason"] else None,
            "parents": [x for x in e["parents"] if x in entries],
            "vegan": p.get("vegan"),
            "vegetarian": p.get("vegetarian"),
            "names": [n for n in wanted[key] if owners.get(n) == [key]],
        }
        additives.append({k: v for k, v in item.items() if v not in (None, [], "")})

    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    document = {
        "source": "Open Food Facts additives taxonomy, https://world.openfoodfacts.org",
        "license": "Open Database License (ODbL) 1.0",
        "sourceDate": SOURCE_DATE,
        "notes": "Levels: limit = EFSA exposure above the acceptable intake at average consumption, or an EU "
                 "label warning, or no longer authorised in the EU. caution = above only for high consumers. "
                 "ok = no group above. unknown = no exposure assessment recorded. Vietnamese names and the "
                 "two EU flags were added by Soi.",
        "additives": additives,
    }
    OUTPUT.write_text(json.dumps(document, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")

    levels = {}
    for a in additives:
        levels[a["level"]] = levels.get(a["level"], 0) + 1
    print(f"{len(additives)} additives -> {OUTPUT.relative_to(ROOT)} ({OUTPUT.stat().st_size // 1024} KB)")
    print("levels:", levels)
    print("names:", sum(len(a.get('names', [])) for a in additives), "| with Vietnamese name:", sum(1 for a in additives if a.get('vi')))
    print("names shared by unrelated additives, not matched:", len(dropped))
    if "--review" in sys.argv:
        for name, keys in sorted(dropped):
            print("  shared:", name, keys)
        single = sorted({n for a in additives for n in a.get("names", []) if " " not in n})
        print("single-word names:", ", ".join(single))


if __name__ == "__main__":
    build()