#!/usr/bin/env python3 """Deterministic benefit tagging for the free-text `benefits` field of D2 forms. No model calls: the text is split into clauses and each clause is matched against keyword rules on its diacritic-folded form. Every tag therefore traces back to a clause (`python3 app/benefits.py` prints the audit table). Tags (closed vocabulary, see BENEFITS): stipend money paid to the intern: lương, phụ cấp/trợ cấp, hỗ trợ chi phí thực tập, scholarship... meal lunch / dinner / meal vouchers / canteen transport commuting allowance, shuttle bus, Grab parking motorbike parking housing accommodation support snacks coffee, tea, snacks, pantry equipment laptop / devices / tools / uniform / workspace infrastructure training training, courses, mentoring, workshops job-offer chance of a full-time / part-time offer team team building, parties, sports, trips bonus bonuses, prizes, gifts insurance insurance certificate completion certificate / company stamp flexible flexible hours none form explicitly states there are no benefits """ import re BENEFITS = [ # (slug, label) in display order ("stipend", "Stipend / salary"), ("meal", "Lunch / meals"), ("transport", "Transport"), ("parking", "Parking"), ("housing", "Housing"), ("snacks", "Snacks & drinks"), ("equipment", "Equipment & workspace"), ("training", "Training & mentoring"), ("job-offer", "Job offer prospects"), ("team", "Team activities"), ("bonus", "Bonus & gifts"), ("insurance", "Insurance"), ("certificate", "Certificate / stamp"), ("flexible", "Flexible hours"), ("none", "No benefits stated"), ] # Rules run on the folded clause (lowercase, no diacritics, đ -> d, punctuation -> space). RULES = { "meal": r"\ban (trua|sang|toi)\b|\bbua (trua|an|sang|toi)\b|\bsuat an\b|\bphieu an\b|\b(tien )?com\b|\blunch\b|\bmeals?\b|\bcanteen\b|\bcan tin\b", "transport": r"\bdi lai\b|\bdua don\b|\bgrab\b|\btravel(l)?ing\b|\bairfare\b|\bxe buyt\b|\bxang xe\b", "parking": r"\bgui xe\b|\bgiu xe\b|\bphi xe\b|\bbai xe\b|\bparking\b", "housing": r"\bcho o\b|\bnha o\b|(? VND per month ------------------------------------------------------- # Approximate conversion constants; every converted figure is flagged `approx`. FX_VND = {"vnd": 1, "usd": 25000, "sgd": 19000, "ntd": 800} DAYS_PER_MONTH, HOURS_PER_MONTH, WEEKS_PER_MONTH, DEFAULT_WEEKS = 22, 176, 4.33, 8 NUM = r"\d{1,3}(?:[.,]\d{3})+|\d+(?:[.,]\d+)?" UNIT_RX = r"(?:tri[eệ]u|tr\b|k\b|ngh[iì]n|ng[aà]n|vn[dđ]|[đd]\b|đ[oồ]ng|usd|sgd|nt\$|ntd|\$)" MONEY = re.compile(r"(?P
vn[dđ]|usd|sgd|nt\$|ntd|\$)?\s*(?P" + NUM + r")\s*(?Ptri[eệ]u|tr\b|k\b|ngh[iì]n|ng[aà]n)?\s*(?Pvn[dđ]|[đd]\b|đ[oồ]ng|usd|sgd|nt\$|ntd|\$)?", re.I)
RANGE_SEP = re.compile(r"^\s*(?:-|–|—|~|đến|den|to)\s*$", re.I)


def _num(s):
    if re.fullmatch(r"\d{1,3}(?:[.,]\d{3})+", s):
        return float(re.sub(r"[.,]", "", s))
    return float(s.replace(",", "."))


def _amounts(cl):
    """[(value, currency, end_pos)] money mentions in a clause (number needs a currency or triệu/k unit,
    which a range's first number may inherit from its second)."""
    items = []
    for m in MONEY.finditer(cl):
        cur = (m.group("cur") or m.group("pre") or "").lower()
        mult = (m.group("mult") or "").lower()
        items.append({"m": m, "cur": cur, "mult": mult, "v": _num(m.group("num"))})
    for i, it in enumerate(items):  # "3,000,000 – 6,000,000 VND", "8-10 triệu": inherit unit across a range
        if not (it["cur"] or it["mult"]) and i + 1 < len(items):
            nxt = items[i + 1]
            if RANGE_SEP.match(cl[it["m"].end():nxt["m"].start()]) and (nxt["cur"] or nxt["mult"]):
                it["cur"], it["mult"] = nxt["cur"], nxt["mult"] if it["v"] < 1000 else ""
    out = []
    for it in items:
        if not (it["cur"] or it["mult"]):
            if it["v"] < 10000:   # bare number: only a plausible VND figure ("3,000,000 – 6,000,000/ tháng")
                continue
            it["cur"] = "vnd"
        cur = it["cur"]
        cur = "usd" if cur in ("usd", "$") else "sgd" if cur == "sgd" else "ntd" if cur in ("nt$", "ntd") else "vnd"
        v = it["v"] * {"": 1, "k": 1e3, "nghìn": 1e3, "nghin": 1e3, "ngàn": 1e3, "ngan": 1e3}.get(it["mult"], 1e6)
        out.append({"v": v, "cur": cur, "end": it["m"].end(), "start": it["m"].start()})
    return out


def _basis(cl, f, a, nxt_start):
    """How an amount is paid: month | day | hour | week | total | unspecified (looks right after the number)."""
    after = fold_local(cl[a["end"]:nxt_start or a["end"] + 25])[:25]
    if re.search(r"\b(thang|month|th)\b", after): return "month"
    if re.search(r"\b(ngay|day)\b", after): return "day"
    if re.search(r"\b(gio|hour|hr)\b", after): return "hour"
    if re.search(r"\b(tuan|week)\b", after): return "week"
    if re.search(r"\b(hang thang|monthly|moi thang|per month)\b", f): return "month"
    if re.search(r"\btong cong\b|\btotal\b|\bhet chuong trinh\b|\bsau khi xong\b|\bscholarship support\b|\bsupport of\b", f): return "total"
    return "unspecified"


def fold_local(s):
    import unicodedata
    s = unicodedata.normalize("NFD", (s or "").lower().replace("đ", "d"))
    return re.sub(r"[^a-z0-9]+", " ", "".join(c for c in s if unicodedata.category(c) != "Mn")).strip()


def stipend_amount(text, fold, weeks=None):
    """Parse the internship stipend into VND/month.

    Returns None when no money figure is stated, else
    {"min", "max": VND/month, "text": clause, "basis": how it was paid, "approx": bool}.
    `weeks` is the internship length used to spread one-off totals (default 8, the forms' stated minimum).
    `approx` is true whenever a conversion was needed (currency, day/hour/total) or the period was assumed.
    """
    found = []
    for cl, hits in _clause_hits((text or "").strip(), fold):
        if "stipend" not in hits:
            continue
        f = fold(cl)
        ams = _amounts(cl)
        if not ams:
            continue
        # a range "a – b" is two numbers sharing one basis; otherwise each number stands alone
        per = []
        for i, a in enumerate(ams):
            nxt = ams[i + 1]["start"] if i + 1 < len(ams) else None
            per.append((a, _basis(cl, f, a, nxt)))
        # an amount without its own period inherits the period of the following range end
        for i in range(len(per) - 1):
            if per[i][1] == "unspecified" and RANGE_SEP.match(cl[per[i][0]["end"]:per[i + 1][0]["start"]]):
                per[i] = (per[i][0], per[i + 1][1])
        # prefer a number explicitly stated per month (e.g. "23,000/giờ (tương đương 4,048,000/tháng)")
        if any(b == "month" for _, b in per):
            per = [(a, b) for a, b in per if b == "month"]
        for a, b in per:
            vnd = a["v"] * FX_VND[a["cur"]]
            approx = a["cur"] != "vnd"
            if b == "day": vnd, approx = vnd * DAYS_PER_MONTH, True
            elif b == "hour": vnd, approx = vnd * HOURS_PER_MONTH, True
            elif b == "week": vnd, approx = vnd * WEEKS_PER_MONTH, True
            elif b == "total":
                wk = re.search(r"(\d+)\s*(?:tu[aầ]n|weeks?)", cl, re.I)
                vnd, approx = vnd / ((int(wk.group(1)) if wk else weeks or DEFAULT_WEEKS) / WEEKS_PER_MONTH), True
            elif b == "unspecified":
                approx = True  # assumed per month
            found.append((vnd, approx, b, cl))
    if not found:
        return None
    lo, hi = min(x[0] for x in found), max(x[0] for x in found)
    return {"min": round(lo, -3), "max": round(hi, -3), "text": found[0][3],
            "basis": sorted({x[2] for x in found}), "approx": any(x[1] for x in found)}


if __name__ == "__main__":  # audit: every company's latest benefits text with its tags
    import json, sys, unicodedata
    from pathlib import Path

    def fold(s):
        s = unicodedata.normalize("NFD", (s or "").lower().replace("đ", "d"))
        return re.sub(r"[^a-z0-9]+", " ", "".join(c for c in s if unicodedata.category(c) != "Mn")).strip()

    d2 = json.load(open(Path(__file__).resolve().parent.parent / "forms/extracted/d2.json"))
    latest = {}
    for f in sorted((f for f in d2 if f.get("fields")), key=lambda f: f.get("form_date") or "", reverse=True):
        b = f["fields"].get("benefits")
        if b and f["company_shortname"] not in latest:
            latest[f["company_shortname"]] = b
    from collections import Counter
    cnt = Counter()
    for name, b in sorted(latest.items()):
        t = tag_benefits(b, fold)
        cnt.update(t.keys())
        if "--amounts" in sys.argv and "stipend" in t:
            a = stipend_amount(b, fold)
            print(f"{name:45} {('%.1fM-%.1fM' % (a['min']/1e6, a['max']/1e6)) if a else '-':14} {'~' if a and a['approx'] else ' '} {a['basis'] if a else ''}  | {a['text'][:70] if a else ''}")
        if "-v" in sys.argv:
            print(f"{name}\n   {str(b)[:300]!r}\n   -> {sorted(t)}")
    print(f"{len(latest)} companies with benefits text")
    for slug, lab in BENEFITS:
        print(f"  {slug:12} {cnt[slug]:3}  {lab}")