Upload pii_hybrid_decode.py with huggingface_hub

pii_hybrid_decode.py

@@ -70,6 +70,19 @@ _SNAP = {
 }
 _AUTH_TYPES = {t for t, _, _ in _AUTH}
 
+# boundary-snap: the token classifier drops leading/trailing sub-tokens of Latin-script
+# words ('Ibuprofen'->'profen'). Extend a span across contiguous Latin word-chars to
+# complete the partial word(s). Latin-only -> CJK/Arabic left untouched (no over-extend).
+_LAT = re.compile(r"[0-9A-Za-zÀ-ÖØ-öø-ÿ]")
+def _snap_word(text, s, e):
+    n = len(text)
+    while s > 0 and _LAT.match(text[s - 1]) and _LAT.match(text[s]): s -= 1
+    while e < n and _LAT.match(text[e]) and _LAT.match(text[e - 1]): e += 1
+    return s, e
+_SWIFT_CUE = re.compile(r"(?i)(swift|bic)")
+def _swift_ok(text, s, val):  # kill ALL-CAPS-word false BICs (PARTICULARS, CONFIDENTIAL)
+    return bool(re.search(r"\d", val)) or bool(_SWIFT_CUE.search(text[max(0, s - 12):s]))
+
 def hybrid_spans(text: str, model_spans: list[dict]) -> list[dict]:
     """model_spans: [{'start','end','type'}...] from the token classifier. Returns the
     hybrid-decoded spans (dicts with start/end/type/text)."""
@@ -81,6 +94,7 @@ def hybrid_spans(text: str, model_spans: list[dict]) -> list[dict]:
             s, e = mm.start(), mm.end()
             if any(claimed[s:e]): continue
             if val and not val(mm.group(0)): continue
+            if t == "financial.swift_bic" and not _swift_ok(text, s, mm.group(0)): continue
             for i in range(s, e): claimed[i] = True
             auth.append({"start": s, "end": e, "type": t, "text": mm.group(0)})
     # 2. model spans for non-AUTH types; SNAP types expand to overlapping regex match
@@ -97,14 +111,15 @@ def hybrid_spans(text: str, model_spans: list[dict]) -> list[dict]:
             if snap:
                 out.append({"start": snap.start(), "end": snap.end(), "type": t,
                             "text": text[snap.start():snap.end()]}); continue
-        out.append({"start": m["start"], "end": m["end"], "type": t,
-                    "text": text[m["start"]:m["end"]]})
+        ss, ee = _snap_word(text, m["start"], m["end"])  # complete partial Latin words
+        out.append({"start": ss, "end": ee, "type": t, "text": text[ss:ee]})
     out.extend(auth)
     seen, uniq = set(), []
     for sp in sorted(out, key=lambda s: (s["start"], s["end"])):
         k = (sp["start"], sp["end"], sp["type"])
-        if k not in seen:
-            seen.add(k); uniq.append(sp)
+        if k in seen: continue
+        if len(text[sp["start"]:sp["end"]].strip()) < 3: continue  # drop fragments
+        seen.add(k); uniq.append(sp)
     return uniq
 
 def model_spans(text: str, tok, model):