Make NEL pipeline generic for historical entity linking

.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/copilot.data.migration.agent.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AgentMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

.idea/historical-nel.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.14 (historical-nel)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,50 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="73" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="false" level="WARNING" enabled_by_default="false">
+      <option name="ignoredPackages">
+        <list>
+          <option value="beautifulsoup4" />
+          <option value="pycld2" />
+          <option value="colorama" />
+          <option value="epub" />
+          <option value="evaluate" />
+          <option value="huggingface_hub" />
+          <option value="dask" />
+          <option value="sentence_transformers" />
+          <option value="jsonschema" />
+          <option value="importlib_resources" />
+          <option value="sentence-transformers" />
+          <option value="jsonlines" />
+          <option value="langdetect" />
+          <option value="pandas" />
+          <option value="pybloomfiltermmap3" />
+          <option value="pysbd" />
+          <option value="python-dotenv" />
+          <option value="PyYAML" />
+          <option value="scikit_learn" />
+          <option value="spacy" />
+          <option value="tiktoken" />
+          <option value="tqdm" />
+          <option value="trl" />
+          <option value="boto3" />
+          <option value="s3fs" />
+          <option value="jiwer" />
+          <option value="dkpro-cassis" />
+          <option value="dvc" />
+          <option value="dvc_s3" />
+          <option value="chonkie" />
+          <option value="levenshtein" />
+          <option value="pillow" />
+          <option value="openai" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyTupleAssignmentBalanceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.14 (historical-nel)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.14 (historical-nel)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/historical-nel.iml" filepath="$PROJECT_DIR$/.idea/historical-nel.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

generic_nel.py

@@ -1,6 +1,5 @@
 from transformers import Pipeline
 import requests
-import torch
 import time
 import os
 import numpy as np
@@ -16,16 +15,14 @@ logger = logging.getLogger("nel_wikidata")
 
 NEL_MODEL = "nel-mgenre-multilingual"
 
-HEADERS = {
-    "User-Agent": "NEL-HF-Space/0.1 (contact: emanuela.boros@univ-lr.fr)"
-}
+HEADERS = {"User-Agent": "NEL-HF-Space/0.1 (contact: emanuela.boros@univ-lr.fr)"}
 
 WIKIMEDIA_QPS = float(os.getenv("WIKIMEDIA_QPS", "3.0"))  # total across ALL processes
 _MIN_INTERVAL = 1.0 / max(WIKIMEDIA_QPS, 0.1)
 
 _LOCK = FileLock(os.getenv("WIKIMEDIA_RATELOCK", "/tmp/wikimedia_rate.lock"))
 _TS_FILE = os.getenv("WIKIMEDIA_RATETS", "/tmp/wikimedia_rate.ts")
-        
+
 requests_cache.install_cache(
     cache_name=os.getenv("WIKIMEDIA_CACHE", "wikimedia_cache"),
     backend="sqlite",
@@ -44,7 +41,7 @@ _retry = Retry(
     backoff_factor=float(os.getenv("WIKIMEDIA_BACKOFF", "1.5")),
     status_forcelist=(429, 500, 502, 503, 504),
     allowed_methods=frozenset(["GET"]),
-    respect_retry_after_header=False,   # <-- IMPORTANT
+    respect_retry_after_header=False,  # <-- IMPORTANT
     raise_on_status=False,
 )
 
@@ -52,6 +49,7 @@ _adapter = HTTPAdapter(max_retries=_retry, pool_connections=50, pool_maxsize=50)
 _session.mount("https://", _adapter)
 _session.mount("http://", _adapter)
 
+
 def safe_get(url, params, retries=3, backoff=2):
     for attempt in range(retries):
         try:
@@ -59,7 +57,7 @@ def safe_get(url, params, retries=3, backoff=2):
             if resp.status_code == 200:
                 return resp
             elif resp.status_code in (403, 429):  # forbidden or rate limited
-                wait = backoff ** attempt
+                wait = backoff**attempt
                 print(f"Retrying {url} in {wait}s (status={resp.status_code})")
                 time.sleep(wait)
                 continue
@@ -67,7 +65,7 @@ def safe_get(url, params, retries=3, backoff=2):
                 resp.raise_for_status()
         except Exception as e:
             print(f"Request error: {e}")
-            time.sleep(backoff ** attempt)
+            time.sleep(backoff**attempt)
     return None
 
 
@@ -106,6 +104,7 @@ def get_claim_strings(claims, prop):
             values.append(v["time"])
     return values
 
+
 def extract_datavalue(claim):
     try:
         return claim["mainsnak"]["datavalue"]["value"]
@@ -135,6 +134,7 @@ def get_claim_string_values(claims, prop):
                 vals.append(v["time"])
     return vals
 
+
 def get_claim_coordinates(claims, prop="P625"):
     """
     Return coordinates from Wikidata if present.
@@ -143,11 +143,13 @@ def get_claim_coordinates(claims, prop="P625"):
     for c in claims.get(prop, []):
         v = extract_datavalue(c)
         if isinstance(v, dict) and "latitude" in v and "longitude" in v:
-            coords.append({
-                "latitude": v["latitude"],
-                "longitude": v["longitude"],
-                "precision": v.get("precision")
-            })
+            coords.append(
+                {
+                    "latitude": v["latitude"],
+                    "longitude": v["longitude"],
+                    "precision": v.get("precision"),
+                }
+            )
     return coords
 
 
@@ -227,11 +229,15 @@ def safe_get_old(url, params, timeout=10):
             # Even with Retry-After handling, sometimes you still get here.
             ra = resp.headers.get("Retry-After")
             sleep = float(ra) if (ra and ra.isdigit()) else (1.0 + random.random())
-            logger.warning(f"[WIKI] 429 rate limited. retry-after={ra} sleeping={sleep:.2f}s url={url}")
+            logger.warning(
+                f"[WIKI] 429 rate limited. retry-after={ra} sleeping={sleep:.2f}s url={url}"
+            )
             time.sleep(sleep)
             return resp  # caller may treat as failure; Retry already happened.
 
-        logger.warning(f"[WIKI] HTTP {resp.status_code} url={url} params={_short(params)}")
+        logger.warning(
+            f"[WIKI] HTTP {resp.status_code} url={url} params={_short(params)}"
+        )
         return resp
     except Exception as e:
         logger.exception(f"[WIKI] Request error: {e} url={url}")
@@ -242,187 +248,33 @@ def _short(params: dict, maxlen: int = 120) -> str:
     s = str(params)
     return s if len(s) <= maxlen else s[:maxlen] + "..."
 
-# def get_wikidata_details(qid, language="en"):
-#     """
-#     Return richer Wikidata + Wikipedia details for a QID.
-#     """
-#     if not qid or qid == "NIL":
-#         return {
-#             "wikidata_label": "NIL",
-#             "wikidata_description": None,
-#             "wikidata_aliases": [],
-#             "wikidata_summary": None,
-#             "wikipedia_title": "NIL",
-#             "wikipedia_url": "None",
-#             "wikipedia_summary": None,
-#             "entity_type": "UNK",
-#             "instance_of": [],
-#             "occupations": [],
-#             "country": [],
-#             "date_of_birth": None,
-#             "date_of_death": None,
-#             "coordinates": [],
-#         }
-
-#     # ---- Wikidata entity data ----
-#     wd_url = "https://www.wikidata.org/w/api.php"
-#     wd_params = {
-#         "action": "wbgetentities",
-#         "format": "json",
-#         "ids": qid,
-#         "props": "labels|descriptions|aliases|sitelinks/urls|claims",
-#         "languages": language,
-#         "sitefilter": f"{language}wiki",
-#     }
-
-#     resp = safe_get(wd_url, wd_params)
-#     if not resp:
-#         return {
-#             "wikidata_label": qid,
-#             "wikidata_description": None,
-#             "wikidata_aliases": [],
-#             "wikidata_summary": None,
-#             "wikipedia_title": "NIL",
-#             "wikipedia_url": "None",
-#             "wikipedia_summary": None,
-#             "entity_type": "UNK",
-#             "instance_of": [],
-#             "occupations": [],
-#             "country": [],
-#             "date_of_birth": None,
-#             "date_of_death": None,
-#             "coordinates": [],
-#         }
-
-#     try:
-#         data = resp.json()
-#         entity = data["entities"][qid]
-
-#         labels = entity.get("labels", {})
-#         descriptions = entity.get("descriptions", {})
-#         aliases = entity.get("aliases", {})
-#         sitelinks = entity.get("sitelinks", {})
-#         claims = entity.get("claims", {})
-
-#         wikidata_label = (
-#             labels.get(language, {}).get("value")
-#             or labels.get("en", {}).get("value")
-#             or qid
-#         )
-
-#         wikidata_description = (
-#             descriptions.get(language, {}).get("value")
-#             or descriptions.get("en", {}).get("value")
-#         )
-
-#         wikidata_aliases = [
-#             a["value"]
-#             for a in aliases.get(language, []) or aliases.get("en", [])
-#         ]
-
-#         if f"{language}wiki" in sitelinks:
-#             wikipedia_title = sitelinks[f"{language}wiki"]["title"]
-#             wikipedia_url = sitelinks[f"{language}wiki"]["url"]
-#         else:
-#             wikipedia_title, wikipedia_url = "NIL", "None"
-
-#         # coarse type
-#         p31_qids = get_claim_ids(claims, "P31")
-#         entity_type = resolve_type_from_wikidata(p31_qids) if p31_qids else "UNK"
-
-#         # useful extra claims
-#         occupation_qids = get_claim_ids(claims, "P106")
-#         country_qids = get_claim_ids(claims, "P17")
-#         instance_of_qids = get_claim_ids(claims, "P31")
-
-#         date_of_birth = next(iter(get_claim_strings(claims, "P569")), None)
-#         date_of_death = next(iter(get_claim_strings(claims, "P570")), None)
-#         coordinates = get_claim_coordinates(claims, "P625")
-
-#         label_qids = list(set(occupation_qids + country_qids + instance_of_qids))
-#         resolved_labels = get_entity_labels(label_qids, language=language)
-
-#         occupations = [resolved_labels.get(q, q) for q in occupation_qids]
-#         country = [resolved_labels.get(q, q) for q in country_qids]
-#         instance_of = [resolved_labels.get(q, q) for q in instance_of_qids]
-
-#         # ---- Wikipedia summary ----
-#         wikipedia_summary = None
-#         if wikipedia_title != "NIL":
-#             wp_summary_url = f"https://{language}.wikipedia.org/api/rest_v1/page/summary/{wikipedia_title}"
-#             wp_resp = safe_get(wp_summary_url, params={})
-#             if wp_resp and wp_resp.status_code == 200:
-#                 try:
-#                     wp_data = wp_resp.json()
-#                     wikipedia_summary = wp_data.get("extract")
-#                 except Exception:
-#                     wikipedia_summary = None
-
-#         return {
-#             "wikidata_label": wikidata_label,
-#             "wikidata_description": wikidata_description,
-#             "wikidata_aliases": wikidata_aliases,
-#             "wikidata_summary": wikidata_description,  # practical summary proxy
-#             "wikipedia_title": wikipedia_title,
-#             "wikipedia_url": wikipedia_url,
-#             "wikipedia_summary": wikipedia_summary,
-#             "entity_type": entity_type,
-#             "instance_of": instance_of,
-#             "occupations": occupations,
-#             "country": country,
-#             "date_of_birth": date_of_birth,
-#             "date_of_death": date_of_death,
-#             "coordinates": coordinates,
-#         }
-
-    # except Exception as e:
-    #     print(f"Error parsing details for {qid}: {e}")
-    #     return {
-    #         "wikidata_label": qid,
-    #         "wikidata_description": None,
-    #         "wikidata_aliases": [],
-    #         "wikidata_summary": None,
-    #         "wikipedia_title": "NIL",
-    #         "wikipedia_url": "None",
-    #         "wikipedia_summary": None,
-    #         "entity_type": "UNK",
-    #         "instance_of": [],
-    #         "occupations": [],
-    #         "country": [],
-    #         "date_of_birth": None,
-    #         "date_of_death": None,
-    #         "coordinates": [],
-    #     }
-
-
-# GLOBALISE
+
+# Generic Wikidata/Wikipedia enrichment
 def get_wikidata_details(qid, language="en"):
     """
-    Wikidata/Wikipedia enrichment tailored to GLOBALISE-style commodity / materia medica entities.
+    Generic Wikidata/Wikipedia enrichment for historical named entity linking.
+    Works for persons, locations, organizations, and other entities.
     """
+    empty = {
+        "wikidata_label": qid or "NIL",
+        "wikidata_description": None,
+        "wikidata_aliases": [],
+        "wikipedia_title": "NIL",
+        "wikipedia_url": "None",
+        "wikipedia_summary": None,
+        "entity_type": "UNK",
+        "instance_of": [],
+        "subclass_of": [],
+        "date_of_birth": [],
+        "date_of_death": [],
+        "coordinates": [],
+        "country": [],
+        "occupation": [],
+    }
+
     if not qid or qid == "NIL":
-        return {
-            "wikidata_label": "NIL",
-            "wikidata_description": None,
-            "wikidata_aliases": [],
-            "wikipedia_title": "NIL",
-            "wikipedia_url": "None",
-            "wikipedia_summary": None,
-            "entity_type": "other",
-            "instance_of": [],
-            "subclass_of": [],
-            "scientific_name": [],
-            "source_organism": [],
-            "parent_taxon": [],
-            "taxon_rank": [],
-            "part_used": [],
-            "derived_from": "unknown",
-            "native_to": [],
-            "country_of_origin": [],
-            "use_categories": [],
-            "medicinal_uses": [],
-            "has_effect": [],
-        }
+        empty["wikidata_label"] = "NIL"
+        return empty
 
     wd_url = "https://www.wikidata.org/w/api.php"
     wd_params = {
@@ -436,32 +288,11 @@ def get_wikidata_details(qid, language="en"):
 
     resp = safe_get(wd_url, wd_params)
     if not resp:
-        return {
-            "wikidata_label": qid,
-            "wikidata_description": None,
-            "wikidata_aliases": [],
-            "wikipedia_title": "NIL",
-            "wikipedia_url": "None",
-            "wikipedia_summary": None,
-            "entity_type": "other",
-            "instance_of": [],
-            "subclass_of": [],
-            "scientific_name": [],
-            "source_organism": [],
-            "parent_taxon": [],
-            "taxon_rank": [],
-            "part_used": [],
-            "derived_from": "unknown",
-            "native_to": [],
-            "country_of_origin": [],
-            "use_categories": [],
-            "medicinal_uses": [],
-            "has_effect": [],
-        }
+        return empty
 
     try:
         data = resp.json()
-        entity = data["entities"][qid]
+        entity = data.get("entities", {}).get(qid, {})
 
         labels = entity.get("labels", {})
         descriptions = entity.get("descriptions", {})
@@ -475,10 +306,9 @@ def get_wikidata_details(qid, language="en"):
             or qid
         )
 
-        wikidata_description = (
-            descriptions.get(language, {}).get("value")
-            or descriptions.get("en", {}).get("value")
-        )
+        wikidata_description = descriptions.get(language, {}).get(
+            "value"
+        ) or descriptions.get("en", {}).get("value")
 
         wikidata_aliases = [
             a["value"] for a in (aliases.get(language) or aliases.get("en") or [])
@@ -503,60 +333,21 @@ def get_wikidata_details(qid, language="en"):
 
         instance_of_qids = get_claim_entity_ids(claims, "P31")
         subclass_of_qids = get_claim_entity_ids(claims, "P279")
-        parent_taxon_qids = get_claim_entity_ids(claims, "P171")
-        taxon_rank_qids = get_claim_entity_ids(claims, "P105")
-        country_of_origin_qids = get_claim_entity_ids(claims, "P495")
-        use_qids = get_claim_entity_ids(claims, "P366")
-        medicinal_use_qids = get_claim_entity_ids(claims, "P2176")
-
-        scientific_name = get_claim_string_values(claims, "P225")
-
-        label_qids = list(set(
-            instance_of_qids
-            + subclass_of_qids
-            + parent_taxon_qids
-            + taxon_rank_qids
-            + country_of_origin_qids
-            + use_qids
-            + medicinal_use_qids
-        ))
+        occupation_qids = get_claim_entity_ids(claims, "P106")
+        country_qids = get_claim_entity_ids(claims, "P17")
+
+        label_qids = list(
+            set(instance_of_qids + subclass_of_qids + occupation_qids + country_qids)
+        )
+
         label_map = get_entity_labels(label_qids, language=language)
 
         instance_of = [label_map.get(q, q) for q in instance_of_qids]
         subclass_of = [label_map.get(q, q) for q in subclass_of_qids]
-        parent_taxon = [label_map.get(q, q) for q in parent_taxon_qids]
-        taxon_rank = [label_map.get(q, q) for q in taxon_rank_qids]
-        country_of_origin = [label_map.get(q, q) for q in country_of_origin_qids]
-        medicinal_uses = [label_map.get(q, q) for q in medicinal_use_qids]
-
-        combined_text = " ".join(filter(None, [
-            wikidata_label,
-            wikidata_description,
-            wikipedia_summary,
-            " ".join(instance_of),
-            " ".join(subclass_of),
-        ]))
-
-        use_categories = infer_use_categories(combined_text)
-        derived_from = infer_origin_type(instance_of, subclass_of, wikidata_description, wikipedia_summary)
-
-        entity_type = resolve_globalise_type(
-            instance_of_labels=instance_of,
-            subclass_of_labels=subclass_of,
-            description=wikidata_description
-        )
+        occupation = [label_map.get(q, q) for q in occupation_qids]
+        country = [label_map.get(q, q) for q in country_qids]
 
-        # weak heuristic fields derived from text
-        part_used = []
-        text_lower = combined_text.lower()
-        for part in ["root", "seed", "fruit", "flower", "pod", "bark", "wood", "resin", "gum", "oil", "leaf", "bulb"]:
-            if part in text_lower:
-                part_used.append(part)
-
-        has_effect = []
-        for effect in ["laxative", "styptic", "febrifuge", "aromatic", "fragrant", "bitter", "blood-staunching"]:
-            if effect in text_lower:
-                has_effect.append(effect)
+        entity_type = resolve_type_from_wikidata(instance_of_qids)
 
         return {
             "wikidata_label": wikidata_label,
@@ -565,117 +356,21 @@ def get_wikidata_details(qid, language="en"):
             "wikipedia_title": wikipedia_title,
             "wikipedia_url": wikipedia_url,
             "wikipedia_summary": wikipedia_summary,
-
             "entity_type": entity_type,
             "instance_of": instance_of,
             "subclass_of": subclass_of,
-
-            "scientific_name": scientific_name,
-            "source_organism": parent_taxon,   # pragmatic reuse
-            "parent_taxon": parent_taxon,
-            "taxon_rank": taxon_rank,
-
-            "part_used": sorted(set(part_used)),
-            "derived_from": derived_from,
-            "native_to": [],
-            "country_of_origin": country_of_origin,
-
-            "use_categories": use_categories,
-            "medicinal_uses": medicinal_uses,
-            "has_effect": sorted(set(has_effect)),
+            "date_of_birth": get_claim_string_values(claims, "P569"),
+            "date_of_death": get_claim_string_values(claims, "P570"),
+            "coordinates": get_claim_coordinates(claims, "P625"),
+            "country": country,
+            "occupation": occupation,
         }
 
     except Exception as e:
-        print(f"Error parsing details for {qid}: {e}")
-        return {
-            "wikidata_label": qid,
-            "wikidata_description": None,
-            "wikidata_aliases": [],
-            "wikipedia_title": "NIL",
-            "wikipedia_url": "None",
-            "wikipedia_summary": None,
-            "entity_type": "other",
-            "instance_of": [],
-            "subclass_of": [],
-            "scientific_name": [],
-            "source_organism": [],
-            "parent_taxon": [],
-            "taxon_rank": [],
-            "part_used": [],
-            "derived_from": "unknown",
-            "native_to": [],
-            "country_of_origin": [],
-            "use_categories": [],
-            "medicinal_uses": [],
-            "has_effect": [],
-        }
-
-
-        
-def infer_use_categories(text):
-
-    """
-
-    Infer useful GLOBALISE-style categories from Wikipedia/Wikidata text.
-
-    """
-
-    text = (text or "").lower()
-
-    cats = []
-
-    if any(x in text for x in ["medicine", "medicinal", "medicament", "drug", "pharmacopoeia", "laxative", "styptic", "febrifuge", "remedy"]):
-
-        cats.append("medicine")
-
-    if any(x in text for x in ["dye", "pigment", "color", "colour", "colorant"]):
-
-        cats.append("dye")
-
-    if any(x in text for x in ["spice", "seasoning", "aromatic", "cardamom", "pepper", "cubeb"]):
-
-        cats.append("spice")
-
-    if any(x in text for x in ["perfume", "fragrant", "incense", "odoriferous", "aromatic resin"]):
-
-        cats.append("perfume_incense")
-
-    if any(x in text for x in ["food", "edible", "eaten", "culinary", "delicacy", "sweet", "used as food"]):
-
-        cats.append("food")
-
-    if any(x in text for x in ["trade", "commodity", "commerce", "traded", "export"]):
+        logger.exception(f"Error parsing Wikidata details for {qid}: {e}")
+        return empty
 
-        cats.append("trade_good")
 
-    return sorted(set(cats))
-
-def infer_origin_type(instance_of_labels, subclass_of_labels, description, wikipedia_summary):
-
-    """
-
-    Infer whether the entity is mainly plant-derived, animal-derived, mineral, or mixed/unknown.
-
-    """
-
-    text = " ".join((instance_of_labels or []) + (subclass_of_labels or [])) + " " + (description or "") + " " + (wikipedia_summary or "")
-
-    text = text.lower()
-
-    if any(x in text for x in ["plant", "tree", "shrub", "herb", "seed", "root", "fruit", "flower", "resin from", "gum from", "taxon", "species"]):
-
-        return "plant"
-
-    if any(x in text for x in ["animal", "whale", "pig", "secretion", "gall", "insect"]):
-
-        return "animal"
-
-    if any(x in text for x in ["mineral", "earth", "clay", "ore", "stone"]):
-
-        return "mineral"
-
-    return "unknown"
-        
 def get_entity_labels(qids, language="en"):
     if not qids:
         return {}
@@ -706,7 +401,8 @@ def get_entity_labels(qids, language="en"):
         return out
     except Exception:
         return {}
-        
+
+
 def get_wikipedia_page_props(input_str: str):
     """
     Retrieve the Wikidata QID for a given Wikipedia page name.
@@ -819,9 +515,17 @@ def resolve_type_from_wikidata(p31_qids, visited=None, depth=0, max_depth=3):
         # direct anchors
         if q == "Q5":
             return "pers"
-        if q in qid_to_organization_label_keys or q in TYPE_MAP and TYPE_MAP[q] == "organization":
+        if (
+            q in qid_to_organization_label_keys
+            or q in TYPE_MAP
+            and TYPE_MAP[q] == "organization"
+        ):
             return "org"
-        if q in qid_to_location_label_keys or q in TYPE_MAP and TYPE_MAP[q] == "location":
+        if (
+            q in qid_to_location_label_keys
+            or q in TYPE_MAP
+            and TYPE_MAP[q] == "location"
+        ):
             return "loc"
 
         # subclass traversal
@@ -844,7 +548,9 @@ def resolve_type_from_wikidata(p31_qids, visited=None, depth=0, max_depth=3):
                         for c in claims["P279"]
                         if "datavalue" in c["mainsnak"]
                     ]
-                    resolved = resolve_type_from_wikidata(parents, visited, depth + 1, max_depth)
+                    resolved = resolve_type_from_wikidata(
+                        parents, visited, depth + 1, max_depth
+                    )
                     if resolved != "UNK":
                         return resolved
 
@@ -900,47 +606,7 @@ def get_wikipedia_title(qid, language="en"):
         print(f"Error parsing Wikidata for {qid}: {e}")
         return "NIL", "None", "UNK"
 
-def resolve_globalise_type(instance_of_labels, subclass_of_labels, description=None):
-    """
-    Map Wikidata labels/descriptions into a domain-specific coarse type
-    suitable for GLOBALISE glossary entities.
-    """
-    labels = " | ".join((instance_of_labels or []) + (subclass_of_labels or [])).lower()
-    desc = (description or "").lower()
-    text = labels + " " + desc
-
-    if any(x in text for x in ["plant", "taxon", "species", "genus", "tree", "shrub", "herb"]):
-        return "plant"
-
-    if any(x in text for x in ["spice", "condiment", "seed", "pepper", "cardamom", "cubeb"]):
-        return "spice"
-
-    if any(x in text for x in ["resin", "gum", "gum resin", "oleoresin", "benzoin", "myrrh", "lac"]):
-        return "resin_gum"
-
-    if any(x in text for x in ["oil", "essential oil"]):
-        return "oil_extract"
-
-    if any(x in text for x in ["extract", "compound", "chemical substance", "preparation"]):
-        return "extract_preparation"
 
-    if any(x in text for x in ["medicine", "drug", "materia medica", "medicament", "pharmaceutical"]):
-        return "medicinal_substance"
-
-    if any(x in text for x in ["dye", "pigment", "colorant"]):
-        return "dye_colorant"
-
-    if any(x in text for x in ["food", "fruit", "root vegetable", "yam", "edible plant", "crop"]):
-        return "foodstuff"
-
-    if any(x in text for x in ["animal product", "secretion", "ambergris", "bezoar", "gallstone"]):
-        return "animal_derived_substance"
-
-    if any(x in text for x in ["commodity", "trade good", "article of commerce", "material"]):
-        return "commodity"
-
-    return "other"
-    
 class NelPipeline(Pipeline):
     """
     Hugging Face Pipeline for NEL that supports:
@@ -981,12 +647,9 @@ class NelPipeline(Pipeline):
             else:
                 enclosed, lOffset, rOffset = None, None, None
 
-            batch.append({
-                "text": t,
-                "surface": enclosed,
-                "lOffset": lOffset,
-                "rOffset": rOffset
-            })
+            batch.append(
+                {"text": t, "surface": enclosed, "lOffset": lOffset, "rOffset": rOffset}
+            )
 
         return batch
 
@@ -998,18 +661,14 @@ class NelPipeline(Pipeline):
         and keeps the first 3 predictions per input.
         """
         texts = [b["text"] for b in batch]
-    
+
         enc = self.tokenizer(
-            texts,
-            return_tensors="pt",
-            padding=True,
-            truncation=True,
-            max_length=30
+            texts, return_tensors="pt", padding=True, truncation=True, max_length=30
         ).to(self.device)
-    
+
         num_beams = 5
         num_return_sequences = 5
-    
+
         outputs = self.model.generate(
             **enc,
             num_beams=num_beams,
@@ -1017,152 +676,88 @@ class NelPipeline(Pipeline):
             return_dict_in_generate=True,
             output_scores=True,
         )
-    
+
         decoded = self.tokenizer.batch_decode(
-            outputs.sequences,
-            skip_special_tokens=True
+            outputs.sequences, skip_special_tokens=True
         )
-    
+
         sequence_scores = outputs["sequences_scores"].cpu().numpy()
         confidences = np.exp(sequence_scores)
-    
+
         top_k = 3
-    
+
         # flattened:
         # [input1_seq1, ..., input1_seq5, input2_seq1, ..., inputN_seq5]
         for i, item in enumerate(batch):
             start = i * num_return_sequences
             end = start + num_return_sequences
-    
+
             item_predictions = decoded[start:end]
             item_confidences = confidences[start:end]
-    
+
             item["top_predictions"] = item_predictions[:top_k]
             item["top_confidences"] = [float(x) for x in item_confidences[:top_k]]
-    
+
             # optional debug: keep all 5
             item["all_predictions"] = item_predictions
             item["all_confidences"] = [float(x) for x in item_confidences]
-    
+
         return batch
 
     def postprocess(self, batch, **kwargs):
         """
-        N items -> N linked entities, each with top 3 enriched predictions
-        adapted to GLOBALISE commodity / medicinal / botanical entities.
+        N items -> N linked entities, each with top-k enriched Wikidata/Wikipedia predictions.
+        Generic version for historical NEL.
         """
         results = []
-    
+
         for item in batch:
             surface = item["surface"]
             lOffset = item.get("lOffset")
             rOffset = item.get("rOffset")
-    
+
             resolved_predictions = []
-    
+
             for pred, conf in zip(item["top_predictions"], item["top_confidences"]):
                 qid, lang = get_wikipedia_page_props(pred)
                 details = get_wikidata_details(qid, language=lang)
-    
+
                 if conf > 1:
                     conf = conf / 100
-                conf = round(conf, 4)
-    
-                resolved_predictions.append({
-                    "wkd_id": qid,
-                    "wkd_pred": pred,
-    
-                    "wkpedia_pagename": details["wikipedia_title"],
-                    "wkpedia_url": details["wikipedia_url"],
-                    "wikipedia_summary": details["wikipedia_summary"],
-    
-                    "wikidata_label": details["wikidata_label"],
-                    "wikidata_description": details["wikidata_description"],
-                    "wikidata_aliases": details["wikidata_aliases"],
-    
-                    "entity_type": details["entity_type"],
-                    "instance_of": details["instance_of"],
-                    "subclass_of": details["subclass_of"],
-    
-                    "scientific_name": details["scientific_name"],
-                    "source_organism": details["source_organism"],
-                    "parent_taxon": details["parent_taxon"],
-                    "taxon_rank": details["taxon_rank"],
-    
-                    "part_used": details["part_used"],
-                    "derived_from": details["derived_from"],
-                    "native_to": details["native_to"],
-                    "country_of_origin": details["country_of_origin"],
-    
-                    "use_categories": details["use_categories"],
-                    "medicinal_uses": details["medicinal_uses"],
-                    "has_effect": details["has_effect"],
-    
-                    "confidence_nel": conf,
-                })
-    
-            results.append({
-                "surface": surface,
-                "lOffset": lOffset,
-                "rOffset": rOffset,
-                "predictions": resolved_predictions,
-            })
-    
+                conf = round(float(conf), 4)
+
+                resolved_predictions.append(
+                    {
+                        "wkd_id": qid,
+                        "wkd_pred": pred,
+                        "wkpedia_pagename": details["wikipedia_title"],
+                        "wkpedia_url": details["wikipedia_url"],
+                        "wikipedia_summary": details["wikipedia_summary"],
+                        "wikidata_label": details["wikidata_label"],
+                        "wikidata_description": details["wikidata_description"],
+                        "wikidata_aliases": details["wikidata_aliases"],
+                        "entity_type": details["entity_type"],
+                        "instance_of": details["instance_of"],
+                        "subclass_of": details["subclass_of"],
+                        "date_of_birth": details["date_of_birth"],
+                        "date_of_death": details["date_of_death"],
+                        "coordinates": details["coordinates"],
+                        "country": details["country"],
+                        "occupation": details["occupation"],
+                        "confidence_nel": conf,
+                    }
+                )
+
+            results.append(
+                {
+                    "surface": surface,
+                    "lOffset": lOffset,
+                    "rOffset": rOffset,
+                    "predictions": resolved_predictions,
+                }
+            )
+
         return results
-    
-    # def postprocess(self, batch, **kwargs):
-    #     """
-    #     N items -> N linked entities, each with top 3 enriched predictions
-    #     """
-    #     results = []
-    
-    #     for item in batch:
-    #         surface = item["surface"]
-    #         lOffset = item.get("lOffset")
-    #         rOffset = item.get("rOffset")
-    
-    #         resolved_predictions = []
-    
-    #         for pred, conf in zip(item["top_predictions"], item["top_confidences"]):
-    #             qid, lang = get_wikipedia_page_props(pred)
-    #             details = get_wikidata_details(qid, language=lang)
-    
-    #             if conf > 1:
-    #                 conf = conf / 100
-    #             conf = round(conf, 4)
-    
-    #             resolved_predictions.append({
-    #                 "wkd_id": qid,
-    #                 "wkd_pred": pred,
-    
-    #                 "wkpedia_pagename": details["wikipedia_title"],
-    #                 "wkpedia_url": details["wikipedia_url"],
-    #                 "wikipedia_summary": details["wikipedia_summary"],
-    
-    #                 "wikidata_label": details["wikidata_label"],
-    #                 "wikidata_description": details["wikidata_description"],
-    #                 "wikidata_summary": details["wikidata_summary"],
-    #                 "wikidata_aliases": details["wikidata_aliases"],
-    
-    #                 "type": details["entity_type"],
-    #                 "instance_of": details["instance_of"],
-    #                 # "occupations": details["occupations"],
-    #                 "country": details["country"],
-    #                 # "date_of_birth": details["date_of_birth"],
-    #                 # "date_of_death": details["date_of_death"],
-    #                 "coordinates": details["coordinates"],
-    
-    #                 "confidence_nel": conf,
-    #             })
-    
-    #         results.append({
-    #             "surface": surface,
-    #             "lOffset": lOffset,
-    #             "rOffset": rOffset,
-    #             "predictions": resolved_predictions,
-    #         })
-    
-    #     return results
 
 
 qid_to_location_label = {