📝 : Update README.md

2025-10-28 19:06:09 +01:00
parent 9c5de8868f
commit 3f5639ac4d
5 changed files with 1573 additions and 42 deletions
@@ -135,6 +135,61 @@
    "])"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18b534c7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Ancienne version\n",
+    "def clean_text_old(example):\n",
+    "    text = example[\"complete_text\"]\n",
+    "    date = example.get(\"date\", None)\n",
+    "\n",
+    "    # Si la date contient un \"-\", on essaie d'extraire l'année connue (format \"1234-????\" ou \"????-1234\") ou moyenne des deux années\n",
+    "    if \"-\" in str(date) and date is not None:\n",
+    "        parts = str(date).split(\"-\")\n",
+    "        if (parts[1].isdigit() and len(parts[1]) == 4) and parts[0] == \"????\":\n",
+    "            date = str(parts[1])\n",
+    "        else:\n",
+    "            date = str(parts[0])\n",
+    "\n",
+    "    # 1. Retirer les numéros de page\n",
+    "    text = re.sub(r\"[—\\-–]\\s*\\d+\\s*[—\\-–]\", \" \", text)\n",
+    "    \n",
+    "    # 2. Corriger les apostrophes et guillemets échappés\n",
+    "    text = text.replace(\"\\\\'\", \"'\")\n",
+    "    text = text.replace(\"\\\\\\\"\", \"\\\"\")\n",
+    "    text = text.replace(\"\\\\n\", \" \")\n",
+    "    text = text.replace(\"\\\\r\", \" \")\n",
+    "    text = text.replace(\"\\\\t\", \" \")\n",
+    "    \n",
+    "    # 3. Corriger les mots coupés (pattern plus précis)\n",
+    "    text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s+([a-zàâäæçéèêëïîôùûüœ]{2,})', \n",
+    "                  r'\\1\\2', text)\n",
+    "    \n",
+    "    # 4. Corriger les cas avec plusieurs espaces\n",
+    "    text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s{2,}([a-zàâäæçéèêëïîôùûüœ])', \n",
+    "                  r'\\1\\2', text)\n",
+    "    \n",
+    "    # 5. Normaliser les espaces multiples\n",
+    "    text = re.sub(r\"\\s+\", \" \", text)\n",
+    "    \n",
+    "    # 6. Nettoyer les caractères spéciaux\n",
+    "    text = re.sub(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", \" \", text)\n",
+    "    \n",
+    "    # 7. Re-normaliser après nettoyage\n",
+    "    text = re.sub(r\"\\s+\", \" \", text)\n",
+    "    \n",
+    "    # 8. Corriger la ponctuation\n",
+    "    text = re.sub(r\"\\s+([,.\\?!;:])\", r\"\\1\", text)\n",
+    "    text = re.sub(r\"([,.\\?!;:])\\s*([,.\\?!;:])\", r\"\\1\\2\", text)\n",
+    "    \n",
+    "    text = text.strip()\n",
+    "    return {\"text\": text, \"date\": str(date)}"
+   ]
+  },
  {
   "cell_type": "code",
   "execution_count": null,