{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "9aeaf08b-26da-4300-b4d4-e20e9b835876", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: datasets in /opt/conda/lib/python3.13/site-packages (4.2.0)\n", "Requirement already satisfied: filelock in /opt/conda/lib/python3.13/site-packages (from datasets) (3.13.1)\n", "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.13/site-packages (from datasets) (2.3.3)\n", "Requirement already satisfied: pyarrow>=21.0.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (21.0.0)\n", "Requirement already satisfied: dill<0.4.1,>=0.3.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.4.0)\n", "Requirement already satisfied: pandas in /opt/conda/lib/python3.13/site-packages (from datasets) (2.3.3)\n", "Requirement already satisfied: requests>=2.32.2 in /opt/conda/lib/python3.13/site-packages (from datasets) (2.32.5)\n", "Requirement already satisfied: httpx<1.0.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.28.1)\n", "Requirement already satisfied: tqdm>=4.66.3 in /opt/conda/lib/python3.13/site-packages (from datasets) (4.67.1)\n", "Requirement already satisfied: xxhash in /opt/conda/lib/python3.13/site-packages (from datasets) (3.6.0)\n", "Requirement already satisfied: multiprocess<0.70.17 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.70.16)\n", "Requirement already satisfied: fsspec<=2025.9.0,>=2023.1.0 in /opt/conda/lib/python3.13/site-packages (from fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (2025.9.0)\n", "Requirement already satisfied: huggingface-hub<2.0,>=0.25.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.35.3)\n", "Requirement already satisfied: packaging in /opt/conda/lib/python3.13/site-packages (from datasets) (25.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.13/site-packages (from datasets) (6.0.3)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /opt/conda/lib/python3.13/site-packages (from fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (3.13.0)\n", "Requirement already satisfied: anyio in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (4.11.0)\n", "Requirement already satisfied: certifi in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (2025.10.5)\n", "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (1.0.9)\n", "Requirement already satisfied: idna in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (3.10)\n", "Requirement already satisfied: h11>=0.16 in /opt/conda/lib/python3.13/site-packages (from httpcore==1.*->httpx<1.0.0->datasets) (0.16.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.13/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (4.15.0)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.1.3 in /opt/conda/lib/python3.13/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.1.10)\n", "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (2.6.1)\n", "Requirement already satisfied: aiosignal>=1.4.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.4.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (25.3.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.8.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (6.7.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (0.4.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.22.0)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.13/site-packages (from requests>=2.32.2->datasets) (3.4.3)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.13/site-packages (from requests>=2.32.2->datasets) (2.5.0)\n", "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.13/site-packages (from anyio->httpx<1.0.0->datasets) (1.3.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2025.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2025.2)\n", "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "%pip install datasets" ] }, { "cell_type": "code", "execution_count": 2, "id": "07d7e876-b3ba-40ed-b6d6-81683e7ee513", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7ec0afc0b89148f19b72ec8f82927298", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Resolving data files: 0%| | 0/145 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Extraire les années de publication\n", "years = ds['train']['date']\n", "\n", "# Nettoyer et convertir en entiers (certaines dates peuvent être manquantes ou non numériques)\n", "years_clean = [int(y) for y in years if str(y).isdigit()]\n", "\n", "# Créer l'histogramme\n", "plt.figure(figsize=(10, 6))\n", "plt.hist(years_clean, bins=50, color='skyblue', edgecolor='black')\n", "plt.xlabel('Année de publication')\n", "plt.ylabel('Nombre de textes')\n", "plt.title('Répartition des textes dans le temps')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 4, "id": "91c483b8-ae5b-4f1d-9f8a-dffc8eea16c9", "metadata": {}, "outputs": [], "source": [ "import re\n", "\n", "def clean_text(example):\n", " text = example[\"complete_text\"]\n", " date = example.get(\"date\", None)\n", "\n", " # Si la date contient un \"-\", on essaie d'extraire l'année connue (format \"1234-????\" ou \"????-1234\") ou moyenne des deux années\n", " if \"-\" in str(date) and date is not None:\n", " parts = str(date).split(\"-\")\n", " if (parts[1].isdigit() and len(parts[1]) == 4) and parts[0] == \"????\":\n", " date = str(parts[1])\n", " else:\n", " date = str(parts[0])\n", "\n", " # 1. Retirer les numéros de page\n", " text = re.sub(r\"[—\\-–]\\s*\\d+\\s*[—\\-–]\", \" \", text)\n", " \n", " # 2. Corriger les apostrophes et guillemets échappés\n", " text = text.replace(\"\\\\'\", \"'\")\n", " text = text.replace(\"\\\\\\\"\", \"\\\"\")\n", " text = text.replace(\"\\\\n\", \" \")\n", " text = text.replace(\"\\\\r\", \" \")\n", " text = text.replace(\"\\\\t\", \" \")\n", " \n", " # 3. Corriger les mots coupés (pattern plus précis)\n", " text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s+([a-zàâäæçéèêëïîôùûüœ]{2,})', \n", " r'\\1\\2', text)\n", " \n", " # 4. Corriger les cas avec plusieurs espaces\n", " text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s{2,}([a-zàâäæçéèêëïîôùûüœ])', \n", " r'\\1\\2', text)\n", " \n", " # 5. Normaliser les espaces multiples\n", " text = re.sub(r\"\\s+\", \" \", text)\n", " \n", " # 6. Nettoyer les caractères spéciaux\n", " text = re.sub(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", \" \", text)\n", " \n", " # 7. Re-normaliser après nettoyage\n", " text = re.sub(r\"\\s+\", \" \", text)\n", " \n", " # 8. Corriger la ponctuation\n", " text = re.sub(r\"\\s+([,.\\?!;:])\", r\"\\1\", text)\n", " text = re.sub(r\"([,.\\?!;:])\\s*([,.\\?!;:])\", r\"\\1\\2\", text)\n", " \n", " text = text.strip()\n", " return {\"text\": text, \"date\": str(date)}\n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "1d8bc092", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Le texte nettoyé semble correct.\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "example = ds['train'][500]\n", "\n", "cleaned_example = clean_text(example)\n", "\n", "def verify_clean_text(original_example, cleaned_example):\n", " # Vérifie que le texte nettoyé n'est pas vide\n", " if not cleaned_example[\"text\"]:\n", " print(\"Le texte nettoyé est vide.\")\n", " return False\n", "\n", " # Vérifie que la date est bien une année à 4 chiffres\n", " if not (cleaned_example[\"date\"].isdigit() and len(cleaned_example[\"date\"]) == 4):\n", " print(f\"Date nettoyée incorrecte: {cleaned_example['date']}\")\n", " return False\n", "\n", " # Vérifie que les caractères spéciaux indésirables ont été retirés\n", " if re.search(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", cleaned_example[\"text\"]):\n", " print(\"Caractères spéciaux indésirables présents dans le texte nettoyé.\")\n", " return False\n", "\n", " # Vérifie qu'il n'y a pas de séquences d'espaces multiples\n", " if re.search(r\"\\s{2,}\", cleaned_example[\"text\"]):\n", " print(\"Espaces multiples détectés dans le texte nettoyé.\")\n", " return False\n", "\n", " print(\"Le texte nettoyé semble correct.\")\n", " return True\n", "\n", "# Exemple d'utilisation\n", "verify_clean_text(example, cleaned_example)" ] }, { "cell_type": "code", "execution_count": 6, "id": "39e38bbf", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a1dd55cc88a34a7fb447d36646c86c4d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/50000 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\\n------ ÉVALUATION FINALE ------\n", "🎯 Accuracy finale: 0.2550\\n\n", "📄 Rapport de classification détaillé :\n", " precision recall f1-score support\n", "\n", " 1560 0.00 0.00 0.00 1\n", " 1570 0.00 0.00 0.00 2\n", " 1580 0.00 0.00 0.00 2\n", " 1590 0.00 0.00 0.00 1\n", " 1600 0.00 0.00 0.00 12\n", " 1610 0.00 0.00 0.00 27\n", " 1620 0.00 0.00 0.00 24\n", " 1630 0.20 0.04 0.07 23\n", " 1640 0.00 0.00 0.00 27\n", " 1650 0.00 0.00 0.00 27\n", " 1660 0.00 0.00 0.00 12\n", " 1670 0.00 0.00 0.00 7\n", " 1680 0.00 0.00 0.00 10\n", " 1690 0.00 0.00 0.00 25\n", " 1700 0.00 0.00 0.00 21\n", " 1710 0.00 0.00 0.00 31\n", " 1720 0.00 0.00 0.00 45\n", " 1730 0.00 0.00 0.00 52\n", " 1740 0.49 0.30 0.37 101\n", " 1750 0.33 0.01 0.02 123\n", " 1760 0.32 0.15 0.20 122\n", " 1770 0.07 0.06 0.07 119\n", " 1780 0.22 0.18 0.19 176\n", " 1790 0.35 0.39 0.37 213\n", " 1800 0.08 0.01 0.01 167\n", " 1810 0.25 0.07 0.11 308\n", " 1820 0.28 0.26 0.27 447\n", " 1830 0.27 0.17 0.21 534\n", " 1840 0.29 0.22 0.25 687\n", " 1850 0.20 0.22 0.21 783\n", " 1860 0.21 0.35 0.26 1036\n", " 1870 0.25 0.40 0.31 1063\n", " 1880 0.23 0.36 0.28 1017\n", " 1890 0.29 0.29 0.29 898\n", " 1900 0.29 0.35 0.32 767\n", " 1910 0.34 0.18 0.23 466\n", " 1920 0.29 0.04 0.07 265\n", " 1930 0.61 0.20 0.30 217\n", " 1940 0.00 0.00 0.00 34\n", " 1950 0.00 0.00 0.00 23\n", " 1960 0.00 0.00 0.00 20\n", " 1970 0.20 0.06 0.09 18\n", " 1980 0.00 0.00 0.00 23\n", " 1990 0.00 0.00 0.00 10\n", " 2000 0.00 0.00 0.00 14\n", "\n", " accuracy 0.26 10000\n", " macro avg 0.14 0.10 0.10 10000\n", "weighted avg 0.26 0.26 0.24 10000\n", "\n" ] } ], "source": [ "# Entraîner et évaluer avec TF-IDF et suivi de la perte\n", "#model_tfidf_sgd = train_with_loss_tracking(\n", "# X_train_tfidf, X_test_tfidf, \n", "# train_labels, test_labels,\n", "# \"TF-IDF\",\n", "# n_epochs=30\n", "#)\n", "\n", "# Entraîner et évaluer avec Embeddings et suivi de la perte\n", "model_embedding_sgd = train_with_loss_tracking(\n", " X_train_embedding, X_test_embedding,\n", " train_labels, test_labels,\n", " \"Embeddings (CamemBERT)\",\n", " n_epochs=50\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "id": "02f2c811", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Interrogation d'un texte aléatoire ---\n", "📚 Titre : Les conserves de fruits pour la consommation familiale et pour la vente : la conservation des matières alimentaires dans les ménages, à la ferme et dans les coopératives agricoles\n", "🗓️ Date réelle : 1912\n", "----------------------------------------\n", "🤖 Prédiction (TF-IDF) : 1900s\n", "Création des embeddings pour 1 textes...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "3b376ef8bc3a4497917ecb41ebbd114e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Batches: 0%| | 0/1 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import random\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "def test_and_plot_predictions(test_dataset, num_trials=15):\n", " \"\"\"\n", " Sélectionne plusieurs textes aléatoires, prédit leur date avec deux modèles,\n", " et affiche un graphique comparatif des résultats.\n", " \n", " Args:\n", " test_dataset (list): La liste d'exemples à tester.\n", " num_trials (int): Le nombre d'essais à effectuer.\n", " \"\"\"\n", " real_dates = []\n", " tfidf_predictions = []\n", " embedding_predictions = []\n", "\n", " print(f\"--- Lancement de {num_trials} essais de prédiction ---\")\n", "\n", " for i in range(num_trials):\n", " # 1. Sélectionner un exemple aléatoire\n", " random_example = random.choice(test_dataset)\n", " \n", " # Extrait la date et la convertit en entier pour le graphique\n", " # On suppose que la date est au format 'YYYYs' (ex: '1990s')\n", " try:\n", " true_date_str = random_example.get('date', '0s')\n", " true_date_int = int(true_date_str[:4]) # Convertit '1990s' en 1990\n", " except (ValueError, IndexError):\n", " print(f\"Avertissement : impossible de parser la date '{true_date_str}'. Essai ignoré.\")\n", " continue\n", "\n", " # 2. Nettoyer le texte\n", " cleaned_text = clean_text(random_example)['text']\n", " text_to_predict = [cleaned_text]\n", "\n", " # 3. Prédiction avec le modèle TF-IDF\n", " #tfidf_features = tfidf_vectorizer.transform(text_to_predict)\n", " #prediction_tfidf = model_tfidf_sgd.predict(tfidf_features)[0]\n", "\n", " # 4. Prédiction avec le modèle Embeddings\n", " embedding_features = create_embeddings(text_to_predict, batch_size=1)\n", " prediction_embedding = model_embedding_sgd.predict(embedding_features)[0]\n", " \n", " # 5. Stocker les résultats\n", " real_dates.append(true_date_int)\n", " #tfidf_predictions.append(prediction_tfidf)\n", " embedding_predictions.append(prediction_embedding)\n", " \n", " #print(f\"Essai {i+1}/{num_trials} | Réel : {true_date_int} | TF-IDF : {prediction_tfidf} | Embeddings : {prediction_embedding}\")\n", " print(f\"Essai {i+1}/{num_trials} | Réel : {true_date_int} | Embeddings : {prediction_embedding}\")\n", "\n", " # 6. Générer le graphique\n", " if not real_dates:\n", " print(\"Aucune donnée à afficher.\")\n", " return\n", "\n", " trials = range(1, len(real_dates) + 1)\n", " \n", " plt.style.use('seaborn-v0_8-whitegrid') # Style visuel agréable\n", " plt.figure(figsize=(14, 7)) # Taille de la figure\n", "\n", " # Tracer chaque série de données\n", " plt.plot(trials, real_dates, 'o-', color='green', label='Date Réelle', markersize=8)\n", " #plt.plot(trials, tfidf_predictions, 's--', color='blue', label='Prédiction (TF-IDF)')\n", " plt.plot(trials, embedding_predictions, '^:', color='red', label='Prédiction (Embeddings)')\n", "\n", " # Ajouter les titres et les légendes\n", " plt.title('Comparaison des Prédictions de Date sur Plusieurs Essais', fontsize=16)\n", " plt.xlabel(\"Numéro de l'essai\", fontsize=12)\n", " plt.ylabel(\"Date (Année)\", fontsize=12)\n", " plt.xticks(trials) # Assure que chaque essai a une graduation sur l'axe X\n", " plt.legend(fontsize=10)\n", " plt.tight_layout() # Ajuste le graphique pour qu'il s'affiche bien\n", "\n", " # Afficher le graphique\n", " plt.show()\n", "\n", "\n", "# --- Lancer le test ---\n", "# Utilisez ds['train'] et spécifiez le nombre d'essais souhaités (ex: 20)\n", "test_and_plot_predictions(ds['train'], num_trials=20)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.7" } }, "nbformat": 4, "nbformat_minor": 5 }