{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9aeaf08b-26da-4300-b4d4-e20e9b835876",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.13/site-packages (4.2.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.13/site-packages (from datasets) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.13/site-packages (from datasets) (2.3.3)\n",
      "Requirement already satisfied: pyarrow>=21.0.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (21.0.0)\n",
      "Requirement already satisfied: dill<0.4.1,>=0.3.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.4.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.13/site-packages (from datasets) (2.3.3)\n",
      "Requirement already satisfied: requests>=2.32.2 in /opt/conda/lib/python3.13/site-packages (from datasets) (2.32.5)\n",
      "Requirement already satisfied: httpx<1.0.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.28.1)\n",
      "Requirement already satisfied: tqdm>=4.66.3 in /opt/conda/lib/python3.13/site-packages (from datasets) (4.67.1)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.13/site-packages (from datasets) (3.6.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2025.9.0,>=2023.1.0 in /opt/conda/lib/python3.13/site-packages (from fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (2025.9.0)\n",
      "Requirement already satisfied: huggingface-hub<2.0,>=0.25.0 in /opt/conda/lib/python3.13/site-packages (from datasets) (0.35.3)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.13/site-packages (from datasets) (25.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.13/site-packages (from datasets) (6.0.3)\n",
      "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /opt/conda/lib/python3.13/site-packages (from fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (3.13.0)\n",
      "Requirement already satisfied: anyio in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (4.11.0)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (2025.10.5)\n",
      "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (1.0.9)\n",
      "Requirement already satisfied: idna in /opt/conda/lib/python3.13/site-packages (from httpx<1.0.0->datasets) (3.10)\n",
      "Requirement already satisfied: h11>=0.16 in /opt/conda/lib/python3.13/site-packages (from httpcore==1.*->httpx<1.0.0->datasets) (0.16.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.13/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (4.15.0)\n",
      "Requirement already satisfied: hf-xet<2.0.0,>=1.1.3 in /opt/conda/lib/python3.13/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.1.10)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (2.6.1)\n",
      "Requirement already satisfied: aiosignal>=1.4.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.4.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (25.3.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.8.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (6.7.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (0.4.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/conda/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.9.0,>=2023.1.0->datasets) (1.22.0)\n",
      "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.13/site-packages (from requests>=2.32.2->datasets) (3.4.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.13/site-packages (from requests>=2.32.2->datasets) (2.5.0)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.13/site-packages (from anyio->httpx<1.0.0->datasets) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2025.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.13/site-packages (from pandas->datasets) (2025.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "07d7e876-b3ba-40ed-b6d6-81683e7ee513",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ec0afc0b89148f19b72ec8f82927298",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/145 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a68949f462cc462cac490123368d8e35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset shards:   0%|          | 0/145 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "ds = load_dataset(\"PleIAs/French-PD-Books\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c5400ed-494e-4bae-a540-71655b000319",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33128 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Extraire les années de publication\n",
    "years = ds['train']['date']\n",
    "\n",
    "# Nettoyer et convertir en entiers (certaines dates peuvent être manquantes ou non numériques)\n",
    "years_clean = [int(y) for y in years if str(y).isdigit()]\n",
    "\n",
    "# Créer l'histogramme\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(years_clean, bins=50, color='skyblue', edgecolor='black')\n",
    "plt.xlabel('Année de publication')\n",
    "plt.ylabel('Nombre de textes')\n",
    "plt.title('Répartition des textes dans le temps')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "91c483b8-ae5b-4f1d-9f8a-dffc8eea16c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def clean_text(example):\n",
    "    text = example[\"complete_text\"]\n",
    "    date = example.get(\"date\", None)\n",
    "\n",
    "    # Si la date contient un \"-\", on essaie d'extraire l'année connue (format \"1234-????\" ou \"????-1234\") ou moyenne des deux années\n",
    "    if \"-\" in str(date) and date is not None:\n",
    "        parts = str(date).split(\"-\")\n",
    "        if (parts[1].isdigit() and len(parts[1]) == 4) and parts[0] == \"????\":\n",
    "            date = str(parts[1])\n",
    "        else:\n",
    "            date = str(parts[0])\n",
    "\n",
    "    # 1. Retirer les numéros de page\n",
    "    text = re.sub(r\"[—\\-–]\\s*\\d+\\s*[—\\-–]\", \" \", text)\n",
    "    \n",
    "    # 2. Corriger les apostrophes et guillemets échappés\n",
    "    text = text.replace(\"\\\\'\", \"'\")\n",
    "    text = text.replace(\"\\\\\\\"\", \"\\\"\")\n",
    "    text = text.replace(\"\\\\n\", \" \")\n",
    "    text = text.replace(\"\\\\r\", \" \")\n",
    "    text = text.replace(\"\\\\t\", \" \")\n",
    "    \n",
    "    # 3. Corriger les mots coupés (pattern plus précis)\n",
    "    text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s+([a-zàâäæçéèêëïîôùûüœ]{2,})', \n",
    "                  r'\\1\\2', text)\n",
    "    \n",
    "    # 4. Corriger les cas avec plusieurs espaces\n",
    "    text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s{2,}([a-zàâäæçéèêëïîôùûüœ])', \n",
    "                  r'\\1\\2', text)\n",
    "    \n",
    "    # 5. Normaliser les espaces multiples\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    \n",
    "    # 6. Nettoyer les caractères spéciaux\n",
    "    text = re.sub(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", \" \", text)\n",
    "    \n",
    "    # 7. Re-normaliser après nettoyage\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    \n",
    "    # 8. Corriger la ponctuation\n",
    "    text = re.sub(r\"\\s+([,.\\?!;:])\", r\"\\1\", text)\n",
    "    text = re.sub(r\"([,.\\?!;:])\\s*([,.\\?!;:])\", r\"\\1\\2\", text)\n",
    "    \n",
    "    text = text.strip()\n",
    "    return {\"text\": text, \"date\": str(date)}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1d8bc092",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le texte nettoyé semble correct.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example = ds['train'][500]\n",
    "\n",
    "cleaned_example = clean_text(example)\n",
    "\n",
    "def verify_clean_text(original_example, cleaned_example):\n",
    "    # Vérifie que le texte nettoyé n'est pas vide\n",
    "    if not cleaned_example[\"text\"]:\n",
    "        print(\"Le texte nettoyé est vide.\")\n",
    "        return False\n",
    "\n",
    "    # Vérifie que la date est bien une année à 4 chiffres\n",
    "    if not (cleaned_example[\"date\"].isdigit() and len(cleaned_example[\"date\"]) == 4):\n",
    "        print(f\"Date nettoyée incorrecte: {cleaned_example['date']}\")\n",
    "        return False\n",
    "\n",
    "    # Vérifie que les caractères spéciaux indésirables ont été retirés\n",
    "    if re.search(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", cleaned_example[\"text\"]):\n",
    "        print(\"Caractères spéciaux indésirables présents dans le texte nettoyé.\")\n",
    "        return False\n",
    "\n",
    "    # Vérifie qu'il n'y a pas de séquences d'espaces multiples\n",
    "    if re.search(r\"\\s{2,}\", cleaned_example[\"text\"]):\n",
    "        print(\"Espaces multiples détectés dans le texte nettoyé.\")\n",
    "        return False\n",
    "\n",
    "    print(\"Le texte nettoyé semble correct.\")\n",
    "    return True\n",
    "\n",
    "# Exemple d'utilisation\n",
    "verify_clean_text(example, cleaned_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39e38bbf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1dd55cc88a34a7fb447d36646c86c4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/50000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "reduced_ds = ds['train'].shuffle(seed=42).select(range(50000)) # Pour accélérer le traitement\n",
    "\n",
    "cleaned_ds = reduced_ds.map(clean_text, remove_columns=reduced_ds.column_names)\n",
    "# cleaned_ds = ds['train'].map(clean_text, remove_columns=ds['train'].column_names, num_proc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "685a1521",
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i in range(3):\n",
    "#    print(f\"\\nExemple {i+1}:\")\n",
    "#    print(cleaned_ds[i])\n",
    "#    print(cleaned_ds[i][\"text\"][:200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0474c7fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a0b038366ca40c4bf0d80dbab30b8c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/50000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Taille de l'ensemble d'entraînement : 40000 textes.\n",
      "Taille de l'ensemble de test : 10000 textes.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Créer les labels par tranches de 10 ans\n",
    "def create_decade_label(example):\n",
    "    \"\"\"Crée une étiquette numérique (ex: 1880) pour chaque décennie.\"\"\"\n",
    "    year = int(example['date'])\n",
    "    decade = (year // 10) * 10\n",
    "    return {\"decade\": decade}\n",
    "\n",
    "# Appliquer la fonction à votre dataset\n",
    "dataset_with_labels = cleaned_ds.map(create_decade_label)\n",
    "\n",
    "# Séparer en ensembles d'entraînement (80%) et de test (20%)\n",
    "train_test_split = dataset_with_labels.train_test_split(test_size=0.2, seed=42)\n",
    "train_dataset = train_test_split['train']\n",
    "test_dataset = train_test_split['test']\n",
    "\n",
    "# Extraire les textes et les labels pour une utilisation plus simple\n",
    "train_texts = [ex['text'] for ex in train_dataset]\n",
    "test_texts = [ex['text'] for ex in test_dataset]\n",
    "train_labels = [ex['decade'] for ex in train_dataset]\n",
    "test_labels = [ex['decade'] for ex in test_dataset]\n",
    "\n",
    "print(f\"Taille de l'ensemble d'entraînement : {len(train_texts)} textes.\")\n",
    "print(f\"Taille de l'ensemble de test : {len(test_texts)} textes.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4acca337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vectorisation TF-IDF en cours...\n",
      "Shape des features TF-IDF (entraînement) : (40000, 10000)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import gc\n",
    "\n",
    "# Configuration du vectoriseur TF-IDF\n",
    "# max_features limite la taille du vocabulaire pour économiser la mémoire\n",
    "# min_df et max_df ignorent les mots trop rares ou trop fréquents\n",
    "tfidf_vectorizer = TfidfVectorizer(\n",
    "    max_features=10000,\n",
    "    min_df=5,\n",
    "    max_df=0.8,\n",
    "    ngram_range=(1, 1) # On utilise uniquement des unigrammes (mots seuls)\n",
    ")\n",
    "\n",
    "print(\"Vectorisation TF-IDF en cours...\")\n",
    "X_train_tfidf = tfidf_vectorizer.fit_transform(train_texts)\n",
    "X_test_tfidf = tfidf_vectorizer.transform(test_texts)\n",
    "\n",
    "# Libérer de la mémoire\n",
    "del train_texts\n",
    "del test_texts\n",
    "gc.collect()\n",
    "\n",
    "print(f\"Shape des features TF-IDF (entraînement) : {X_train_tfidf.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2f38c125",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Création des embeddings pour 40000 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5de39b5ab7ec4e2cb40fc8a96ab62ac9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/2500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Création des embeddings pour 10000 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "659c7b138aaf4f58a23cb0055aa03499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/625 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape des embeddings (entraînement) : (40000, 768)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pip install sentence-transformers -q\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import numpy as np\n",
    "import random\n",
    "import gc\n",
    "\n",
    "def get_text_sample(text, selection_method=\"start_tokens\", n_words=500):\n",
    "    \"\"\"\n",
    "    Extrait un échantillon de texte en sélectionnant des mots (tokens)\n",
    "    du début, du milieu, de la fin, et de manière aléatoire.\n",
    "    \"\"\"\n",
    "    tokens = text.split()\n",
    "    \n",
    "    # Si le texte est trop court pour un échantillonnage significatif,\n",
    "    # on retourne le texte entier.\n",
    "    if len(tokens) < n_words * 4:\n",
    "        return text\n",
    "\n",
    "    if selection_method == \"start_tokens\":\n",
    "        # 1. 500 mots du début\n",
    "        sorted_tokens = tokens[:n_words]\n",
    "\n",
    "    if selection_method == \"end_tokens\":\n",
    "        # 2. 500 mots de la fin\n",
    "        sorted_tokens = tokens[-n_words:]\n",
    "\n",
    "    if selection_method == \"middle_tokens\":\n",
    "        # 3. 500 mots du milieu\n",
    "        middle_index = len(tokens) // 2\n",
    "        middle_start = max(0, middle_index - (n_words // 2))\n",
    "        sorted_tokens = tokens[middle_start : middle_start + n_words]\n",
    "\n",
    "    if selection_method == \"random_tokens\":\n",
    "        # 4. 500 mots aléatoires (en évitant les doublons avec les sections déjà extraites)\n",
    "        # On construit la liste des indices déjà utilisés\n",
    "        middle_index = len(tokens) // 2\n",
    "        middle_start = max(0, middle_index - (n_words // 2))\n",
    "        used_indices = set(range(n_words)) | set(range(len(tokens) - n_words, len(tokens))) | set(range(middle_start, middle_start + n_words))\n",
    "        # On crée la liste des indices restants disponibles\n",
    "        remaining_indices = [i for i in range(len(tokens)) if i not in used_indices]\n",
    "        \n",
    "        if len(remaining_indices) < n_words:\n",
    "            random_indices = remaining_indices\n",
    "        else:\n",
    "            random_indices = random.sample(remaining_indices, n_words)\n",
    "        \n",
    "        sorted_tokens = [tokens[i] for i in sorted(random_indices)] # trier pour garder un semblant d'ordre\n",
    "\n",
    "    return \" \".join(sorted_tokens)\n",
    "\n",
    "# Charger un modèle CamemBERT léger, spécialisé dans la création d'embeddings\n",
    "embedding_model = SentenceTransformer('dangvantuan/sentence-camembert-base')\n",
    "\n",
    "# Fonction pour créer les embeddings par lots (pour optimiser l'usage de la RAM)\n",
    "def create_embeddings(texts, selection_method=\"start_tokens\", batch_size=32):\n",
    "    # Créer des échantillons de texte représentatifs\n",
    "    sampled_texts = [get_text_sample(text, selection_method, n_words=500) for text in texts]\n",
    "    \n",
    "    print(f\"Création des embeddings pour {len(sampled_texts)} textes...\")\n",
    "    return embedding_model.encode(\n",
    "        sampled_texts,\n",
    "        batch_size=batch_size,\n",
    "        show_progress_bar=True,\n",
    "        convert_to_numpy=True,\n",
    "        normalize_embeddings=True\n",
    "    )\n",
    "\n",
    "# Recréer les listes de textes car elles ont été supprimées\n",
    "# Assurez-vous que les variables `train_dataset` et `test_dataset` sont bien définies en exécutant les cellules précédentes.\n",
    "train_texts = [ex['text'] for ex in train_dataset]\n",
    "test_texts = [ex['text'] for ex in test_dataset]\n",
    "\n",
    "# start_tokens, end_tokens, middle_tokens, random_tokens\n",
    "selection_method = \"start_tokens\"\n",
    "X_train_embedding = create_embeddings(train_texts, selection_method, batch_size=16)\n",
    "X_test_embedding = create_embeddings(test_texts, selection_method, batch_size=16)\n",
    "\n",
    "print(f\"Shape des embeddings (entraînement) : {X_train_embedding.shape}\")\n",
    "\n",
    "# Libérer de la mémoire\n",
    "del train_texts\n",
    "del test_texts\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dfafa732",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.metrics import log_loss, classification_report, accuracy_score\n",
    "import numpy as np # Assurez-vous que numpy est importé\n",
    "\n",
    "def train_with_loss_tracking(X_train, X_test, y_train, y_test, method_name, n_epochs=20):\n",
    "    \"\"\"\n",
    "    Entraîne un classifieur SGD, suit la perte à chaque époque,\n",
    "    et affiche les résultats ainsi qu'un graphique de la perte.\n",
    "    \"\"\"\n",
    "    print(f\"\\\\n{'='*50}\")\n",
    "    print(f\"Entraînement avec suivi de la perte ({method_name})\")\n",
    "    print(f\"{'='*50}\\\\n\")\n",
    "\n",
    "    classifier = SGDClassifier(\n",
    "        loss='log_loss', \n",
    "        random_state=42, \n",
    "        n_jobs=-1, \n",
    "        #learning_rate='constant',\n",
    "        #eta0=0.001\n",
    "    )\n",
    "    \n",
    "    train_loss_history = []\n",
    "    test_loss_history = []\n",
    "    \n",
    "    # --- CORRECTION 1 : Définir la liste complète de TOUS les labels ---\n",
    "    # On combine les labels du train et du test pour être sûr de n'en oublier aucun.\n",
    "    all_classes = np.unique(np.concatenate((y_train, y_test)))\n",
    "\n",
    "    print(f\"Début de l'entraînement pour {n_epochs} époques...\")\n",
    "    for epoch in range(n_epochs):\n",
    "        classifier.partial_fit(X_train, y_train, classes=all_classes)\n",
    "        \n",
    "        y_train_pred_proba = classifier.predict_proba(X_train)\n",
    "        y_test_pred_proba = classifier.predict_proba(X_test)\n",
    "        \n",
    "        # --- CORRECTION 2 : Passer `labels=all_classes` à log_loss ---\n",
    "        train_loss = log_loss(y_train, y_train_pred_proba, labels=all_classes)\n",
    "        test_loss = log_loss(y_test, y_test_pred_proba, labels=all_classes)\n",
    "        \n",
    "        train_loss_history.append(train_loss)\n",
    "        test_loss_history.append(test_loss)\n",
    "        \n",
    "        print(f\"Époque {epoch+1}/{n_epochs} - Perte entraînement: {train_loss:.4f} - Perte test: {test_loss:.4f}\")\n",
    "\n",
    "    # Visualisation de la fonction de perte\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    plt.plot(train_loss_history, label=\"Perte d'entraînement (Train Loss)\")\n",
    "    plt.plot(test_loss_history, label=\"Perte de validation (Test Loss)\")\n",
    "    plt.title(f\"Évolution de la fonction de perte ({method_name})\")\n",
    "    plt.xlabel(\"Époques\")\n",
    "    plt.ylabel(\"Log Loss\")\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "    plt.show()\n",
    "\n",
    "    # Évaluation finale sur l'ensemble de test\n",
    "    print(\"\\\\n------ ÉVALUATION FINALE ------\")\n",
    "    y_pred = classifier.predict(X_test)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    \n",
    "    print(f\"🎯 Accuracy finale: {accuracy:.4f}\\\\n\")\n",
    "    print(\"📄 Rapport de classification détaillé :\")\n",
    "    print(classification_report(y_test, y_pred, zero_division=0)) # Ajout de zero_division pour éviter les warnings\n",
    "    \n",
    "    return classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c366fc0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\n==================================================\n",
      "Entraînement avec suivi de la perte (Embeddings (CamemBERT))\n",
      "==================================================\\n\n",
      "Début de l'entraînement pour 50 époques...\n",
      "Époque 1/50 - Perte entraînement: 2.4359 - Perte test: 2.4721\n",
      "Époque 2/50 - Perte entraînement: 2.4084 - Perte test: 2.4434\n",
      "Époque 3/50 - Perte entraînement: 2.3990 - Perte test: 2.4334\n",
      "Époque 4/50 - Perte entraînement: 2.3940 - Perte test: 2.4281\n",
      "Époque 5/50 - Perte entraînement: 2.3907 - Perte test: 2.4247\n",
      "Époque 6/50 - Perte entraînement: 2.3883 - Perte test: 2.4222\n",
      "Époque 7/50 - Perte entraînement: 2.3865 - Perte test: 2.4204\n",
      "Époque 8/50 - Perte entraînement: 2.3852 - Perte test: 2.4191\n",
      "Époque 9/50 - Perte entraînement: 2.3841 - Perte test: 2.4180\n",
      "Époque 10/50 - Perte entraînement: 2.3832 - Perte test: 2.4172\n",
      "Époque 11/50 - Perte entraînement: 2.3826 - Perte test: 2.4165\n",
      "Époque 12/50 - Perte entraînement: 2.3820 - Perte test: 2.4160\n",
      "Époque 13/50 - Perte entraînement: 2.3815 - Perte test: 2.4156\n",
      "Époque 14/50 - Perte entraînement: 2.3811 - Perte test: 2.4152\n",
      "Époque 15/50 - Perte entraînement: 2.3808 - Perte test: 2.4149\n",
      "Époque 16/50 - Perte entraînement: 2.3805 - Perte test: 2.4146\n",
      "Époque 17/50 - Perte entraînement: 2.3803 - Perte test: 2.4144\n",
      "Époque 18/50 - Perte entraînement: 2.3800 - Perte test: 2.4142\n",
      "Époque 19/50 - Perte entraînement: 2.3798 - Perte test: 2.4140\n",
      "Époque 20/50 - Perte entraînement: 2.3796 - Perte test: 2.4138\n",
      "Époque 21/50 - Perte entraînement: 2.3795 - Perte test: 2.4137\n",
      "Époque 22/50 - Perte entraînement: 2.3793 - Perte test: 2.4135\n",
      "Époque 23/50 - Perte entraînement: 2.3792 - Perte test: 2.4134\n",
      "Époque 24/50 - Perte entraînement: 2.3792 - Perte test: 2.4134\n",
      "Époque 25/50 - Perte entraînement: 2.3790 - Perte test: 2.4132\n",
      "Époque 26/50 - Perte entraînement: 2.3788 - Perte test: 2.4131\n",
      "Époque 27/50 - Perte entraînement: 2.3789 - Perte test: 2.4132\n",
      "Époque 28/50 - Perte entraînement: 2.3787 - Perte test: 2.4130\n",
      "Époque 29/50 - Perte entraînement: 2.3787 - Perte test: 2.4130\n",
      "Époque 30/50 - Perte entraînement: 2.3785 - Perte test: 2.4128\n",
      "Époque 31/50 - Perte entraînement: 2.3786 - Perte test: 2.4129\n",
      "Époque 32/50 - Perte entraînement: 2.3784 - Perte test: 2.4127\n",
      "Époque 33/50 - Perte entraînement: 2.3784 - Perte test: 2.4127\n",
      "Époque 34/50 - Perte entraînement: 2.3783 - Perte test: 2.4127\n",
      "Époque 35/50 - Perte entraînement: 2.3782 - Perte test: 2.4125\n",
      "Époque 36/50 - Perte entraînement: 2.3783 - Perte test: 2.4126\n",
      "Époque 37/50 - Perte entraînement: 2.3782 - Perte test: 2.4126\n",
      "Époque 38/50 - Perte entraînement: 2.3780 - Perte test: 2.4124\n",
      "Époque 39/50 - Perte entraînement: 2.3780 - Perte test: 2.4125\n",
      "Époque 40/50 - Perte entraînement: 2.3782 - Perte test: 2.4126\n",
      "Époque 41/50 - Perte entraînement: 2.3779 - Perte test: 2.4124\n",
      "Époque 42/50 - Perte entraînement: 2.3778 - Perte test: 2.4123\n",
      "Époque 43/50 - Perte entraînement: 2.3779 - Perte test: 2.4123\n",
      "Époque 44/50 - Perte entraînement: 2.3781 - Perte test: 2.4125\n",
      "Époque 45/50 - Perte entraînement: 2.3779 - Perte test: 2.4123\n",
      "Époque 46/50 - Perte entraînement: 2.3777 - Perte test: 2.4121\n",
      "Époque 47/50 - Perte entraînement: 2.3776 - Perte test: 2.4121\n",
      "Époque 48/50 - Perte entraînement: 2.3777 - Perte test: 2.4122\n",
      "Époque 49/50 - Perte entraînement: 2.3780 - Perte test: 2.4124\n",
      "Époque 50/50 - Perte entraînement: 2.3778 - Perte test: 2.4123\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 50368 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\n------ ÉVALUATION FINALE ------\n",
      "🎯 Accuracy finale: 0.2550\\n\n",
      "📄 Rapport de classification détaillé :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        1560       0.00      0.00      0.00         1\n",
      "        1570       0.00      0.00      0.00         2\n",
      "        1580       0.00      0.00      0.00         2\n",
      "        1590       0.00      0.00      0.00         1\n",
      "        1600       0.00      0.00      0.00        12\n",
      "        1610       0.00      0.00      0.00        27\n",
      "        1620       0.00      0.00      0.00        24\n",
      "        1630       0.20      0.04      0.07        23\n",
      "        1640       0.00      0.00      0.00        27\n",
      "        1650       0.00      0.00      0.00        27\n",
      "        1660       0.00      0.00      0.00        12\n",
      "        1670       0.00      0.00      0.00         7\n",
      "        1680       0.00      0.00      0.00        10\n",
      "        1690       0.00      0.00      0.00        25\n",
      "        1700       0.00      0.00      0.00        21\n",
      "        1710       0.00      0.00      0.00        31\n",
      "        1720       0.00      0.00      0.00        45\n",
      "        1730       0.00      0.00      0.00        52\n",
      "        1740       0.49      0.30      0.37       101\n",
      "        1750       0.33      0.01      0.02       123\n",
      "        1760       0.32      0.15      0.20       122\n",
      "        1770       0.07      0.06      0.07       119\n",
      "        1780       0.22      0.18      0.19       176\n",
      "        1790       0.35      0.39      0.37       213\n",
      "        1800       0.08      0.01      0.01       167\n",
      "        1810       0.25      0.07      0.11       308\n",
      "        1820       0.28      0.26      0.27       447\n",
      "        1830       0.27      0.17      0.21       534\n",
      "        1840       0.29      0.22      0.25       687\n",
      "        1850       0.20      0.22      0.21       783\n",
      "        1860       0.21      0.35      0.26      1036\n",
      "        1870       0.25      0.40      0.31      1063\n",
      "        1880       0.23      0.36      0.28      1017\n",
      "        1890       0.29      0.29      0.29       898\n",
      "        1900       0.29      0.35      0.32       767\n",
      "        1910       0.34      0.18      0.23       466\n",
      "        1920       0.29      0.04      0.07       265\n",
      "        1930       0.61      0.20      0.30       217\n",
      "        1940       0.00      0.00      0.00        34\n",
      "        1950       0.00      0.00      0.00        23\n",
      "        1960       0.00      0.00      0.00        20\n",
      "        1970       0.20      0.06      0.09        18\n",
      "        1980       0.00      0.00      0.00        23\n",
      "        1990       0.00      0.00      0.00        10\n",
      "        2000       0.00      0.00      0.00        14\n",
      "\n",
      "    accuracy                           0.26     10000\n",
      "   macro avg       0.14      0.10      0.10     10000\n",
      "weighted avg       0.26      0.26      0.24     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Entraîner et évaluer avec TF-IDF et suivi de la perte\n",
    "#model_tfidf_sgd = train_with_loss_tracking(\n",
    "#    X_train_tfidf, X_test_tfidf, \n",
    "#    train_labels, test_labels,\n",
    "#    \"TF-IDF\",\n",
    "#    n_epochs=30\n",
    "#)\n",
    "\n",
    "# Entraîner et évaluer avec Embeddings et suivi de la perte\n",
    "model_embedding_sgd = train_with_loss_tracking(\n",
    "    X_train_embedding, X_test_embedding,\n",
    "    train_labels, test_labels,\n",
    "    \"Embeddings (CamemBERT)\",\n",
    "    n_epochs=50\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "02f2c811",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Interrogation d'un texte aléatoire ---\n",
      "📚 Titre : Les conserves de fruits pour la consommation familiale et pour la vente : la conservation des matières alimentaires dans les ménages, à la ferme et dans les coopératives agricoles\n",
      "🗓️  Date réelle : 1912\n",
      "----------------------------------------\n",
      "🤖 Prédiction (TF-IDF) : 1900s\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b376ef8bc3a4497917ecb41ebbd114e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🧠 Prédiction (Embeddings) : 1900s\n",
      "----------------------------------------\\n\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def predict_random_text(test_dataset):\n",
    "    \"\"\"\n",
    "    Sélectionne un texte aléatoire dans le jeu de données, le prédit avec les deux modèles\n",
    "    et affiche une comparaison.\n",
    "    \"\"\"\n",
    "    # 1. Sélectionner un exemple aléatoire\n",
    "    random_index = random.randint(0, len(test_dataset) - 1)\n",
    "    random_example = test_dataset[random_index]\n",
    "    \n",
    "    title = random_example.get('title', 'Titre non disponible')\n",
    "    true_date = random_example.get('date', 'Date non disponible')\n",
    "    full_text = random_example.get('complete_text', '')\n",
    "\n",
    "    print(f\"--- Interrogation d'un texte aléatoire ---\")\n",
    "    print(f\"📚 Titre : {title}\")\n",
    "    print(f\"🗓️  Date réelle : {true_date}\")\n",
    "    print(\"-\" * 40)\n",
    "\n",
    "    # 2. Nettoyer le texte comme pour l'entraînement\n",
    "    # La fonction clean_text renvoie un dictionnaire, nous extrayons le texte\n",
    "    cleaned_text_dict = clean_text(random_example)\n",
    "    cleaned_text = cleaned_text_dict['text']\n",
    "    \n",
    "    # Le texte doit être dans une liste pour les transformateurs\n",
    "    text_to_predict = [cleaned_text]\n",
    "\n",
    "    # 3. Prédiction avec le modèle TF-IDF\n",
    "    #tfidf_features = tfidf_vectorizer.transform(text_to_predict)\n",
    "    #prediction_tfidf = model_tfidf_sgd.predict(tfidf_features)[0]\n",
    "    #print(f\"🤖 Prédiction (TF-IDF) : {prediction_tfidf}s\")\n",
    "\n",
    "    # 4. Prédiction avec le modèle Embeddings\n",
    "    embedding_features = create_embeddings(text_to_predict, batch_size=1)\n",
    "    prediction_embedding = model_embedding_sgd.predict(embedding_features)[0]\n",
    "    print(f\"🧠 Prédiction (Embeddings) : {prediction_embedding}s\")\n",
    "    print(f\"----------------------------------------\\\\n\")\n",
    "\n",
    "# --- Lancer le test ---\n",
    "# Utilisez ds['test'] qui contient les métadonnées comme le titre.\n",
    "predict_random_text(ds['train'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e5a99b09-681b-4a44-82f7-7c1ffe1ac84d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Lancement de 20 essais de prédiction ---\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "472255ee1dbb4b95b14eb983f027b22c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 1/20 | Réel : 1743 | Embeddings : 1820\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19269415e0cd4499bfb606d82042a57f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 2/20 | Réel : 1910 | Embeddings : 1870\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de75adcf21f6484984a07e70e6bd05a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 3/20 | Réel : 1758 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f2f611e9d049f5a8e8c73cc086df22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 4/20 | Réel : 1894 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dd535bac4ec4e9fbc3dec4420885708",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 5/20 | Réel : 1621 | Embeddings : 1770\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7336da60a6544e79a29e485712273e39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 6/20 | Réel : 1834 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af189c27c73d47ad829fb52812b85c73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 7/20 | Réel : 1860 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41613bba41b548ef86a1dfc9ebf94081",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 8/20 | Réel : 1914 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b4db1579a1c483986ba743ff7badfda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 9/20 | Réel : 1832 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdbdd3a3cc234f50afb5201c02f71712",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 10/20 | Réel : 1876 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce842d3378794e96a537af7a58acb383",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 11/20 | Réel : 1885 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a72024d894b45528e594f863b871049",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 12/20 | Réel : 1883 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fae7fb5e8e03448c80a373a571a00373",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 13/20 | Réel : 1879 | Embeddings : 1880\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f98f7e2e9a140e9b4619fa4a520dcbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 14/20 | Réel : 1853 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f00a08f651474e5e944b9af348ec876e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 15/20 | Réel : 1799 | Embeddings : 1840\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abfbd36b9d9c4c6f92617bf36cb2e253",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 16/20 | Réel : 1865 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc987afe6e21477b8332e2be97d326b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 17/20 | Réel : 1833 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4921ec5c3ef347d284a9b07989334cc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 18/20 | Réel : 1828 | Embeddings : 1860\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9103f891674540a6ad26fe130a907294",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 19/20 | Réel : 1828 | Embeddings : 1870\n",
      "Création des embeddings pour 1 textes...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ad5af296e384ca3b36ab85c305bfab0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Essai 20/20 | Réel : 1872 | Embeddings : 1890\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 116972 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def test_and_plot_predictions(test_dataset, num_trials=15):\n",
    "    \"\"\"\n",
    "    Sélectionne plusieurs textes aléatoires, prédit leur date avec deux modèles,\n",
    "    et affiche un graphique comparatif des résultats.\n",
    "    \n",
    "    Args:\n",
    "        test_dataset (list): La liste d'exemples à tester.\n",
    "        num_trials (int): Le nombre d'essais à effectuer.\n",
    "    \"\"\"\n",
    "    real_dates = []\n",
    "    tfidf_predictions = []\n",
    "    embedding_predictions = []\n",
    "\n",
    "    print(f\"--- Lancement de {num_trials} essais de prédiction ---\")\n",
    "\n",
    "    for i in range(num_trials):\n",
    "        # 1. Sélectionner un exemple aléatoire\n",
    "        random_example = random.choice(test_dataset)\n",
    "        \n",
    "        # Extrait la date et la convertit en entier pour le graphique\n",
    "        # On suppose que la date est au format 'YYYYs' (ex: '1990s')\n",
    "        try:\n",
    "            true_date_str = random_example.get('date', '0s')\n",
    "            true_date_int = int(true_date_str[:4]) # Convertit '1990s' en 1990\n",
    "        except (ValueError, IndexError):\n",
    "            print(f\"Avertissement : impossible de parser la date '{true_date_str}'. Essai ignoré.\")\n",
    "            continue\n",
    "\n",
    "        # 2. Nettoyer le texte\n",
    "        cleaned_text = clean_text(random_example)['text']\n",
    "        text_to_predict = [cleaned_text]\n",
    "\n",
    "        # 3. Prédiction avec le modèle TF-IDF\n",
    "        #tfidf_features = tfidf_vectorizer.transform(text_to_predict)\n",
    "        #prediction_tfidf = model_tfidf_sgd.predict(tfidf_features)[0]\n",
    "\n",
    "        # 4. Prédiction avec le modèle Embeddings\n",
    "        embedding_features = create_embeddings(text_to_predict, batch_size=1)\n",
    "        prediction_embedding = model_embedding_sgd.predict(embedding_features)[0]\n",
    "        \n",
    "        # 5. Stocker les résultats\n",
    "        real_dates.append(true_date_int)\n",
    "        #tfidf_predictions.append(prediction_tfidf)\n",
    "        embedding_predictions.append(prediction_embedding)\n",
    "        \n",
    "        #print(f\"Essai {i+1}/{num_trials} | Réel : {true_date_int} | TF-IDF : {prediction_tfidf} | Embeddings : {prediction_embedding}\")\n",
    "        print(f\"Essai {i+1}/{num_trials} | Réel : {true_date_int} | Embeddings : {prediction_embedding}\")\n",
    "\n",
    "    # 6. Générer le graphique\n",
    "    if not real_dates:\n",
    "        print(\"Aucune donnée à afficher.\")\n",
    "        return\n",
    "\n",
    "    trials = range(1, len(real_dates) + 1)\n",
    "    \n",
    "    plt.style.use('seaborn-v0_8-whitegrid') # Style visuel agréable\n",
    "    plt.figure(figsize=(14, 7)) # Taille de la figure\n",
    "\n",
    "    # Tracer chaque série de données\n",
    "    plt.plot(trials, real_dates, 'o-', color='green', label='Date Réelle', markersize=8)\n",
    "    #plt.plot(trials, tfidf_predictions, 's--', color='blue', label='Prédiction (TF-IDF)')\n",
    "    plt.plot(trials, embedding_predictions, '^:', color='red', label='Prédiction (Embeddings)')\n",
    "\n",
    "    # Ajouter les titres et les légendes\n",
    "    plt.title('Comparaison des Prédictions de Date sur Plusieurs Essais', fontsize=16)\n",
    "    plt.xlabel(\"Numéro de l'essai\", fontsize=12)\n",
    "    plt.ylabel(\"Date (Année)\", fontsize=12)\n",
    "    plt.xticks(trials) # Assure que chaque essai a une graduation sur l'axe X\n",
    "    plt.legend(fontsize=10)\n",
    "    plt.tight_layout() # Ajuste le graphique pour qu'il s'affiche bien\n",
    "\n",
    "    # Afficher le graphique\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# --- Lancer le test ---\n",
    "# Utilisez ds['train'] et spécifiez le nombre d'essais souhaités (ex: 20)\n",
    "test_and_plot_predictions(ds['train'], num_trials=20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
