633 lines
33 KiB
Plaintext
633 lines
33 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "7374bcb9-58a6-4cad-b50b-2324c7290400",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Installation terminée.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Installation des librairies nécessaires\n",
|
||
"!pip install datasets transformers[torch] accelerate -q\n",
|
||
"# Librairie pour la métrique d'évaluation ROUGE\n",
|
||
"!pip install rouge_score -q\n",
|
||
"\n",
|
||
"print(\"Installation terminée.\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "02ccff69-deac-4188-9478-3a0eded45a16",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Modèle choisi pour cette session : moussaKam/barthez\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"import pandas as pd\n",
|
||
"from datasets import load_dataset\n",
|
||
"from transformers import AutoTokenizer\n",
|
||
"\n",
|
||
"# --- PARAMÈTRE PRINCIPAL DU NOTEBOOK ---\n",
|
||
"# Changez cette variable pour tester un autre modèle (ex: \"plguillou/t5-base-fr\")\n",
|
||
"model_checkpoint = \"moussaKam/barthez\" \n",
|
||
"\n",
|
||
"print(f\"Modèle choisi pour cette session : {model_checkpoint}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "1301a599-a67e-4566-875e-64917eaa2f0f",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Chargement du jeu de données PleIAs/French-PD-Books...\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"application/vnd.jupyter.widget-view+json": {
|
||
"model_id": "db73dc72d8a74927b2c6267c5783b007",
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
},
|
||
"text/plain": [
|
||
"Resolving data files: 0%| | 0/145 [00:00<?, ?it/s]"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
},
|
||
{
|
||
"data": {
|
||
"application/vnd.jupyter.widget-view+json": {
|
||
"model_id": "c22b630c067e498ca0944b20027d23a4",
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
},
|
||
"text/plain": [
|
||
"Loading dataset shards: 0%| | 0/145 [00:00<?, ?it/s]"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Jeu de données chargé.\n",
|
||
"Nombre d'exemples avant nettoyage : 289577\n",
|
||
"\n",
|
||
"Application du nettoyage sur le dataset...\n",
|
||
"Filtrage des données invalides...\n",
|
||
"Nombre d'exemples après nettoyage et filtrage : 1000\n",
|
||
"\n",
|
||
"--- Exemple après nettoyage ---\n",
|
||
"Date: 1923\n",
|
||
"Texte: MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT BRITANNIQUE DU il AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 PARIS IMPRIMERIE NATIONALE MDCGCCXMII MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT RRITANNIQUE DU 11 AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 LI\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Chargement du jeu de données\n",
|
||
"print(\"Chargement du jeu de données PleIAs/French-PD-Books...\")\n",
|
||
"ds = load_dataset(\"PleIAs/French-PD-Books\", split=\"train\")\n",
|
||
"print(\"Jeu de données chargé.\")\n",
|
||
"print(f\"Nombre d'exemples avant nettoyage : {len(ds)}\")\n",
|
||
"\n",
|
||
"# --- Vos fonctions de nettoyage et de vérification ---\n",
|
||
"def clean_text(example):\n",
|
||
" text = example[\"complete_text\"]\n",
|
||
" date = example.get(\"date\", None)\n",
|
||
"\n",
|
||
" # Si la date contient un \"-\", on essaie d'extraire l'année connue ou moyenne\n",
|
||
" if date and \"-\" in str(date):\n",
|
||
" parts = str(date).split(\"-\")\n",
|
||
" if (parts[1].isdigit() and len(parts[1]) == 4) and parts[0] == \"????\":\n",
|
||
" date = str(parts[1])\n",
|
||
" else:\n",
|
||
" date = str(parts[0])\n",
|
||
"\n",
|
||
" # Appliquer les nettoyages seulement si le texte est une chaîne de caractères\n",
|
||
" if isinstance(text, str):\n",
|
||
" # 1. Retirer les numéros de page\n",
|
||
" text = re.sub(r\"[—\\-–]\\s*\\d+\\s*[—\\-–]\", \" \", text)\n",
|
||
" # 2. Corriger les caractères échappés\n",
|
||
" text = text.replace(\"\\\\'\", \"'\").replace(\"\\\\\\\"\", \"\\\"\").replace(\"\\\\n\", \" \").replace(\"\\\\r\", \" \").replace(\"\\\\t\", \" \")\n",
|
||
" # 3. & 4. Corriger les mots coupés et espaces multiples (simplifié)\n",
|
||
" text = re.sub(r'([a-zàâäæçéèêëïîôùûüœ])\\s{2,}([a-zàâäæçéèêëïîôùûüœ])', r'\\1 \\2', text)\n",
|
||
" # 5. Normaliser les espaces\n",
|
||
" text = re.sub(r\"\\s+\", \" \", text)\n",
|
||
" # 6. Nettoyer les caractères spéciaux\n",
|
||
" text = re.sub(r\"[^\\w\\s\\.,;:\\?!'\\-\\\"«»À-ÖØ-öø-ÿœŒ]\", \" \", text)\n",
|
||
" # 7. Re-normaliser\n",
|
||
" text = re.sub(r\"\\s+\", \" \", text)\n",
|
||
" # 8. Corriger la ponctuation\n",
|
||
" text = re.sub(r\"\\s+([,.\\?!;:])\", r\"\\1\", text)\n",
|
||
" text = re.sub(r\"([,.\\?!;:])\\s*([,.\\?!;:])\", r\"\\1\\2\", text)\n",
|
||
" text = text.strip()\n",
|
||
" else:\n",
|
||
" text = \"\" # Si le texte n'est pas une chaîne, le remplacer par une chaîne vide\n",
|
||
"\n",
|
||
" return {\"text\": text, \"date\": str(date)}\n",
|
||
"\n",
|
||
"# Application de la fonction de nettoyage\n",
|
||
"print(\"\\nApplication du nettoyage sur le dataset...\")\n",
|
||
"reduced_ds = ds.shuffle(seed=42).select(range(1000))\n",
|
||
"ds_cleaned = reduced_ds.map(clean_text, num_proc=4) # Utilise plusieurs processeurs pour accélérer\n",
|
||
"\n",
|
||
"# Filtrage des données invalides (dates incorrectes ou textes vides)\n",
|
||
"print(\"Filtrage des données invalides...\")\n",
|
||
"ds_filtered = ds_cleaned.filter(lambda example: \n",
|
||
" example['date'].isdigit() and \n",
|
||
" len(example['date']) == 4 and \n",
|
||
" len(example['text']) > 50 # Garder des textes d'une longueur minimale\n",
|
||
")\n",
|
||
"\n",
|
||
"print(f\"Nombre d'exemples après nettoyage et filtrage : {len(ds_filtered)}\")\n",
|
||
"\n",
|
||
"# Affichage d'un exemple pour vérification\n",
|
||
"print(\"\\n--- Exemple après nettoyage ---\")\n",
|
||
"example = ds_filtered[500]\n",
|
||
"print(f\"Date: {example['date']}\")\n",
|
||
"print(f\"Texte: {example['text'][:400]}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "c761be3f-049c-4b48-aa37-4e2f445eac6a",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Formatage des données pour la tâche sequence-to-sequence...\n",
|
||
"Formatage terminé.\n",
|
||
"\n",
|
||
"--- Exemple au format Seq2Seq ---\n",
|
||
"SOURCE (entrée modèle):\n",
|
||
"réécris ce texte dans le style des années 1920: MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT BRITANNIQUE DU il AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 PARIS IMPRIMERIE NATIONALE MDCGCCXMII MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT RRITANNIQUE DU 11 AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 LIVRE «.UNE. Réponse du Gouv. français. MINISTÈRE DES \n",
|
||
"\n",
|
||
"TARGET (sortie attendue):\n",
|
||
"MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT BRITANNIQUE DU il AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 PARIS IMPRIMERIE NATIONALE MDCGCCXMII MINISTÈRE DES AFFAIRES ÉTRANGÈRES DOCUMENTS DIPLOMATIQUES REPONSE DU GOUVERNEMENT FRANÇAIS À LA LETTRE DU GOUVERNEMENT RRITANNIQUE DU 11 AOÛT 1923 SUR LES RÉPARATIONS 20 AOUT 1923 LI\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Fonction pour créer les entrées et sorties du modèle\n",
|
||
"def create_seq2seq_format(example):\n",
|
||
" text = example['text']\n",
|
||
" year = int(example['date'])\n",
|
||
" \n",
|
||
" # Création de la décennie (ex: 1854 -> 1850)\n",
|
||
" decade = (year // 10) * 10\n",
|
||
" \n",
|
||
" # Formatage de l'entrée pour le modèle\n",
|
||
" # Le préfixe est crucial pour que le modèle comprenne la tâche\n",
|
||
" prefix = f\"réécris ce texte dans le style des années {decade}: \"\n",
|
||
" \n",
|
||
" # Source : ce que le modèle voit en entrée\n",
|
||
" example['source'] = prefix + text\n",
|
||
" # Cible : ce que le modèle doit apprendre à prédire\n",
|
||
" example['target'] = text\n",
|
||
" \n",
|
||
" return example\n",
|
||
"\n",
|
||
"print(\"\\nFormatage des données pour la tâche sequence-to-sequence...\")\n",
|
||
"ds_seq2seq = ds_filtered.map(create_seq2seq_format, num_proc=4)\n",
|
||
"\n",
|
||
"print(\"Formatage terminé.\")\n",
|
||
"print(\"\\n--- Exemple au format Seq2Seq ---\")\n",
|
||
"example = ds_seq2seq[500]\n",
|
||
"print(f\"SOURCE (entrée modèle):\\n{example['source'][:500]}\\n\")\n",
|
||
"print(f\"TARGET (sortie attendue):\\n{example['target'][:400]}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "4861e244-c3e1-4f94-9c7f-9afb8cd30028",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Lancement de la tokenisation avec le tokenizer de 'moussaKam/barthez'...\n",
|
||
"\n",
|
||
"--- Traitement manuel des 1000 premiers exemples ---\n",
|
||
"--- Traitement et tokenisation des 1000 exemples ---\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"application/vnd.jupyter.widget-view+json": {
|
||
"model_id": "0b9ca76da4d040b098455e50f23f1fca",
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
},
|
||
"text/plain": [
|
||
" 0%| | 0/1000 [00:00<?, ?it/s]"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"✅ Tokenisation manuelle terminée.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import os\n",
|
||
"from tqdm.auto import tqdm\n",
|
||
"from datasets import Dataset\n",
|
||
"\n",
|
||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
|
||
"\n",
|
||
"# Chargement du tokenizer\n",
|
||
"tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
|
||
"\n",
|
||
"# Longueurs maximales pour les séquences source et cible\n",
|
||
"# À ajuster en fonction de la mémoire disponible et des capacités du modèle\n",
|
||
"max_input_length = 512\n",
|
||
"max_target_length = 512\n",
|
||
"\n",
|
||
"def tokenize_function(examples):\n",
|
||
" # Tokenisation des textes sources\n",
|
||
" model_inputs = tokenizer(\n",
|
||
" examples[\"source\"], \n",
|
||
" max_length=max_input_length, \n",
|
||
" truncation=True, \n",
|
||
" padding=\"max_length\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # Tokenisation des textes cibles pour les labels\n",
|
||
" labels = tokenizer(\n",
|
||
" text_target=examples[\"target\"], \n",
|
||
" max_length=max_target_length, \n",
|
||
" truncation=True, \n",
|
||
" padding=\"max_length\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # Assigner les labels tokenisés à la clé 'labels' que le modèle attend\n",
|
||
" model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
|
||
" \n",
|
||
" # Remplacer les tokens de padding dans les labels par -100 pour qu'ils soient ignorés dans le calcul de la loss\n",
|
||
" for i, label_ids in enumerate(model_inputs[\"labels\"]):\n",
|
||
" model_inputs[\"labels\"][i] = [l if l != tokenizer.pad_token_id else -100 for l in label_ids]\n",
|
||
"\n",
|
||
" return model_inputs\n",
|
||
"\n",
|
||
"print(f\"\\nLancement de la tokenisation avec le tokenizer de '{model_checkpoint}'...\")\n",
|
||
"\n",
|
||
"# On va tester sur les 1000 premiers exemples\n",
|
||
"num_to_test = 1000\n",
|
||
"problematic_index = -1\n",
|
||
"\n",
|
||
"print(f\"\\n--- Traitement manuel des {num_to_test} premiers exemples ---\")\n",
|
||
"\n",
|
||
"# La barre de progression (tqdm) nous montrera exactement où ça bloque\n",
|
||
"# Liste pour stocker les résultats tokenisés\n",
|
||
"tokenized_results = []\n",
|
||
"\n",
|
||
"# On utilise l'intégralité du dataset\n",
|
||
"num_to_process = len(ds_seq2seq) \n",
|
||
"\n",
|
||
"print(f\"--- Traitement et tokenisation des {num_to_process} exemples ---\")\n",
|
||
"\n",
|
||
"# La boucle parcourt tout le dataset\n",
|
||
"for i in tqdm(range(num_to_process)):\n",
|
||
" # On récupère un exemple\n",
|
||
" example = ds_seq2seq[i]\n",
|
||
" \n",
|
||
" # On le met dans un format de \"batch\" avec un seul élément\n",
|
||
" batch = {\n",
|
||
" \"source\": [example[\"source\"]],\n",
|
||
" \"target\": [example[\"target\"]]\n",
|
||
" }\n",
|
||
" \n",
|
||
" # On applique la fonction de tokenisation\n",
|
||
" tokenized_output = tokenize_function(batch)\n",
|
||
" \n",
|
||
" # On \"déballe\" le résultat du batch pour n'avoir qu'un seul dictionnaire\n",
|
||
" # et on l'ajoute à notre liste de résultats\n",
|
||
" tokenized_results.append({\n",
|
||
" 'input_ids': tokenized_output['input_ids'][0],\n",
|
||
" 'attention_mask': tokenized_output['attention_mask'][0],\n",
|
||
" 'labels': tokenized_output['labels'][0]\n",
|
||
" })\n",
|
||
"\n",
|
||
"print(\"\\n✅ Tokenisation manuelle terminée.\")\n",
|
||
"\n",
|
||
"# Conversion de la liste de dictionnaires en un objet Dataset de Hugging Face\n",
|
||
"ds_tokenized = Dataset.from_list(tokenized_results)\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "24742f22",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Taille du jeu d'entraînement : 900\n",
|
||
"Taille du jeu de validation : 100\n",
|
||
"DatasetDict({\n",
|
||
" train: Dataset({\n",
|
||
" features: ['input_ids', 'attention_mask', 'labels'],\n",
|
||
" num_rows: 900\n",
|
||
" })\n",
|
||
" validation: Dataset({\n",
|
||
" features: ['input_ids', 'attention_mask', 'labels'],\n",
|
||
" num_rows: 100\n",
|
||
" })\n",
|
||
"})\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Fractionnement du jeu de données : 90% pour l'entraînement, 10% pour la validation\n",
|
||
"ds_final = ds_tokenized.train_test_split(test_size=0.1)\n",
|
||
"\n",
|
||
"# Renommage pour plus de clarté\n",
|
||
"ds_final[\"train\"] = ds_final.pop(\"train\")\n",
|
||
"ds_final[\"validation\"] = ds_final.pop(\"test\")\n",
|
||
"\n",
|
||
"print(\"Taille du jeu d'entraînement :\", len(ds_final[\"train\"]))\n",
|
||
"print(\"Taille du jeu de validation :\", len(ds_final[\"validation\"]))\n",
|
||
"print(ds_final)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "b5c47718",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Note: you may need to restart the kernel to use updated packages.\n",
|
||
"Fonction de calcul des métriques définie.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%pip install evaluate -q\n",
|
||
"\n",
|
||
"import numpy as np\n",
|
||
"import evaluate # Librairie de Hugging Face pour les métriques\n",
|
||
"\n",
|
||
"# Chargement de la métrique ROUGE\n",
|
||
"rouge_metric = evaluate.load(\"rouge\")\n",
|
||
"\n",
|
||
"def compute_metrics(eval_preds):\n",
|
||
" preds, labels = eval_preds\n",
|
||
"\n",
|
||
" # Décodage des prédictions générées par le modèle\n",
|
||
" # On remplace les tokens -100 (ignorés dans la loss) par le token de padding\n",
|
||
" preds[preds == -100] = tokenizer.pad_token_id\n",
|
||
" decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n",
|
||
" \n",
|
||
" # Décodage des vrais labels de référence\n",
|
||
" labels[labels == -100] = tokenizer.pad_token_id\n",
|
||
" decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
|
||
" \n",
|
||
" # Calcul du score ROUGE\n",
|
||
" result = rouge_metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
|
||
" \n",
|
||
" # Ajout d'une métrique sur la longueur moyenne des prédictions\n",
|
||
" prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n",
|
||
" result[\"gen_len\"] = np.mean(prediction_lens)\n",
|
||
" \n",
|
||
" return {k: round(v, 4) for k, v in result.items()}\n",
|
||
"\n",
|
||
"print(\"Fonction de calcul des métriques définie.\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6d675f3e",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\u001b[33mWARNING: Ignoring invalid distribution ~orch (/opt/conda/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n",
|
||
"\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/opt/conda/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n",
|
||
"\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/opt/conda/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n",
|
||
"\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Install/upgrade all libraries in one command for better dependency resolution\n",
|
||
"%pip install -U bitsandbytes accelerate peft transformers torch -q\n",
|
||
"\n",
|
||
"import torch\n",
|
||
"import os\n",
|
||
"from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
|
||
"from transformers import BitsAndBytesConfig\n",
|
||
"from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
|
||
"\n",
|
||
"# Isolate your script on the GPU 1\n",
|
||
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
|
||
"\n",
|
||
"# --- 1. Configuration de la Quantization (QLoRA) ---\n",
|
||
"quantization_config = BitsAndBytesConfig(\n",
|
||
" load_in_4bit=True,\n",
|
||
" bnb_4bit_quant_type=\"nf4\",\n",
|
||
" bnb_4bit_compute_dtype=torch.bfloat16,\n",
|
||
" bnb_4bit_use_double_quant=True,\n",
|
||
")\n",
|
||
"\n",
|
||
"# --- 2. Chargement du Modèle quantizé ---\n",
|
||
"model = AutoModelForSeq2SeqLM.from_pretrained(\n",
|
||
" model_checkpoint,\n",
|
||
" quantization_config=quantization_config,\n",
|
||
" device_map=\"auto\"\n",
|
||
")\n",
|
||
"\n",
|
||
"print(\"Modèle chargé en 4-bit (QLoRA) sur le GPU 1.\")\n",
|
||
"\n",
|
||
"# --- 3. Préparation du modèle et Configuration de LoRA ---\n",
|
||
"model = prepare_model_for_kbit_training(model)\n",
|
||
"\n",
|
||
"# ## CORRECTIF ##: Ciblez toutes les couches linéaires pour assurer un chemin de gradient complet.\n",
|
||
"lora_config = LoraConfig(\n",
|
||
" r=16,\n",
|
||
" lora_alpha=32,\n",
|
||
" target_modules=[\"q_proj\", \"v_proj\", \"k_proj\", \"out_proj\", \"fc1\", \"fc2\"],\n",
|
||
" lora_dropout=0.05,\n",
|
||
" bias=\"none\",\n",
|
||
" task_type=\"SEQ_2_SEQ_LM\"\n",
|
||
")\n",
|
||
"\n",
|
||
"model = get_peft_model(model, lora_config)\n",
|
||
"model.print_trainable_parameters()\n",
|
||
"\n",
|
||
"# --- Le reste du code reste identique ---\n",
|
||
"\n",
|
||
"data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n",
|
||
"model_name = model_checkpoint.split(\"/\")[-1]\n",
|
||
"output_dir = f\"{model_name}-qlora-finetuned-style-transfer\"\n",
|
||
"\n",
|
||
"training_args = Seq2SeqTrainingArguments(\n",
|
||
" output_dir=output_dir,\n",
|
||
" learning_rate=2e-4,\n",
|
||
" per_device_train_batch_size=4,\n",
|
||
" per_device_eval_batch_size=4,\n",
|
||
" gradient_accumulation_steps=4,\n",
|
||
" weight_decay=0.01,\n",
|
||
" num_train_epochs=3,\n",
|
||
" eval_strategy=\"epoch\",\n",
|
||
" save_strategy=\"epoch\",\n",
|
||
" load_best_model_at_end=True,\n",
|
||
" predict_with_generate=True,\n",
|
||
" fp16=False,\n",
|
||
" bf16=True,\n",
|
||
" push_to_hub=False,\n",
|
||
")\n",
|
||
"\n",
|
||
"# On retire l'argument 'tokenizer' qui est obsolète\n",
|
||
"trainer = Seq2SeqTrainer(\n",
|
||
" model=model,\n",
|
||
" args=training_args,\n",
|
||
" train_dataset=ds_final[\"train\"],\n",
|
||
" eval_dataset=ds_final[\"validation\"],\n",
|
||
" data_collator=data_collator,\n",
|
||
" compute_metrics=compute_metrics,\n",
|
||
")\n",
|
||
"\n",
|
||
"print(\"Entraîneur (Trainer) prêt pour l'entraînement avec QLoRA sur le GPU 1.\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"id": "7249d1e7",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"🚀 Début de l'entraînement...\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/opt/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py:929: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.5 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
|
||
" first_ctx: bool = False,\n",
|
||
"/opt/conda/lib/python3.13/site-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
|
||
" warnings.warn(\n"
|
||
]
|
||
},
|
||
{
|
||
"ename": "RuntimeError",
|
||
"evalue": "element 0 of tensors does not require grad and does not have a grad_fn",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||
"\u001b[31mRuntimeError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Lancement de l'entraînement !\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m🚀 Début de l\u001b[39m\u001b[33m'\u001b[39m\u001b[33mentraînement...\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[43mtrainer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m✅ Entraînement terminé !\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 6\u001b[39m \u001b[38;5;66;03m# Sauvegarde du meilleur modèle dans le dossier de sortie\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/transformers/trainer.py:2325\u001b[39m, in \u001b[36mTrainer.train\u001b[39m\u001b[34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[39m\n\u001b[32m 2323\u001b[39m hf_hub_utils.enable_progress_bars()\n\u001b[32m 2324\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2325\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 2326\u001b[39m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m=\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2327\u001b[39m \u001b[43m \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2328\u001b[39m \u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2329\u001b[39m \u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m=\u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2330\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/transformers/trainer.py:2674\u001b[39m, in \u001b[36mTrainer._inner_training_loop\u001b[39m\u001b[34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[39m\n\u001b[32m 2667\u001b[39m context = (\n\u001b[32m 2668\u001b[39m functools.partial(\u001b[38;5;28mself\u001b[39m.accelerator.no_sync, model=model)\n\u001b[32m 2669\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m i != \u001b[38;5;28mlen\u001b[39m(batch_samples) - \u001b[32m1\u001b[39m\n\u001b[32m 2670\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.accelerator.distributed_type != DistributedType.DEEPSPEED\n\u001b[32m 2671\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m contextlib.nullcontext\n\u001b[32m 2672\u001b[39m )\n\u001b[32m 2673\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m context():\n\u001b[32m-> \u001b[39m\u001b[32m2674\u001b[39m tr_loss_step = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_items_in_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2676\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[32m 2677\u001b[39m args.logging_nan_inf_filter\n\u001b[32m 2678\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[32m 2679\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m (torch.isnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch.isinf(tr_loss_step))\n\u001b[32m 2680\u001b[39m ):\n\u001b[32m 2681\u001b[39m \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[32m 2682\u001b[39m tr_loss = tr_loss + tr_loss / (\u001b[32m1\u001b[39m + \u001b[38;5;28mself\u001b[39m.state.global_step - \u001b[38;5;28mself\u001b[39m._globalstep_last_logged)\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/transformers/trainer.py:4071\u001b[39m, in \u001b[36mTrainer.training_step\u001b[39m\u001b[34m(***failed resolving arguments***)\u001b[39m\n\u001b[32m 4068\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.accelerator.distributed_type == DistributedType.DEEPSPEED:\n\u001b[32m 4069\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mscale_wrt_gas\u001b[39m\u001b[33m\"\u001b[39m] = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m4071\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43maccelerator\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4073\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m loss.detach()\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/accelerate/accelerator.py:2734\u001b[39m, in \u001b[36mAccelerator.backward\u001b[39m\u001b[34m(self, loss, **kwargs)\u001b[39m\n\u001b[32m 2732\u001b[39m \u001b[38;5;28mself\u001b[39m.lomo_backward(loss, learning_rate)\n\u001b[32m 2733\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2734\u001b[39m \u001b[43mloss\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/torch/_tensor.py:647\u001b[39m, in \u001b[36mTensor.backward\u001b[39m\u001b[34m(self, gradient, retain_graph, create_graph, inputs)\u001b[39m\n\u001b[32m 637\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 638\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[32m 639\u001b[39m Tensor.backward,\n\u001b[32m 640\u001b[39m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[32m (...)\u001b[39m\u001b[32m 645\u001b[39m inputs=inputs,\n\u001b[32m 646\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m647\u001b[39m \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mautograd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 648\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m=\u001b[49m\u001b[43minputs\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/torch/autograd/__init__.py:354\u001b[39m, in \u001b[36mbackward\u001b[39m\u001b[34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[39m\n\u001b[32m 349\u001b[39m retain_graph = create_graph\n\u001b[32m 351\u001b[39m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[32m 352\u001b[39m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[32m 353\u001b[39m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m354\u001b[39m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 355\u001b[39m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 356\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 357\u001b[39m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 358\u001b[39m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 359\u001b[39m \u001b[43m \u001b[49m\u001b[43minputs_tuple\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 360\u001b[39m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 361\u001b[39m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 362\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/opt/conda/lib/python3.13/site-packages/torch/autograd/graph.py:829\u001b[39m, in \u001b[36m_engine_run_backward\u001b[39m\u001b[34m(t_outputs, *args, **kwargs)\u001b[39m\n\u001b[32m 827\u001b[39m unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[32m 828\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m829\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_execution_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[32m 830\u001b[39m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\n\u001b[32m 831\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[32m 832\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 833\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n",
|
||
"\u001b[31mRuntimeError\u001b[39m: element 0 of tensors does not require grad and does not have a grad_fn"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Lancement de l'entraînement !\n",
|
||
"print(\"🚀 Début de l'entraînement...\")\n",
|
||
"trainer.train()\n",
|
||
"print(\"✅ Entraînement terminé !\")\n",
|
||
"\n",
|
||
"# Sauvegarde du meilleur modèle dans le dossier de sortie\n",
|
||
"trainer.save_model()\n",
|
||
"print(f\"Meilleur modèle sauvegardé dans le dossier : {output_dir}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "5c62301f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Évaluation finale du meilleur modèle sur le jeu de validation\n",
|
||
"print(\"\\nÉvaluation finale du modèle...\")\n",
|
||
"eval_results = trainer.evaluate()\n",
|
||
"\n",
|
||
"print(\"\\n--- Résultats de l'évaluation ---\")\n",
|
||
"for key, value in eval_results.items():\n",
|
||
" print(f\"{key}: {value}\")"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.13.7"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|