452 lines
22 KiB
Plaintext
452 lines
22 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Partie 1 — Nettoyage du Texte\n",
|
|
"source .venv/bin/activate\n",
|
|
"cd \n",
|
|
"python nom du fichier\n",
|
|
"On prend un texte descriptif fourni par l'utilisateur et on le nettoie étape par étape.\n",
|
|
"\n",
|
|
"```\n",
|
|
"Texte brut → Noise Removal → Tokenization → Stopwords → Lemmatization → Texte propre\n",
|
|
"```"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Installation des dépendances"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Dépendances installées !\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"!pip install nltk --quiet\n",
|
|
"\n",
|
|
"import nltk\n",
|
|
"nltk.download('punkt', quiet=True)\n",
|
|
"nltk.download('punkt_tab', quiet=True)\n",
|
|
"nltk.download('stopwords', quiet=True)\n",
|
|
"nltk.download('wordnet', quiet=True)\n",
|
|
"nltk.download('averaged_perceptron_tagger', quiet=True)\n",
|
|
"nltk.download('averaged_perceptron_tagger_eng', quiet=True)\n",
|
|
"\n",
|
|
"print(\"Dépendances installées !\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Saisie du texte utilisateur"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 86,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "KeyboardInterrupt",
|
|
"evalue": "Interrupted by user",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[86]\u001b[39m\u001b[32m, line 65\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# test_texts = [\u001b[39;00m\n\u001b[32m 2\u001b[39m \n\u001b[32m 3\u001b[39m \u001b[38;5;66;03m# # 0 — Dragon de feu (texte original)\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 62\u001b[39m \n\u001b[32m 63\u001b[39m \u001b[38;5;66;03m# print(f\" Texte de test n°{INDEX} :\")\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m65\u001b[39m raw_text = \u001b[38;5;28;43minput\u001b[39;49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mDécrivez votre Pokémon : \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 67\u001b[39m \u001b[38;5;28mprint\u001b[39m(raw_text)\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/lib/python3.12/site-packages/ipykernel/kernelbase.py:1403\u001b[39m, in \u001b[36mKernel.raw_input\u001b[39m\u001b[34m(self, prompt)\u001b[39m\n\u001b[32m 1401\u001b[39m msg = \u001b[33m\"\u001b[39m\u001b[33mraw_input was called, but this frontend does not support input requests.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 1402\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m StdinNotImplementedError(msg)\n\u001b[32m-> \u001b[39m\u001b[32m1403\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_input_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1404\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1405\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_get_shell_context_var\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_shell_parent_ident\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1406\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mget_parent\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mshell\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1407\u001b[39m \u001b[43m \u001b[49m\u001b[43mpassword\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 1408\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/lib/python3.12/site-packages/ipykernel/kernelbase.py:1448\u001b[39m, in \u001b[36mKernel._input_request\u001b[39m\u001b[34m(self, prompt, ident, parent, password)\u001b[39m\n\u001b[32m 1445\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[32m 1446\u001b[39m \u001b[38;5;66;03m# re-raise KeyboardInterrupt, to truncate traceback\u001b[39;00m\n\u001b[32m 1447\u001b[39m msg = \u001b[33m\"\u001b[39m\u001b[33mInterrupted by user\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m-> \u001b[39m\u001b[32m1448\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1449\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[32m 1450\u001b[39m \u001b[38;5;28mself\u001b[39m.log.warning(\u001b[33m\"\u001b[39m\u001b[33mInvalid Message:\u001b[39m\u001b[33m\"\u001b[39m, exc_info=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m: Interrupted by user"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# test_texts = [\n",
|
|
"\n",
|
|
"# # 0 — Dragon de feu (texte original)\n",
|
|
"# \"\"\"\n",
|
|
"# This is a HUGE fire dragon!!! It has got massive red wings and shoots\n",
|
|
"# powerfull flames from its mouth... It's super fast n really strong!!\n",
|
|
"# Its body is coverd with shiny golden scales & it lives in volcanos.\n",
|
|
"# it luv to fight other pokémons and is very very aggressive >:(\n",
|
|
"# I want to call it Pyrokar.\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# # 1 — Pokémon aquatique calme\n",
|
|
"# \"\"\"\n",
|
|
"# My pokemon is called Aqualis!! its a small blue sea creature w/ big\n",
|
|
"# shiny eyes... very calm n gentle :) it swims super fast in deep oceans\n",
|
|
"# and can breath underwater 4ever. it glows in the dark like a lanternfish\n",
|
|
"# and heals other pokemons with its tears!!! luv this lil guy so much omg\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# # 2 — Pokémon électrique agressif\n",
|
|
"# \"\"\"\n",
|
|
"# ZAPTHORN is da name!! its an electric wolf w/ yellow n black fur and\n",
|
|
"# giant thunder claws !!! it runz at lightning speed thru storms & shoots\n",
|
|
"# bolts from its tail... super scary n powerfull enemy 4 sure >:D\n",
|
|
"# nobody can catch it bcz it disappears in the clouds when threatened\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# # 3 — Pokémon plante timide\n",
|
|
"# \"\"\"\n",
|
|
"# i wanna name it Sylverion... its a shy deer-like pokemon covered in\n",
|
|
"# beautiful flowers n vines. it lives deep in enchanted forests & only\n",
|
|
"# comes out at nite. its antlers r made of ancient wood n bloom every\n",
|
|
"# spring!! it can make plants grow super fast around it... so magical omg\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# # 4 — Pokémon glace / fantôme\n",
|
|
"# \"\"\"\n",
|
|
"# This haunted ice spirit is called Glacyra!!! it floats thru frozen\n",
|
|
"# mountains leavin icy footprints everywhere... its body is trasnparent\n",
|
|
"# like glass n u can see its frozen heart inside >< it whispers 2 trainers\n",
|
|
"# in their sleep n freezes everything it touchez. very misunderstood tbh\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# # 5 — Pokémon combat en franglais\n",
|
|
"# \"\"\"\n",
|
|
"# My Pokémon is called Ferroknux!! It's a big metal gorilla with\n",
|
|
"# gigantic iron fists and super thick armor on its chest... it smashes\n",
|
|
"# rocks with bare hands and trains all day, every day in the mountains!!\n",
|
|
"# Very strong and very aggressive, but loyal to its trainer 4ever :)\n",
|
|
"# \"\"\",\n",
|
|
"# #6 \n",
|
|
"# \"\"\"\n",
|
|
"# Furret is a long, slender, and agile creature with soft fur and a flexible body that allows it to move gracefully through narrow tunnels and hidden pathways. This normal-type Pokémon builds intricate nests perfectly shaped to fit its elongated form, making them nearly impossible for other creatures to enter. Despite its gentle and calm nature, Furret can become surprisingly energetic in battle, using its powerful tail to smash opponents with swift and playful attacks. It is often seen wandering across fields and forests, curiously observing its surroundings, and it shares a close bond with its pre-evolution, Sentret. Known for its endurance and cheerful spirit, Furret can quickly recover its energy, always feeling fine and ready to continue exploring or fighting alongside its trainer.\n",
|
|
"# \"\"\",\n",
|
|
"\n",
|
|
"# ]\n",
|
|
"\n",
|
|
"# # 👇 Changez cet index pour tester un autre texte\n",
|
|
"# INDEX = 6\n",
|
|
"\n",
|
|
"# raw_text = test_texts[INDEX]\n",
|
|
"\n",
|
|
"# print(f\" Texte de test n°{INDEX} :\")\n",
|
|
"\n",
|
|
"raw_text = input(\"Décrivez votre Pokémon : \")\n",
|
|
"\n",
|
|
"print(raw_text)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Étape 1 — Noise Removal\n",
|
|
"\n",
|
|
"On supprime la ponctuation, les caractères spéciaux, les mots trop courts, et on met tout en minuscules.\n",
|
|
"\n",
|
|
"> *Cours page 25-29 — `removePunctuation`, `removeShortWords`, `removePattern`*"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 80,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" Après Noise Removal :\n",
|
|
"furret long slender and agile creature with soft fur and flexible body that allows move gracefully through narrow tunnels and hidden pathways this normaltype pokmon builds intricate nests perfectly shaped fit its elongated form making them nearly impossible for other creatures enter despite its gentle and calm nature furret can become surprisingly energetic battle using its powerful tail smash opponents with swift and playful attacks often seen wandering across fields and forests curiously observing its surroundings and shares close bond with its preevolution sentret known for its endurance and cheerful spirit furret can quickly recover its energy always feeling fine and ready continue exploring fighting alongside its trainer\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import re\n",
|
|
"import string\n",
|
|
"\n",
|
|
"def remove_punctuation(text):\n",
|
|
" \"\"\"Supprime la ponctuation du texte.\"\"\"\n",
|
|
" mapping_table = text.maketrans('', '', string.punctuation)\n",
|
|
" return text.translate(mapping_table)\n",
|
|
"\n",
|
|
"def remove_special_chars(text):\n",
|
|
" \"\"\"Supprime les caractères non-ASCII (emojis, accents parasites...).\"\"\"\n",
|
|
" text = text.encode('ascii', 'ignore').decode('ascii')\n",
|
|
" text = re.sub(r'[^a-zA-Z\\s]', ' ', text)\n",
|
|
" return re.sub(r'\\s+', ' ', text).strip()\n",
|
|
"\n",
|
|
"def remove_short_words(text, min_len=3):\n",
|
|
" \"\"\"Supprime les mots de moins de min_len caractères.\"\"\"\n",
|
|
" return \" \".join([word for word in text.split() if len(word) >= min_len])\n",
|
|
"\n",
|
|
"\n",
|
|
"def remove_alphanum_words(text):\n",
|
|
" \"\"\"Supprime les mots qui contiennent à la fois des lettres et des chiffres\n",
|
|
" (ex: '4ever', 'n1', '2night', 'runz4', 'mp3').\"\"\"\n",
|
|
" words = text.split()\n",
|
|
" cleaned = [word for word in words\n",
|
|
" if not (re.search(r'[a-zA-Z]', word) and re.search(r'[0-9]', word))]\n",
|
|
" return \" \".join(cleaned)\n",
|
|
"\n",
|
|
"# Application\n",
|
|
"text = raw_text.lower() # minuscules\n",
|
|
"text = remove_punctuation(text) # ponctuation\n",
|
|
"text = remove_alphanum_words(text) \n",
|
|
"text = remove_special_chars(text) # caractères spéciaux\n",
|
|
"text = remove_short_words(text) # mots trop courts\n",
|
|
"\n",
|
|
"print(\" Après Noise Removal :\")\n",
|
|
"print(text)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Étape 2 — Object Standardization\n",
|
|
"\n",
|
|
"On remplace les abréviations et l'argot par leurs formes standard.\n",
|
|
"\n",
|
|
"> *Cours page 38 — lookup table `standardize`*"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 81,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" Après Standardisation :\n",
|
|
"furret long slender and agile creature with soft fur and flexible body that allows move gracefully through narrow tunnels and hidden pathways this normaltype pokmon builds intricate nests perfectly shaped fit its elongated form making them nearly impossible for other creatures enter despite its gentle and calm nature furret can become surprisingly energetic battle using its powerful tail smash opponents with swift and playful attacks often seen wandering across fields and forests curiously observing its surroundings and shares close bond with its preevolution sentret known for its endurance and cheerful spirit furret can quickly recover its energy always feeling fine and ready continue exploring fighting alongside its trainer\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"SLANG_LOOKUP = {\n",
|
|
" \"n\": \"and\",\n",
|
|
" \"luv\": \"love\",\n",
|
|
" \"r\": \"are\",\n",
|
|
" \"u\": \"you\",\n",
|
|
" \"ur\": \"your\",\n",
|
|
" \"gonna\": \"going to\",\n",
|
|
" \"wanna\": \"want to\",\n",
|
|
" \"gotta\": \"got to\",\n",
|
|
" \"pokemons\": \"pokemon\",\n",
|
|
" \"pokmons\": \"pokemon\",\n",
|
|
" \"bcz\": \"because\",\n",
|
|
"}\n",
|
|
"\n",
|
|
"def standardize(text, lookup=SLANG_LOOKUP):\n",
|
|
" \"\"\"Remplace les mots d'argot par leur forme standard.\"\"\"\n",
|
|
" words = text.split()\n",
|
|
" return \" \".join([lookup.get(word, word) for word in words])\n",
|
|
"\n",
|
|
"text = standardize(text)\n",
|
|
"\n",
|
|
"print(\" Après Standardisation :\")\n",
|
|
"print(text)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Étape 3 — Tokenization\n",
|
|
"\n",
|
|
"On découpe le texte en tokens individuels.\n",
|
|
"\n",
|
|
"> *Cours page 31 — `word_tokenize` (NLTK)*"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 82,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" 108 tokens :\n",
|
|
"['furret', 'long', 'slender', 'and', 'agile', 'creature', 'with', 'soft', 'fur', 'and', 'flexible', 'body', 'that', 'allows', 'move', 'gracefully', 'through', 'narrow', 'tunnels', 'and', 'hidden', 'pathways', 'this', 'normaltype', 'pokmon', 'builds', 'intricate', 'nests', 'perfectly', 'shaped', 'fit', 'its', 'elongated', 'form', 'making', 'them', 'nearly', 'impossible', 'for', 'other', 'creatures', 'enter', 'despite', 'its', 'gentle', 'and', 'calm', 'nature', 'furret', 'can', 'become', 'surprisingly', 'energetic', 'battle', 'using', 'its', 'powerful', 'tail', 'smash', 'opponents', 'with', 'swift', 'and', 'playful', 'attacks', 'often', 'seen', 'wandering', 'across', 'fields', 'and', 'forests', 'curiously', 'observing', 'its', 'surroundings', 'and', 'shares', 'close', 'bond', 'with', 'its', 'preevolution', 'sentret', 'known', 'for', 'its', 'endurance', 'and', 'cheerful', 'spirit', 'furret', 'can', 'quickly', 'recover', 'its', 'energy', 'always', 'feeling', 'fine', 'and', 'ready', 'continue', 'exploring', 'fighting', 'alongside', 'its', 'trainer']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk import word_tokenize\n",
|
|
"\n",
|
|
"tokens = word_tokenize(text)\n",
|
|
"\n",
|
|
"print(f\" {len(tokens)} tokens :\")\n",
|
|
"print(tokens)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Étape 4 — Suppression des Stopwords\n",
|
|
"\n",
|
|
"On retire les mots grammaticaux qui n'apportent pas de sens (\"the\", \"is\", \"a\"...).\n",
|
|
"\n",
|
|
"> *Cours page 27 — `cleanTextGT` avec `stopwords` (NLTK)*"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 83,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Tokens après suppression des stopwords :\n",
|
|
"['furret', 'long', 'slender', 'agile', 'creature', 'soft', 'fur', 'flexible', 'body', 'allows', 'move', 'gracefully', 'narrow', 'tunnels', 'hidden', 'pathways', 'normaltype', 'pokmon', 'builds', 'intricate', 'nests', 'perfectly', 'shaped', 'fit', 'elongated', 'form', 'making', 'nearly', 'impossible', 'creatures', 'enter', 'despite', 'gentle', 'calm', 'nature', 'furret', 'become', 'surprisingly', 'energetic', 'battle', 'using', 'powerful', 'tail', 'smash', 'opponents', 'swift', 'playful', 'attacks', 'often', 'seen', 'wandering', 'across', 'fields', 'forests', 'curiously', 'observing', 'surroundings', 'shares', 'close', 'bond', 'preevolution', 'sentret', 'known', 'endurance', 'cheerful', 'spirit', 'furret', 'quickly', 'recover', 'energy', 'always', 'feeling', 'fine', 'ready', 'continue', 'exploring', 'fighting', 'alongside', 'trainer']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk.corpus import stopwords\n",
|
|
"\n",
|
|
"stop_words = set(stopwords.words('english'))\n",
|
|
"\n",
|
|
"tokens = [token for token in tokens if token not in stop_words]\n",
|
|
"\n",
|
|
"print(\"Tokens après suppression des stopwords :\")\n",
|
|
"print(tokens)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Étape 5 — Lemmatization\n",
|
|
"\n",
|
|
"On réduit chaque mot à sa forme racine (`flames → flame`, `shooting → shoot`). On utilise le POS tag pour plus de précision.\n",
|
|
"\n",
|
|
"> *Cours page 36-37 — `WordNetLemmatizer` + POS tag*"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 84,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" Tokens après Lemmatization :\n",
|
|
"['furret', 'long', 'slender', 'agile', 'creature', 'soft', 'fur', 'flexible', 'body', 'allow', 'move', 'gracefully', 'narrow', 'tunnel', 'hide', 'pathway', 'normaltype', 'pokmon', 'build', 'intricate', 'nest', 'perfectly', 'shape', 'fit', 'elongated', 'form', 'make', 'nearly', 'impossible', 'creature', 'enter', 'despite', 'gentle', 'calm', 'nature', 'furret', 'become', 'surprisingly', 'energetic', 'battle', 'use', 'powerful', 'tail', 'smash', 'opponent', 'swift', 'playful', 'attack', 'often', 'see', 'wander', 'across', 'field', 'forest', 'curiously', 'observe', 'surroundings', 'share', 'close', 'bond', 'preevolution', 'sentret', 'know', 'endurance', 'cheerful', 'spirit', 'furret', 'quickly', 'recover', 'energy', 'always', 'feel', 'fine', 'ready', 'continue', 'explore', 'fight', 'alongside', 'trainer']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from nltk.stem.wordnet import WordNetLemmatizer\n",
|
|
"from nltk import pos_tag\n",
|
|
"from nltk.corpus import wordnet\n",
|
|
"\n",
|
|
"lem = WordNetLemmatizer()\n",
|
|
"\n",
|
|
"def get_wordnet_pos(treebank_tag):\n",
|
|
" \"\"\"Convertit les tags Penn Treebank en tags WordNet.\"\"\"\n",
|
|
" if treebank_tag.startswith('J'): return wordnet.ADJ\n",
|
|
" elif treebank_tag.startswith('V'): return wordnet.VERB\n",
|
|
" elif treebank_tag.startswith('N'): return wordnet.NOUN\n",
|
|
" elif treebank_tag.startswith('R'): return wordnet.ADV\n",
|
|
" else: return wordnet.NOUN\n",
|
|
"\n",
|
|
"pos_tags = pos_tag(tokens)\n",
|
|
"tokens = [lem.lemmatize(token, get_wordnet_pos(tag)) for token, tag in pos_tags]\n",
|
|
"\n",
|
|
"print(\" Tokens après Lemmatization :\")\n",
|
|
"print(tokens)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Résultat final — Texte nettoyé"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 85,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"📄 Texte brut :\n",
|
|
"Furret is a long, slender, and agile creature with soft fur and a flexible body that allows it to move gracefully through narrow tunnels and hidden pathways. This normal-type Pokémon builds intricate nests perfectly shaped to fit its elongated form, making them nearly impossible for other creatures to enter. Despite its gentle and calm nature, Furret can become surprisingly energetic in battle, using its powerful tail to smash opponents with swift and playful attacks. It is often seen wandering across fields and forests, curiously observing its surroundings, and it shares a close bond with its pre-evolution, Sentret. Known for its endurance and cheerful spirit, Furret can quickly recover its energy, always feeling fine and ready to continue exploring or fighting alongside its trainer.\n",
|
|
"\n",
|
|
"Texte nettoyé :\n",
|
|
"furret long slender agile creature soft fur flexible body allow move gracefully narrow tunnel hide pathway normaltype pokmon build intricate nest perfectly shape fit elongated form make nearly impossible creature enter despite gentle calm nature furret become surprisingly energetic battle use powerful tail smash opponent swift playful attack often see wander across field forest curiously observe surroundings share close bond preevolution sentret know endurance cheerful spirit furret quickly recover energy always feel fine ready continue explore fight alongside trainer\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"clean_text = \" \".join(tokens)\n",
|
|
"\n",
|
|
"print(\"📄 Texte brut :\")\n",
|
|
"print(raw_text.strip())\n",
|
|
"print()\n",
|
|
"print(\"Texte nettoyé :\")\n",
|
|
"print(clean_text)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|