a-simple-guide-to-text-cleaning.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"<div class=\"list-group\" id=\"list-tab\" role=\"tablist\">\n  <h3 class=\"list-group-item list-group-item-action active\" data-toggle=\"list\"  role=\"tab\" aria-controls=\"home\">Notebook Content!</h3>\n  <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"##Library-and-Data\" role=\"tab\" aria-controls=\"profile\">Library and Data<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Word-Cloud\" role=\"tab\" aria-controls=\"messages\">Word Cloud<span class=\"badge badge-primary badge-pill\"></span></a>\n  <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Split-Text\" role=\"tab\" aria-controls=\"messages\">Split Text<span class=\"badge badge-primary badge-pill\"></span></a>\n  <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Lower-Case\" role=\"tab\" aria-controls=\"settings\">Lower Case<span class=\"badge badge-primary badge-pill\"></span></a> \n  <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Upper-Case\" role=\"tab\" aria-controls=\"settings\">Upper-Case<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Unique-Words\" role=\"tab\" aria-controls=\"settings\">Unique Words<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Remove-Punctuations\" role=\"tab\" aria-controls=\"settings\">Remove Punctuations<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Removing-Number\" role=\"tab\" aria-controls=\"settings\"> Removing Number<span class=\"badge badge-primary badge-pill\"></span></a>  \n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Removing-Urls\" role=\"tab\" aria-controls=\"settings\">Removing Urls<span class=\"badge badge-primary badge-pill\"></span></a>  \n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Removing-HTML-tags\" role=\"tab\" aria-controls=\"settings\">Removing HTML Tags<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Remove-Emoji\" role=\"tab\" aria-controls=\"settings\">Remove Emoji<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Remove-Stop-Words\" role=\"tab\" aria-controls=\"settings\">Remove Stop Words<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Stemming\" role=\"tab\" aria-controls=\"settings\">Stemming<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Lemmatization\" role=\"tab\" aria-controls=\"settings\">Lemmatization<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Tokenization\" role=\"tab\" aria-controls=\"settings\">Tokenization<span class=\"badge badge-primary badge-pill\"></span></a>\n      <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Vectorization\" role=\"tab\" aria-controls=\"settings\">Vectorization<span class=\"badge badge-primary badge-pill\"></span></a>\n     <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#Parse-Tree\" role=\"tab\" aria-controls=\"settings\">Parse Tree<span class=\"badge badge-primary badge-pill\"></span></a>\n      "},{"metadata":{},"cell_type":"markdown","source":"# Library and Data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn import feature_extraction, linear_model, model_selection, preprocessing\nimport nltk\nimport nltk as nlp\nimport string\nimport re\nimport spacy\nfrom spacy import displacy\nfrom spacy.util import minibatch, compounding\ndata = pd.read_csv('../input/nlp-getting-started/train.csv', encoding='utf-8')\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/nlp-getting-started/train.csv', encoding='utf-8')\ntrue = pd.read_csv(\"../input/fake-and-real-news-dataset/True.csv\")\nnews = pd.read_csv(\"../input/cbc-news-coronavirus-articles-march-26/news.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Word Cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, ImageColorGenerator\ntext = \" \".join(str(each) for each in news.authors.unique())\nwordcloud = WordCloud(max_words=200,colormap='Set3', background_color=\"black\").generate(text)\nplt.figure(figsize=(15,10))\nplt.imshow(wordcloud, interpolation='Bilinear')\nplt.axis(\"off\")\nplt.figure(1,figsize=(12, 12))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Split Text "},{"metadata":{"trusted":true},"cell_type":"code","source":"def split(text):\n    for word in text:\n        text = text.split()\n        return text\nsplit(\"I am going\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"data['split_text'] = data['text'].apply(lambda x: split(x))\ndata.head(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lower Case"},{"metadata":{"trusted":true},"cell_type":"code","source":"text= \"universe going to expanded infinite\"\nlowercase_text = [word.lower() for word in text.split()]\nprint(lowercase_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Upper Case"},{"metadata":{"trusted":true},"cell_type":"code","source":"uppercase_text = [word.upper() for word in text.split()]\nprint(uppercase_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Unique Words"},{"metadata":{"trusted":true},"cell_type":"code","source":"def unique(text):\n    for word in text:\n        text = text.split()\n        text = set(text)\n        return text\nunique(\"i am going and i will go by bus\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"true['title_unq'] = true['title'].apply(lambda x: unique(x))\ntrue.head(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Remove Punctuations"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_punct(text):\n    text  = \"\".join([char for char in text if char not in string.punctuation])\n    text = re.sub('[0-9]+', '', text)\n    return text\nremove_punct(\"wow!!!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"news['nopunc'] = news['title'].apply(lambda x: remove_punct(x))\nnews.head(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Remove Numbers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_numbers(text):\n    text = ''.join([i for i in text if not i.isdigit()])         \n    return text\nremove_numbers(\"I am 20 years\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing Urls\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_URL(text):\n    url = re.compile(r'https?://\\S+|www\\.\\S+')\n    return url.sub(r'',text)\n\nremove_URL(\"Kaggle id https://www.kaggle.com/vanshjatana\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing HTML tags\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_html(text):\n    html=re.compile(r'<.*?>')\n    return html.sub(r'',text)\nremove_html(\"<h1>Vansh Jatana</h1>\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Remove Emoji"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_emoji(text):\n    emoji_pattern = re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n    return emoji_pattern.sub(r'', text)\n\nremove_emoji(\"Sad😔\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Remove Stop Words"},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(nltk.corpus.stopwords.words())\ndef clean_stopwords(text):\n    res = []\n    for word in text:\n        if word not in stopwords:\n            res.append(word)\n    return res\nclean_stopwords(\"I am coming\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stemming"},{"metadata":{"trusted":true},"cell_type":"code","source":"text = text.lower().split(\" \")\n\nporter_stemmer = nlp.PorterStemmer()\nroots = [porter_stemmer.stem(each) for each in text]\nprint(roots)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lemmatization"},{"metadata":{"trusted":true},"cell_type":"code","source":"lemma = nlp.WordNetLemmatizer()\nlemma_roots = [lemma.lemmatize(each) for each in text]\nprint(lemma_roots)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenization(text):\n    text = re.split('\\W+', text)\n    return text\ntokenization(\"I'm coming, where he's??\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Vectorization"},{"metadata":{"trusted":true},"cell_type":"code","source":"count = feature_extraction.text.CountVectorizer()\ntrain_vectors = count.fit_transform(data[\"text\"][0:2])\nprint(train_vectors[0].todense())\nprint(train_vectors[1].todense())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parse Tree"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"!wget https://raw.githubusercontent.com/tylerneylon/explacy/master/explacy.py\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import explacy\nspacy_tok = spacy.load('en_core_web_sm')\n\nexplacy.print_parse_info(spacy_tok, 'Data is new oil')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}