[{"data":1,"prerenderedAt":597},["ShallowReactive",2],{"content-\u002Fcontents\u002Fpreprocess":3,"surroundPost-\u002Fcontents\u002Fpreprocess":588},{"id":4,"title":5,"body":6,"createdAt":576,"description":113,"draft":577,"extension":578,"meta":579,"navigation":142,"path":580,"seo":581,"stem":582,"tags":583,"thumbnail":586,"updatedAt":576,"__hash__":587},"contents\u002Fcontents\u002Fpreprocess.md","自然言語処理再入門：TfIdfと次元圧縮手法で文章を可視化",{"type":7,"value":8,"toc":569},"minimark",[9,13,16,20,23,59,62,67,70,73,76,79,82,104,107,276,279,282,286,289,293,296,329,332,335,338,341,344,392,396,399,403,406,559,562,565],[10,11,12],"p",{},"世の中 Transformer 系のアルゴリズムが主流になっているところで、あえての TfIdf を復習したいと思います。",[10,14,15],{},"使用するデータセットは livedoor です。",[17,18,19],"h2",{"id":19},"データ探索",[10,21,22],{},"まずは使用するデータがどのようなものか見てみます。データを見るのは大事です。",[24,25,26,39],"table",{},[27,28,29],"thead",{},[30,31,32,36],"tr",{},[33,34,35],"th",{},"項目",[33,37,38],{},"値",[40,41,42,51],"tbody",{},[30,43,44,48],{},[45,46,47],"td",{},"データ数",[45,49,50],{},"5893",[30,52,53,56],{},[45,54,55],{},"ラベル数",[45,57,58],{},"9",[10,60,61],{},"続いて、各文書の長さの分布です。今回は大体 1,000 文字くらいの文書が多いようです。ラベルごとの文書数に若干偏りがありますが、今回は精度をあまり見ないので省略します。",[63,64],"img",{"alt":65,"img-src":66},"sample length","\u002Fimg\u002Ftfidf\u002Fsample_length.png",[17,68,69],{"id":69},"前処理",[10,71,72],{},"BERT 系と異なり、TfIdf では文書の前処理は重要です。",[10,74,75],{},"TfIdf で作成されるベクトルでは文書内の単語の出現回数が重要となってきます。",[10,77,78],{},"そのため、なるべく同じ意味の単語を統一したり、不要な単語を削除しておくことが大事になります。",[10,80,81],{},"前処理で実施する内容は以下の通り。",[83,84,85,89,92,95,98,101],"ul",{},[86,87,88],"li",{},"改行除去",[86,90,91],{},"正規化",[86,93,94],{},"大文字・小文字変換",[86,96,97],{},"URL 除去",[86,99,100],{},"不要記号除去",[86,102,103],{},"数字を 0 に",[10,105,106],{},"こちらがコードです。",[108,109,114],"pre",{"className":110,"code":111,"language":112,"meta":113,"style":113},"language-py shiki shiki-themes github-dark","from typing import List\nimport re\nimport neologdn\n\ndef clean_text(texts:List[str])->List[str]:\n    cleaned_text = []\n    for text in texts:\n        #改行削除\n        text = text.replace(\"\\n\", \"\")\n\n        # URL削除\n        pattern = \"https?:\u002F\u002F[\\w\u002F:%#\\$&\\?\\(\\)~\\.=\\+\\-]+\"\n        text =  re.sub(pattern, ' ', text)\n\n        # 不要記号削除\n        pattern = '[!\"#$%&\\'\\\\\\\\()*+,-.\u002F:;\u003C=>?@[\\\\]^_`{|}~「」〔〕“”◇ᴗ■☆●↓→♪★⊂⊃※△□◎:〈〉『』【】＆＊・（）＄＃＠、？！｀＋￥％]'\n        text =  re.sub(pattern, ' ', text)\n\n        text = neologdn.normalize(text)\n\n        # 大文字・小文字変換\n        text = text.lower()\n\n        # 数字を０に\n        text = re.sub(r'\\d+', '0', text)\n        cleaned_text.append(text)\n    return cleaned_text\n","py","",[115,116,117,125,131,137,144,150,156,162,168,174,179,185,191,197,202,208,214,219,224,230,235,241,247,252,258,264,270],"code",{"__ignoreMap":113},[118,119,122],"span",{"class":120,"line":121},"line",1,[118,123,124],{},"from typing import List\n",[118,126,128],{"class":120,"line":127},2,[118,129,130],{},"import re\n",[118,132,134],{"class":120,"line":133},3,[118,135,136],{},"import neologdn\n",[118,138,140],{"class":120,"line":139},4,[118,141,143],{"emptyLinePlaceholder":142},true,"\n",[118,145,147],{"class":120,"line":146},5,[118,148,149],{},"def clean_text(texts:List[str])->List[str]:\n",[118,151,153],{"class":120,"line":152},6,[118,154,155],{},"    cleaned_text = []\n",[118,157,159],{"class":120,"line":158},7,[118,160,161],{},"    for text in texts:\n",[118,163,165],{"class":120,"line":164},8,[118,166,167],{},"        #改行削除\n",[118,169,171],{"class":120,"line":170},9,[118,172,173],{},"        text = text.replace(\"\\n\", \"\")\n",[118,175,177],{"class":120,"line":176},10,[118,178,143],{"emptyLinePlaceholder":142},[118,180,182],{"class":120,"line":181},11,[118,183,184],{},"        # URL削除\n",[118,186,188],{"class":120,"line":187},12,[118,189,190],{},"        pattern = \"https?:\u002F\u002F[\\w\u002F:%#\\$&\\?\\(\\)~\\.=\\+\\-]+\"\n",[118,192,194],{"class":120,"line":193},13,[118,195,196],{},"        text =  re.sub(pattern, ' ', text)\n",[118,198,200],{"class":120,"line":199},14,[118,201,143],{"emptyLinePlaceholder":142},[118,203,205],{"class":120,"line":204},15,[118,206,207],{},"        # 不要記号削除\n",[118,209,211],{"class":120,"line":210},16,[118,212,213],{},"        pattern = '[!\"#$%&\\'\\\\\\\\()*+,-.\u002F:;\u003C=>?@[\\\\]^_`{|}~「」〔〕“”◇ᴗ■☆●↓→♪★⊂⊃※△□◎:〈〉『』【】＆＊・（）＄＃＠、？！｀＋￥％]'\n",[118,215,217],{"class":120,"line":216},17,[118,218,196],{},[118,220,222],{"class":120,"line":221},18,[118,223,143],{"emptyLinePlaceholder":142},[118,225,227],{"class":120,"line":226},19,[118,228,229],{},"        text = neologdn.normalize(text)\n",[118,231,233],{"class":120,"line":232},20,[118,234,143],{"emptyLinePlaceholder":142},[118,236,238],{"class":120,"line":237},21,[118,239,240],{},"        # 大文字・小文字変換\n",[118,242,244],{"class":120,"line":243},22,[118,245,246],{},"        text = text.lower()\n",[118,248,250],{"class":120,"line":249},23,[118,251,143],{"emptyLinePlaceholder":142},[118,253,255],{"class":120,"line":254},24,[118,256,257],{},"        # 数字を０に\n",[118,259,261],{"class":120,"line":260},25,[118,262,263],{},"        text = re.sub(r'\\d+', '0', text)\n",[118,265,267],{"class":120,"line":266},26,[118,268,269],{},"        cleaned_text.append(text)\n",[118,271,273],{"class":120,"line":272},27,[118,274,275],{},"    return cleaned_text\n",[10,277,278],{},"方針はなるべく単語の多様性を減らし、不要な単語は消すです。ストップワード除去は一旦結果を見てから考えます。",[10,280,281],{},"上記前処理を行い、単語のわかち書きにします。形態素解析は ginza を使用しました。\nわかち書きしたリストを使用し、単語の分布を表示しました。",[63,283],{"alt":284,"img-src":285},"word distribution","\u002Fimg\u002Ftfidf\u002Fword_distribution.png",[10,287,288],{},"日本語で一般的な単語が多いことがわかります。また、「する」「いる」など頻出する単語があることもわかります。",[17,290,292],{"id":291},"tfidf-で文書をベクトル化","TfIdf で文書をベクトル化",[10,294,295],{},"ようやく TfIdf です。",[108,297,299],{"className":110,"code":298,"language":112,"meta":113,"style":113},"from sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(max_df=0.95, min_df=10)\nX = vectorizer.fit_transform(tokenised_texts)\n\n# X.shape >> (5893, 12267)\n",[115,300,301,306,310,315,320,324],{"__ignoreMap":113},[118,302,303],{"class":120,"line":121},[118,304,305],{},"from sklearn.feature_extraction.text import TfidfVectorizer\n",[118,307,308],{"class":120,"line":127},[118,309,143],{"emptyLinePlaceholder":142},[118,311,312],{"class":120,"line":133},[118,313,314],{},"vectorizer = TfidfVectorizer(max_df=0.95, min_df=10)\n",[118,316,317],{"class":120,"line":139},[118,318,319],{},"X = vectorizer.fit_transform(tokenised_texts)\n",[118,321,322],{"class":120,"line":146},[118,323,143],{"emptyLinePlaceholder":142},[118,325,326],{"class":120,"line":152},[118,327,328],{},"# X.shape >> (5893, 12267)\n",[10,330,331],{},"max_df や min_df は何度か試して vocabulary*を確認し、決めました。",[17,333,334],{"id":334},"次元圧縮手法で可視化",[10,336,337],{},"次元圧縮手法はいくつかありますが、今回は t-sne と umap を試しました。",[10,339,340],{},"最終的には umap が一番よく分離できていました。また、計算時間的にも umap が使いやすいです。",[10,342,343],{},"以下が umap のコードと結果です。",[108,345,347],{"className":110,"code":346,"language":112,"meta":113,"style":113},"from sklearn.manifold import TSNE\nimport umap\n\n#t-sneの場合\n#x_embedded =  TSNE(n_components=2, learning_rate='auto',init='pca', perplexity=3)\n\n# umap\nmapper = umap.UMAP(random_state=0)\numap_vecs = mapper.fit_transform(X)\n",[115,348,349,354,359,363,368,373,377,382,387],{"__ignoreMap":113},[118,350,351],{"class":120,"line":121},[118,352,353],{},"from sklearn.manifold import TSNE\n",[118,355,356],{"class":120,"line":127},[118,357,358],{},"import umap\n",[118,360,361],{"class":120,"line":133},[118,362,143],{"emptyLinePlaceholder":142},[118,364,365],{"class":120,"line":139},[118,366,367],{},"#t-sneの場合\n",[118,369,370],{"class":120,"line":146},[118,371,372],{},"#x_embedded =  TSNE(n_components=2, learning_rate='auto',init='pca', perplexity=3)\n",[118,374,375],{"class":120,"line":152},[118,376,143],{"emptyLinePlaceholder":142},[118,378,379],{"class":120,"line":158},[118,380,381],{},"# umap\n",[118,383,384],{"class":120,"line":164},[118,385,386],{},"mapper = umap.UMAP(random_state=0)\n",[118,388,389],{"class":120,"line":170},[118,390,391],{},"umap_vecs = mapper.fit_transform(X)\n",[63,393],{"alt":394,"img-src":395},"umap","\u002Fimg\u002Ftfidf\u002Fumap.png",[10,397,398],{},"こちらが t-sne です。perplexity の値が難しいです。",[63,400],{"alt":401,"img-src":402},"tsne","\u002Fimg\u002Ftfidf\u002Ftsne.png",[10,404,405],{},"以下参考までに可視化用のコードです。",[108,407,409],{"className":110,"code":408,"language":112,"meta":113,"style":113},"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\n\n# category_listはlivedoorのラベルです。\n\nfig = plt.figure(figsize=(20,20))\nlabels = np.array(list(df.label))\nfor label in range(len(category_list)):\n    plt.subplot(3,3,label+1)\n    plt.title(f\"{category_list[label]}\")\n    index = labels==label\n    plt.plot(\n        tsne_vecs[:, 0],\n        tsne_vecs[:, 1],\n        # umap_vecs[:, 0],\n        # umap_vecs[:, 1],\n        'o',\n        markersize=1,\n        color=[0.7,0.7,0.7]\n    )\n    plt.plot(\n        tsne_vecs[index, 0],\n        tsne_vecs[index, 1],\n        # umap_vecs[index, 0],\n        # umap_vecs[index, 1],\n        'o',\n        markersize=2,\n        color ='b'\n    )\n",[115,410,411,416,421,426,430,435,439,444,449,454,459,464,469,474,479,484,489,494,499,504,509,514,518,523,528,533,538,542,548,554],{"__ignoreMap":113},[118,412,413],{"class":120,"line":121},[118,414,415],{},"import matplotlib.pyplot as plt\n",[118,417,418],{"class":120,"line":127},[118,419,420],{},"import seaborn as sns\n",[118,422,423],{"class":120,"line":133},[118,424,425],{},"import numpy as np\n",[118,427,428],{"class":120,"line":139},[118,429,143],{"emptyLinePlaceholder":142},[118,431,432],{"class":120,"line":146},[118,433,434],{},"# category_listはlivedoorのラベルです。\n",[118,436,437],{"class":120,"line":152},[118,438,143],{"emptyLinePlaceholder":142},[118,440,441],{"class":120,"line":158},[118,442,443],{},"fig = plt.figure(figsize=(20,20))\n",[118,445,446],{"class":120,"line":164},[118,447,448],{},"labels = np.array(list(df.label))\n",[118,450,451],{"class":120,"line":170},[118,452,453],{},"for label in range(len(category_list)):\n",[118,455,456],{"class":120,"line":176},[118,457,458],{},"    plt.subplot(3,3,label+1)\n",[118,460,461],{"class":120,"line":181},[118,462,463],{},"    plt.title(f\"{category_list[label]}\")\n",[118,465,466],{"class":120,"line":187},[118,467,468],{},"    index = labels==label\n",[118,470,471],{"class":120,"line":193},[118,472,473],{},"    plt.plot(\n",[118,475,476],{"class":120,"line":199},[118,477,478],{},"        tsne_vecs[:, 0],\n",[118,480,481],{"class":120,"line":204},[118,482,483],{},"        tsne_vecs[:, 1],\n",[118,485,486],{"class":120,"line":210},[118,487,488],{},"        # umap_vecs[:, 0],\n",[118,490,491],{"class":120,"line":216},[118,492,493],{},"        # umap_vecs[:, 1],\n",[118,495,496],{"class":120,"line":221},[118,497,498],{},"        'o',\n",[118,500,501],{"class":120,"line":226},[118,502,503],{},"        markersize=1,\n",[118,505,506],{"class":120,"line":232},[118,507,508],{},"        color=[0.7,0.7,0.7]\n",[118,510,511],{"class":120,"line":237},[118,512,513],{},"    )\n",[118,515,516],{"class":120,"line":243},[118,517,473],{},[118,519,520],{"class":120,"line":249},[118,521,522],{},"        tsne_vecs[index, 0],\n",[118,524,525],{"class":120,"line":254},[118,526,527],{},"        tsne_vecs[index, 1],\n",[118,529,530],{"class":120,"line":260},[118,531,532],{},"        # umap_vecs[index, 0],\n",[118,534,535],{"class":120,"line":266},[118,536,537],{},"        # umap_vecs[index, 1],\n",[118,539,540],{"class":120,"line":272},[118,541,498],{},[118,543,545],{"class":120,"line":544},28,[118,546,547],{},"        markersize=2,\n",[118,549,551],{"class":120,"line":550},29,[118,552,553],{},"        color ='b'\n",[118,555,557],{"class":120,"line":556},30,[118,558,513],{},[17,560,561],{"id":561},"まとめ",[10,563,564],{},"livedoor データは文書分類で頻出のデータセットですが、結果を見るにいい感じに分類できそうな感じがします。",[566,567,568],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":113,"searchDepth":127,"depth":127,"links":570},[571,572,573,574,575],{"id":19,"depth":127,"text":19},{"id":69,"depth":127,"text":69},{"id":291,"depth":127,"text":292},{"id":334,"depth":127,"text":334},{"id":561,"depth":127,"text":561},"2022-09-19",false,"md",{},"\u002Fcontents\u002Fpreprocess",{"title":5,"description":113},"contents\u002Fpreprocess",[584,585],"2022","機械学習","\u002Fimg\u002Ftwitter-card.png","ZN0nivyPuk77bcPql9GeZAJqkTJpvoMM6lC3zJz_LEg",[589,593],{"title":590,"path":591,"stem":592,"children":-1},"Poetryへの挑戦。使い方や利点等を調べる","\u002Fcontents\u002Fpoetry","contents\u002Fpoetry",{"title":594,"path":595,"stem":596,"children":-1},"前処理を行いツイートをWordCloudで可視化する方法","\u002Fcontents\u002Fpreprocessing","contents\u002Fpreprocessing",1784936718639]