Code-Example-032-Text-LSI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c255ac26",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import re\n",
    "import bz2\n",
    "\n",
    "regex = re.compile(\"[^a-zA-Z]\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from nltk.tokenize import RegexpTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c267c31d",
   "metadata": {},
   "source": [
    "# 20 Newsgroups Dataset \n",
    "\n",
    "You should download and look at the 20-news-same-line.txt.bz2\n",
    "http://qwone.com/~jason/20Newsgroups/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c567b14d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Line number:  1000\n",
      "Line number:  2000\n",
      "Line number:  3000\n",
      "Line number:  4000\n",
      "Line number:  5000\n",
      "Line number:  6000\n",
      "Line number:  7000\n",
      "Line number:  8000\n",
      "Line number:  9000\n",
      "Line number:  10000\n",
      "Line number:  11000\n",
      "Line number:  12000\n",
      "Line number:  13000\n",
      "Line number:  14000\n",
      "Line number:  15000\n",
      "Line number:  16000\n",
      "Line number:  17000\n",
      "Line number:  18000\n",
      "Line number:  19000\n",
      "Line number:  19997\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[from, lipman, oasys, navy, robert, lipman, su...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[from, weston, ucssun, sdsu, weston, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[from, coconut, ryan, porter, subject, dmorph,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[from, onyx, virginia, kenneth, hinckley, subj...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[from, joth, ersys, edmonton, tham, subject, w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19992</th>\n",
       "      <td>[from, gmills, chemical, watstar, uwaterloo, p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19993</th>\n",
       "      <td>[from, imager, dave, knapp, subject, branch, a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>[from, pharvey, quack, paul, harvey, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>[date, tuesday, from, subject, info, about, li...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>[from, pharvey, quack, paul, harvey, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19997 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "0      [from, lipman, oasys, navy, robert, lipman, su...      1\n",
       "1      [from, weston, ucssun, sdsu, weston, subject, ...      1\n",
       "2      [from, coconut, ryan, porter, subject, dmorph,...      1\n",
       "3      [from, onyx, virginia, kenneth, hinckley, subj...      1\n",
       "4      [from, joth, ersys, edmonton, tham, subject, w...      1\n",
       "...                                                  ...    ...\n",
       "19992  [from, gmills, chemical, watstar, uwaterloo, p...      1\n",
       "19993  [from, imager, dave, knapp, subject, branch, a...      1\n",
       "19994  [from, pharvey, quack, paul, harvey, subject, ...      1\n",
       "19995  [date, tuesday, from, subject, info, about, li...      1\n",
       "19996  [from, pharvey, quack, paul, harvey, subject, ...      1\n",
       "\n",
       "[19997 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download the file from here \n",
    "# https://github.com/kiat/Elements-of-Data-Analytics/blob/main/datasets/Court-Wiki-Dataset.txt.bz2\n",
    "file = bz2.open(\"./datasets/20-news-same-line.txt.bz2\", \"r\")\n",
    "\n",
    "mlist = []\n",
    "\n",
    "count = 0 \n",
    "for line in file:\n",
    "    count +=1\n",
    "    line = (lambda x : (x[x.index('id=\"') + 4 : x.index('\" url=')], x[x.index('\">') + 2:][:-6]))(str(line))\n",
    "    text = np.array(regex.sub(\" \", line[1]).lower().split())\n",
    "    \n",
    "    text = [x for x in text if len(x) > 3] # drop all word with less than 3 \n",
    "\n",
    "    mlist.append({'text' : text, 'label' : (lambda x: 0 if 'AU' in x else 1 )(line[0])})\n",
    "\n",
    "\n",
    "    if(count%1000==0):\n",
    "        print(\"Line number: \" , count)\n",
    "    # 20 news group dataset has 19997 documents \n",
    "    # We read only 5000 documents of them.\n",
    "\n",
    "\n",
    "# text_list\n",
    "print(\"Line number: \" , count)\n",
    "\n",
    "data = pd.DataFrame.from_dict(mlist)\n",
    "data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c2227790",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[from, lipman, oasys, navy, robert, lipman, su...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[from, weston, ucssun, sdsu, weston, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[from, coconut, ryan, porter, subject, dmorph,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[from, onyx, virginia, kenneth, hinckley, subj...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[from, joth, ersys, edmonton, tham, subject, w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19992</th>\n",
       "      <td>[from, gmills, chemical, watstar, uwaterloo, p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19993</th>\n",
       "      <td>[from, imager, dave, knapp, subject, branch, a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>[from, pharvey, quack, paul, harvey, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>[date, tuesday, from, subject, info, about, li...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>[from, pharvey, quack, paul, harvey, subject, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19997 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text label\n",
       "0      [from, lipman, oasys, navy, robert, lipman, su...     1\n",
       "1      [from, weston, ucssun, sdsu, weston, subject, ...     1\n",
       "2      [from, coconut, ryan, porter, subject, dmorph,...     1\n",
       "3      [from, onyx, virginia, kenneth, hinckley, subj...     1\n",
       "4      [from, joth, ersys, edmonton, tham, subject, w...     1\n",
       "...                                                  ...   ...\n",
       "19992  [from, gmills, chemical, watstar, uwaterloo, p...     1\n",
       "19993  [from, imager, dave, knapp, subject, branch, a...     1\n",
       "19994  [from, pharvey, quack, paul, harvey, subject, ...     1\n",
       "19995  [date, tuesday, from, subject, info, about, li...     1\n",
       "19996  [from, pharvey, quack, paul, harvey, subject, ...     1\n",
       "\n",
       "[19997 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'] = data['label'].apply(lambda x: (str(x).replace( \"20_newsgroups/\" , \"\")).split(\"/\", 1)[0] )\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "38647955",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('that', 70751), ('from', 39653), ('this', 34682), ('have', 31994), ('with', 30218), ('they', 23182), ('subject', 21589), ('lines', 20891), ('date', 20771), ('what', 17693)]\n"
     ]
    }
   ],
   "source": [
    "# Create a Dictionary with all of the words in all documents \n",
    "# Count them up \n",
    "# Get the top 600 most common words. \n",
    "\n",
    "from collections import Counter\n",
    "my_dict=Counter()\n",
    "\n",
    "# iterate through list-string\n",
    "for item in data['text']:\n",
    "    my_dict += Counter(list(item))\n",
    "\n",
    "# Print the top 10 just for checking \n",
    "print(my_dict.most_common(10))\n",
    "\n",
    "no_features = 600 \n",
    "# We use the top no_features as our dictionary\n",
    "top_words= my_dict.most_common(no_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "049d3afa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('that',\n",
       " 'from',\n",
       " 'this',\n",
       " 'have',\n",
       " 'with',\n",
       " 'they',\n",
       " 'subject',\n",
       " 'lines',\n",
       " 'date',\n",
       " 'what',\n",
       " 'there',\n",
       " 'would',\n",
       " 'will',\n",
       " 'writes',\n",
       " 'about',\n",
       " 'your',\n",
       " 'article',\n",
       " 'some',\n",
       " 'which',\n",
       " 'like',\n",
       " 'people',\n",
       " 'more',\n",
       " 'when',\n",
       " 'just',\n",
       " 'were',\n",
       " 'their',\n",
       " 'know',\n",
       " 'other',\n",
       " 'only',\n",
       " 'them',\n",
       " 'than',\n",
       " 'been',\n",
       " 'think',\n",
       " 'also',\n",
       " 'does',\n",
       " 'time',\n",
       " 'then',\n",
       " 'these',\n",
       " 'should',\n",
       " 'good',\n",
       " 'could',\n",
       " 'well',\n",
       " 'because',\n",
       " 'even',\n",
       " 'very',\n",
       " 'into',\n",
       " 'those',\n",
       " 'make',\n",
       " 'many',\n",
       " 'much',\n",
       " 'first',\n",
       " 'right',\n",
       " 'most',\n",
       " 'such',\n",
       " 'world',\n",
       " 'distribution',\n",
       " 'here',\n",
       " 'system',\n",
       " 'where',\n",
       " 'after',\n",
       " 'want',\n",
       " 'anyone',\n",
       " 'said',\n",
       " 'being',\n",
       " 'over',\n",
       " 'used',\n",
       " 'same',\n",
       " 'need',\n",
       " 'work',\n",
       " 'really',\n",
       " 'something',\n",
       " 'please',\n",
       " 'problem',\n",
       " 'believe',\n",
       " 'since',\n",
       " 'still',\n",
       " 'back',\n",
       " 'windows',\n",
       " 'mail',\n",
       " 'years',\n",
       " 'going',\n",
       " 'before',\n",
       " 'find',\n",
       " 'point',\n",
       " 'government',\n",
       " 'help',\n",
       " 'take',\n",
       " 'information',\n",
       " 'file',\n",
       " 'might',\n",
       " 'year',\n",
       " 'better',\n",
       " 'using',\n",
       " 'question',\n",
       " 'never',\n",
       " 'things',\n",
       " 'both',\n",
       " 'last',\n",
       " 'read',\n",
       " 'thanks',\n",
       " 'while',\n",
       " 'must',\n",
       " 'without',\n",
       " 'sure',\n",
       " 'made',\n",
       " 'david',\n",
       " 'another',\n",
       " 'number',\n",
       " 'someone',\n",
       " 'doesn',\n",
       " 'through',\n",
       " 'state',\n",
       " 'between',\n",
       " 'software',\n",
       " 'down',\n",
       " 'thing',\n",
       " 'university',\n",
       " 'space',\n",
       " 'drive',\n",
       " 'part',\n",
       " 'long',\n",
       " 'program',\n",
       " 'data',\n",
       " 'look',\n",
       " 'under',\n",
       " 'fact',\n",
       " 'version',\n",
       " 'available',\n",
       " 'come',\n",
       " 'anything',\n",
       " 'power',\n",
       " 'each',\n",
       " 'case',\n",
       " 'little',\n",
       " 'around',\n",
       " 'however',\n",
       " 'didn',\n",
       " 'against',\n",
       " 'john',\n",
       " 'again',\n",
       " 'give',\n",
       " 'every',\n",
       " 'best',\n",
       " 'different',\n",
       " 'true',\n",
       " 'seems',\n",
       " 'probably',\n",
       " 'least',\n",
       " 'game',\n",
       " 'enough',\n",
       " 'actually',\n",
       " 'computer',\n",
       " 'tell',\n",
       " 'course',\n",
       " 'real',\n",
       " 'news',\n",
       " 'great',\n",
       " 'life',\n",
       " 'line',\n",
       " 'list',\n",
       " 'jesus',\n",
       " 'support',\n",
       " 'high',\n",
       " 'name',\n",
       " 'says',\n",
       " 'either',\n",
       " 'though',\n",
       " 'free',\n",
       " 'next',\n",
       " 'nothing',\n",
       " 'mean',\n",
       " 'hard',\n",
       " 'group',\n",
       " 'post',\n",
       " 'called',\n",
       " 'second',\n",
       " 'possible',\n",
       " 'else',\n",
       " 'call',\n",
       " 'rather',\n",
       " 'card',\n",
       " 'wrong',\n",
       " 'public',\n",
       " 'reason',\n",
       " 'message',\n",
       " 'above',\n",
       " 'bill',\n",
       " 'others',\n",
       " 'christian',\n",
       " 'image',\n",
       " 'person',\n",
       " 'order',\n",
       " 'based',\n",
       " 'keep',\n",
       " 'problems',\n",
       " 'found',\n",
       " 'looking',\n",
       " 'maybe',\n",
       " 'done',\n",
       " 'mark',\n",
       " 'files',\n",
       " 'seen',\n",
       " 'email',\n",
       " 'example',\n",
       " 'able',\n",
       " 'having',\n",
       " 'team',\n",
       " 'place',\n",
       " 'ever',\n",
       " 'always',\n",
       " 'three',\n",
       " 'quite',\n",
       " 'following',\n",
       " 'send',\n",
       " 'general',\n",
       " 'israel',\n",
       " 'heard',\n",
       " 'info',\n",
       " 'wrote',\n",
       " 'control',\n",
       " 'once',\n",
       " 'thought',\n",
       " 'doing',\n",
       " 'evidence',\n",
       " 'means',\n",
       " 'less',\n",
       " 'chip',\n",
       " 'trying',\n",
       " 'jews',\n",
       " 'whether',\n",
       " 'book',\n",
       " 'graphics',\n",
       " 'human',\n",
       " 'internet',\n",
       " 'left',\n",
       " 'children',\n",
       " 'window',\n",
       " 'science',\n",
       " 'start',\n",
       " 'away',\n",
       " 'given',\n",
       " 'phone',\n",
       " 'today',\n",
       " 'idea',\n",
       " 'several',\n",
       " 'times',\n",
       " 'opinions',\n",
       " 'systems',\n",
       " 'michael',\n",
       " 'word',\n",
       " 'getting',\n",
       " 'kind',\n",
       " 'home',\n",
       " 'research',\n",
       " 'seem',\n",
       " 'access',\n",
       " 'note',\n",
       " 'during',\n",
       " 'nasa',\n",
       " 'questions',\n",
       " 'remember',\n",
       " 'change',\n",
       " 'president',\n",
       " 'makes',\n",
       " 'american',\n",
       " 'paul',\n",
       " 'standard',\n",
       " 'saying',\n",
       " 'code',\n",
       " 'source',\n",
       " 'price',\n",
       " 'local',\n",
       " 'full',\n",
       " 'perhaps',\n",
       " 'scsi',\n",
       " 'apple',\n",
       " 'rights',\n",
       " 'steve',\n",
       " 'uucp',\n",
       " 'cannot',\n",
       " 'already',\n",
       " 'until',\n",
       " 'money',\n",
       " 'large',\n",
       " 'national',\n",
       " 'small',\n",
       " 'mike',\n",
       " 'religion',\n",
       " 'disk',\n",
       " 'games',\n",
       " 'stuff',\n",
       " 'answer',\n",
       " 'running',\n",
       " 'address',\n",
       " 'matter',\n",
       " 'issue',\n",
       " 'speed',\n",
       " 'interested',\n",
       " 'days',\n",
       " 'frank',\n",
       " 'original',\n",
       " 'bible',\n",
       " 'agree',\n",
       " 'states',\n",
       " 'show',\n",
       " 'came',\n",
       " 'play',\n",
       " 'pretty',\n",
       " 'told',\n",
       " 'works',\n",
       " 'type',\n",
       " 'whole',\n",
       " 'current',\n",
       " 'fire',\n",
       " 'sale',\n",
       " 'open',\n",
       " 'care',\n",
       " 'hope',\n",
       " 'color',\n",
       " 'claim',\n",
       " 'machine',\n",
       " 'live',\n",
       " 'feel',\n",
       " 'reply',\n",
       " 'everyone',\n",
       " 'armenian',\n",
       " 'center',\n",
       " 'love',\n",
       " 'netcom',\n",
       " 'everything',\n",
       " 'understand',\n",
       " 'often',\n",
       " 'history',\n",
       " 'important',\n",
       " 'simply',\n",
       " 'church',\n",
       " 'turkish',\n",
       " 'hand',\n",
       " 'earth',\n",
       " 'almost',\n",
       " 'memory',\n",
       " 'mind',\n",
       " 'went',\n",
       " 'robert',\n",
       " 'truth',\n",
       " 'wanted',\n",
       " 'including',\n",
       " 'april',\n",
       " 'uiuc',\n",
       " 'certainly',\n",
       " 'server',\n",
       " 'user',\n",
       " 'started',\n",
       " 'side',\n",
       " 'include',\n",
       " 'wouldn',\n",
       " 'display',\n",
       " 'programs',\n",
       " 'guess',\n",
       " 'clipper',\n",
       " 'cause',\n",
       " 'jewish',\n",
       " 'video',\n",
       " 'making',\n",
       " 'value',\n",
       " 'posting',\n",
       " 'house',\n",
       " 'encryption',\n",
       " 'instead',\n",
       " 'area',\n",
       " 'comes',\n",
       " 'christians',\n",
       " 'themselves',\n",
       " 'christ',\n",
       " 'black',\n",
       " 'white',\n",
       " 'later',\n",
       " 'working',\n",
       " 'although',\n",
       " 'technology',\n",
       " 'death',\n",
       " 'newsreader',\n",
       " 'cost',\n",
       " 'country',\n",
       " 'health',\n",
       " 'anyway',\n",
       " 'single',\n",
       " 'clinton',\n",
       " 'copy',\n",
       " 'write',\n",
       " 'hell',\n",
       " 'talk',\n",
       " 'consider',\n",
       " 'check',\n",
       " 'words',\n",
       " 'unless',\n",
       " 'within',\n",
       " 'light',\n",
       " 'opinion',\n",
       " 'difference',\n",
       " 'private',\n",
       " 'faith',\n",
       " 'armenians',\n",
       " 'known',\n",
       " 'tried',\n",
       " 'police',\n",
       " 'sort',\n",
       " 'sense',\n",
       " 'clear',\n",
       " 'anybody',\n",
       " 'major',\n",
       " 'deal',\n",
       " 'text',\n",
       " 'political',\n",
       " 'view',\n",
       " 'contact',\n",
       " 'unix',\n",
       " 'security',\n",
       " 'objective',\n",
       " 'religious',\n",
       " 'morality',\n",
       " 'likely',\n",
       " 'especially',\n",
       " 'similar',\n",
       " 'talking',\n",
       " 'press',\n",
       " 'washington',\n",
       " 'james',\n",
       " 'argument',\n",
       " 'package',\n",
       " 'nice',\n",
       " 'night',\n",
       " 'correct',\n",
       " 'body',\n",
       " 'service',\n",
       " 'koresh',\n",
       " 'summary',\n",
       " 'couple',\n",
       " 'simple',\n",
       " 'certain',\n",
       " 'happened',\n",
       " 'hockey',\n",
       " 'stop',\n",
       " 'women',\n",
       " 'dead',\n",
       " 'hardware',\n",
       " 'exactly',\n",
       " 'self',\n",
       " 'force',\n",
       " 'guns',\n",
       " 'brian',\n",
       " 'sound',\n",
       " 'level',\n",
       " 'size',\n",
       " 'screen',\n",
       " 'took',\n",
       " 'third',\n",
       " 'thus',\n",
       " 'killed',\n",
       " 'dave',\n",
       " 'moral',\n",
       " 'israeli',\n",
       " 'provide',\n",
       " 'company',\n",
       " 'board',\n",
       " 'asked',\n",
       " 'fast',\n",
       " 'driver',\n",
       " 'among',\n",
       " 'fine',\n",
       " 'peter',\n",
       " 'sorry',\n",
       " 'situation',\n",
       " 'common',\n",
       " 'andrew',\n",
       " 'books',\n",
       " 'form',\n",
       " 'taken',\n",
       " 'whatever',\n",
       " 'position',\n",
       " 'format',\n",
       " 'monitor',\n",
       " 'myself',\n",
       " 'theory',\n",
       " 'statement',\n",
       " 'reading',\n",
       " 'haven',\n",
       " 'become',\n",
       " 'goes',\n",
       " 'images',\n",
       " 'written',\n",
       " 'usually',\n",
       " 'players',\n",
       " 'head',\n",
       " 'experience',\n",
       " 'peace',\n",
       " 'itself',\n",
       " 'application',\n",
       " 'except',\n",
       " 'laws',\n",
       " 'test',\n",
       " 'upon',\n",
       " 'city',\n",
       " 'discussion',\n",
       " 'bike',\n",
       " 'business',\n",
       " 'mine',\n",
       " 'interesting',\n",
       " 'society',\n",
       " 'four',\n",
       " 'personal',\n",
       " 'output',\n",
       " 'groups',\n",
       " 'past',\n",
       " 'hear',\n",
       " 'rest',\n",
       " 'particular',\n",
       " 'period',\n",
       " 'easy',\n",
       " 'advance',\n",
       " 'school',\n",
       " 'therefore',\n",
       " 'according',\n",
       " 'turn',\n",
       " 'model',\n",
       " 'study',\n",
       " 'happen',\n",
       " 'needs',\n",
       " 'keys',\n",
       " 'short',\n",
       " 'effect',\n",
       " 'needed',\n",
       " 'radio',\n",
       " 'numbers',\n",
       " 'special',\n",
       " 'network',\n",
       " 'members',\n",
       " 'series',\n",
       " 'early',\n",
       " 'future',\n",
       " 'jpeg',\n",
       " 'lost',\n",
       " 'canada',\n",
       " 'front',\n",
       " 'mode',\n",
       " 'anti',\n",
       " 'uses',\n",
       " 'exist',\n",
       " 'week',\n",
       " 'months',\n",
       " 'kill',\n",
       " 'ones',\n",
       " 'toronto',\n",
       " 'mouse',\n",
       " 'users',\n",
       " 'error',\n",
       " 'points',\n",
       " 'accept',\n",
       " 'department',\n",
       " 'disclaimer',\n",
       " 'drivers',\n",
       " 'longer',\n",
       " 'expect',\n",
       " 'scott',\n",
       " 'plus',\n",
       " 'posted',\n",
       " 'season',\n",
       " 'specific',\n",
       " 'face',\n",
       " 'considered',\n",
       " 'wasn',\n",
       " 'recently',\n",
       " 'gets',\n",
       " 'sell',\n",
       " 'move',\n",
       " 'runs',\n",
       " 'comp',\n",
       " 'coming',\n",
       " 'sometimes',\n",
       " 'aren',\n",
       " 'young',\n",
       " 'values',\n",
       " 'taking',\n",
       " 'baseball',\n",
       " 'medical',\n",
       " 'smith',\n",
       " 'united',\n",
       " 'insurance',\n",
       " 'return',\n",
       " 'shall',\n",
       " 'worth',\n",
       " 'manager',\n",
       " 'build',\n",
       " 'thomas',\n",
       " 'reference',\n",
       " 'offer',\n",
       " 'outside',\n",
       " 'process',\n",
       " 'action',\n",
       " 'million')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sorted List of words based on their frequencies\n",
    "my_dict_words = sorted(top_words, key=lambda k: top_words[1])\n",
    "\n",
    "# This is the order of words based on their frequencies\n",
    "# We use this order to create feature vectors for our text corpus. \n",
    "dict_words= list(zip(*my_dict_words))[0]\n",
    "\n",
    "dict_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0ea7120f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'from': 1, 'lipman': 7, 'oasys': 3, 'navy': 1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'from': 1, 'weston': 4, 'ucssun': 2, 'sdsu': ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'from': 1, 'coconut': 1, 'ryan': 2, 'porter':...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'from': 2, 'onyx': 1, 'virginia': 3, 'kenneth...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'from': 1, 'joth': 2, 'ersys': 2, 'edmonton':...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19992</th>\n",
       "      <td>{'from': 1, 'gmills': 1, 'chemical': 1, 'watst...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19993</th>\n",
       "      <td>{'from': 3, 'imager': 2, 'dave': 2, 'knapp': 2...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>{'from': 1, 'pharvey': 1, 'quack': 1, 'paul': ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>{'date': 1, 'tuesday': 1, 'from': 1, 'subject'...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>{'from': 1, 'pharvey': 1, 'quack': 1, 'paul': ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19997 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text label\n",
       "0      {'from': 1, 'lipman': 7, 'oasys': 3, 'navy': 1...     1\n",
       "1      {'from': 1, 'weston': 4, 'ucssun': 2, 'sdsu': ...     1\n",
       "2      {'from': 1, 'coconut': 1, 'ryan': 2, 'porter':...     1\n",
       "3      {'from': 2, 'onyx': 1, 'virginia': 3, 'kenneth...     1\n",
       "4      {'from': 1, 'joth': 2, 'ersys': 2, 'edmonton':...     1\n",
       "...                                                  ...   ...\n",
       "19992  {'from': 1, 'gmills': 1, 'chemical': 1, 'watst...     1\n",
       "19993  {'from': 3, 'imager': 2, 'dave': 2, 'knapp': 2...     1\n",
       "19994  {'from': 1, 'pharvey': 1, 'quack': 1, 'paul': ...     1\n",
       "19995  {'date': 1, 'tuesday': 1, 'from': 1, 'subject'...     1\n",
       "19996  {'from': 1, 'pharvey': 1, 'quack': 1, 'paul': ...     1\n",
       "\n",
       "[19997 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=data['text'].apply(lambda x:Counter(list(x)) )\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a446b4f6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics = list(set(data['label']))\n",
    "topics "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "50c71a46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8e8d7ca6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 4., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 3., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "       2., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.,\n",
       "       0., 0., 2., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.,\n",
       "       0., 0., 0., 2., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
       "       0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 1., 0., 0., 0., 1., 0., 2., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 2., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 2., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 1., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 3., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 4., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "       1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 2., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "       0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def convert_words_to_numbers(input_dict_text):\n",
    "    # get a numpy of no_features zeros \n",
    "    a = np.zeros(no_features)\n",
    "    for i in range(no_features):\n",
    "        if (dict_words[i] in input_dict_text):\n",
    "            a[i] = input_dict_text.get(dict_words[i])\n",
    "    return a\n",
    "    \n",
    "convert_words_to_numbers(data['text'][0])        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5d2a7357",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>[2.0, 1.0, 0.0, 3.0, 4.0, 2.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>[1.0, 2.0, 0.0, 2.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>[1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19992</th>\n",
       "      <td>1</td>\n",
       "      <td>[4.0, 1.0, 2.0, 2.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19993</th>\n",
       "      <td>1</td>\n",
       "      <td>[6.0, 3.0, 2.0, 0.0, 2.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>1</td>\n",
       "      <td>[3.0, 1.0, 0.0, 0.0, 1.0, 3.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>1</td>\n",
       "      <td>[6.0, 1.0, 0.0, 2.0, 3.0, 7.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19997 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                           features\n",
       "0         1  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "1         1  [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "2         1  [2.0, 1.0, 0.0, 3.0, 4.0, 2.0, 1.0, 1.0, 1.0, ...\n",
       "3         1  [1.0, 2.0, 0.0, 2.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "4         1  [1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "...     ...                                                ...\n",
       "19992     1  [4.0, 1.0, 2.0, 2.0, 1.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "19993     1  [6.0, 3.0, 2.0, 0.0, 2.0, 1.0, 1.0, 1.0, 1.0, ...\n",
       "19994     1  [3.0, 1.0, 0.0, 0.0, 1.0, 3.0, 1.0, 1.0, 1.0, ...\n",
       "19995     1  [6.0, 1.0, 0.0, 2.0, 3.0, 7.0, 1.0, 1.0, 1.0, ...\n",
       "19996     1  [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, ...\n",
       "\n",
       "[19997 rows x 2 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['features']=data['text'].apply(lambda x:convert_words_to_numbers(x))\n",
    "data = data.drop(['text'], axis=1)\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c5980b35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19997, 600)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "X = data['features'].to_numpy()\n",
    "y = data['label']\n",
    "\n",
    "X = np.array(list(map(lambda x: list(x), X)))\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "35316b5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20,)\n",
      "(600, 20)\n"
     ]
    }
   ],
   "source": [
    "# Define the number of topics or components\n",
    "num_components=20\n",
    "\n",
    "# Create SVD object\n",
    "lsa = TruncatedSVD(n_components=num_components, n_iter=100, random_state=42)\n",
    "\n",
    "# Fit SVD model on data\n",
    "lsa.fit_transform(X)\n",
    "\n",
    "# Get Singular values and Components \n",
    "Sigma = lsa.singular_values_ \n",
    "V_transpose = lsa.components_.T\n",
    "\n",
    "print(Sigma.shape)\n",
    "print(V_transpose.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cfba952f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0:  ['that', 'this', 'they', 'with', 'have', 'from', 'there', 'what', 'will', 'were']\n",
      "Topic 1:  ['jpeg', 'file', 'this', 'from', 'image', 'with', 'available', 'version', 'files', 'will']\n",
      "Topic 2:  ['they', 'were', 'jpeg', 'file', 'from', 'there', 'them', 'image', 'didn', 'with']\n",
      "Topic 3:  ['jpeg', 'that', 'image', 'color', 'images', 'format', 'file', 'president', 'free', 'than']\n",
      "Topic 4:  ['file', 'output', 'program', 'build', 'line', 'open', 'name', 'check', 'that', 'your']\n",
      "Topic 5:  ['that', 'windows', 'graphics', 'data', 'available', 'system', 'software', 'server', 'president', 'there']\n",
      "Topic 6:  ['from', 'that', 'image', 'were', 'their', 'data', 'been', 'states', 'turkish', 'president']\n",
      "Topic 7:  ['will', 'this', 'with', 'president', 'were', 'their', 'space', 'been', 'jesus', 'these']\n",
      "Topic 8:  ['windows', 'with', 'president', 'have', 'file', 'think', 'drive', 'card', 'date', 'lines']\n",
      "Topic 9:  ['windows', 'that', 'with', 'disk', 'jesus', 'drive', 'which', 'card', 'their', 'system']\n",
      "Topic 10:  ['what', 'file', 'internet', 'with', 'people', 'control', 'states', 'bill', 'information', 'their']\n",
      "Topic 11:  ['will', 'your', 'have', 'file', 'system', 'hockey', 'windows', 'disk', 'they', 'team']\n",
      "Topic 12:  ['this', 'will', 'from', 'jesus', 'windows', 'window', 'subject', 'said', 'file', 'jpeg']\n",
      "Topic 13:  ['have', 'this', 'windows', 'graphics', 'would', 'they', 'space', 'file', 'their', 'people']\n",
      "Topic 14:  ['file', 'data', 'image', 'scsi', 'they', 'drive', 'with', 'this', 'hard', 'control']\n",
      "Topic 15:  ['what', 'internet', 'output', 'jesus', 'people', 'email', 'mail', 'this', 'computer', 'graphics']\n",
      "Topic 16:  ['space', 'system', 'than', 'nasa', 'more', 'that', 'earth', 'disk', 'jpeg', 'other']\n",
      "Topic 17:  ['your', 'turkish', 'drive', 'disk', 'armenian', 'were', 'hard', 'people', 'jews', 'armenians']\n",
      "Topic 18:  ['jesus', 'space', 'about', 'earth', 'christ', 'have', 'there', 'nasa', 'said', 'what']\n",
      "Topic 19:  ['graphics', 'mail', 'with', 'hockey', 'send', 'your', 'team', 'this', 'game', 'season']\n"
     ]
    }
   ],
   "source": [
    "# Print the topics with their terms\n",
    "terms = dict_words\n",
    "\n",
    "for index, component in enumerate(lsa.components_):\n",
    "    zipped = zip(terms, component)\n",
    "    top_terms_key = sorted(zipped, key = lambda t: t[1], reverse=True)[:10]\n",
    "    top_terms_list = list(dict(top_terms_key).keys())\n",
    "    print(\"Topic \"+str(index)+\": \",top_terms_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27a05bf7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "013a69ce",
   "metadata": {},
   "source": [
    "# We could get the data from sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "06b6c6e6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['alt.atheism',\n",
      " 'comp.graphics',\n",
      " 'comp.os.ms-windows.misc',\n",
      " 'comp.sys.ibm.pc.hardware',\n",
      " 'comp.sys.mac.hardware',\n",
      " 'comp.windows.x',\n",
      " 'misc.forsale',\n",
      " 'rec.autos',\n",
      " 'rec.motorcycles',\n",
      " 'rec.sport.baseball',\n",
      " 'rec.sport.hockey',\n",
      " 'sci.crypt',\n",
      " 'sci.electronics',\n",
      " 'sci.med',\n",
      " 'sci.space',\n",
      " 'soc.religion.christian',\n",
      " 'talk.politics.guns',\n",
      " 'talk.politics.mideast',\n",
      " 'talk.politics.misc',\n",
      " 'talk.religion.misc']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from pprint import pprint\n",
    "\n",
    "newsgroups_data = fetch_20newsgroups()\n",
    "pprint(list(newsgroups_data.target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "458590a6",
   "metadata": {},
   "source": [
    "# TF-IDF -  term frequency–inverse document frequency\n",
    "\n",
    "We can use TF-IDF\n",
    "\n",
    "https://en.wikipedia.org/wiki/Tf%E2%80%93idf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3ed8e489",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.11/site-packages/sklearn/feature_extraction/text.py:528: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(11314, 129839)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "vectorizer = TfidfVectorizer(lowercase=True,\n",
    "                             stop_words='english',\n",
    "                             ngram_range=(1,1),\n",
    "                             tokenizer = tokenizer.tokenize)\n",
    "vectors = vectorizer.fit_transform(newsgroups_data.data)\n",
    "vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6820074a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the number of topics or components\n",
    "num_components=20\n",
    "\n",
    "# Create SVD object\n",
    "lsa = TruncatedSVD(n_components=num_components, n_iter=100, random_state=42)\n",
    "\n",
    "# Fit SVD model on data\n",
    "lsa.fit_transform(vectors)\n",
    "\n",
    "# Get Singular values and Components \n",
    "Sigma = lsa.singular_values_ \n",
    "V_transpose = lsa.components_.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1a5466cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0:  ['edu', 's', 't', 'com', '1', 'writes', 'article', 'people', 'don', 'subject']\n",
      "Topic 1:  ['1', '0', '_', '2', '3', 'windows', 'x', '4', '5', '6']\n",
      "Topic 2:  ['_', '___', '__', 'god', 'jesus', 'o', 'people', 'bnr', '____', 'bible']\n",
      "Topic 3:  ['key', 'clipper', 'encryption', 'com', 'chip', 'government', 'escrow', 'keys', 'access', '_']\n",
      "Topic 4:  ['edu', 'cs', 'pitt', 'geb', 'university', 'gordon', 'banks', 'nntp', 'host', 'article']\n",
      "Topic 5:  ['0', 'team', 'game', 'year', '1', 'hockey', '2', 'players', 's', 'games']\n",
      "Topic 6:  ['pitt', 'geb', 'banks', 'gordon', 'cs', 'key', 'god', '0', 'encryption', 'clipper']\n",
      "Topic 7:  ['israel', 'edu', 'israeli', 'jews', 'state', 'ohio', 'cleveland', 'university', 'cwru', 'turkish']\n",
      "Topic 8:  ['com', 'israel', 'geb', 'gordon', 'pitt', 'banks', 'armenian', 'turkish', 'israeli', 'armenians']\n",
      "Topic 9:  ['scsi', 'drive', 'ide', 'ca', 's', 'controller', 'card', 'bus', 'hard', 'drives']\n",
      "Topic 10:  ['scsi', 'drive', 'keith', 'ide', 'caltech', 'livesey', 'sgi', '0', 'morality', '_']\n",
      "Topic 11:  ['nasa', 'scsi', 'space', 'gov', 'drive', 'henry', 'ide', 'keith', 'caltech', 'alaska']\n",
      "Topic 12:  ['keith', 'caltech', 'livesey', 'morality', 'sgi', 'windows', 'objective', 't', 'cco', 'solntze']\n",
      "Topic 13:  ['israel', 'israeli', 'ca', 'jews', 'com', 'scsi', 'arab', 'team', 'sandvik', 'jake']\n",
      "Topic 14:  ['armenian', 'turkish', 'armenians', 'armenia', 'serdar', 'argic', 'scsi', 'sandvik', 'com', 'turks']\n",
      "Topic 15:  ['cc', 'columbia', 'gld', 'cunixb', 'edu', 'gun', 'dare', 'gary', 'l', 'stratus']\n",
      "Topic 16:  ['sandvik', 'windows', 'access', 'apple', 'kent', 'digex', 'newton', 'net', 'dos', 'pat']\n",
      "Topic 17:  ['sandvik', 'apple', 'kent', 'o', 'newton', 'columbia', 'ohio', 'uk', 'gld', 'x']\n",
      "Topic 18:  ['columbia', 'gld', 'cc', 'cunixb', 'access', 'ohio', 'digex', 'windows', 'dare', 'state']\n",
      "Topic 19:  ['cleveland', 'cwru', 'freenet', 'ins', 'reserve', 'western', 'po', 'case', 'usa', 'car']\n"
     ]
    }
   ],
   "source": [
    "# Print the topics with their terms\n",
    "terms  = vectorizer.get_feature_names_out()\n",
    "\n",
    "for index, component in enumerate(lsa.components_):\n",
    "    zipped = zip(terms, component)\n",
    "    top_terms_key=sorted(zipped, key = lambda t: t[1], reverse=True)[:10]\n",
    "    top_terms_list=list(dict(top_terms_key).keys())\n",
    "    print(\"Topic \"+str(index)+\": \",top_terms_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf740eb1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}