pretrained_clean_df.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "#import nltk\n",
        "#from nltk.corpus import stopwords\n",
        "#from nltk.tokenize import word_tokenize\n",
        "#from nltk.stem import WordNetLemmatizer\n",
        "#import re\n",
        "\n",
        "# Download required NLTK data\n",
        "#nltk.download('punkt')\n",
        "#nltk.download('stopwords')\n",
        "#nltk.download('wordnet')\n",
        "\n",
        "# Display the entire sorted DataFrame\n",
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', 500)\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "\n",
        "# File paths\n",
        "df_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/final_df.csv'\n",
        "desc_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/desc.csv'\n",
        "fitment_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/ftmnt_train.csv'\n",
        "\n",
        "over_train_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/over_train_indices.npy'\n",
        "val_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/val_indices.npy'\n",
        "test_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/test_indices.npy'\n",
        "\n",
        "# Read the CSV files\n",
        "df = pd.read_csv(df_file)\n",
        "df.replace({\"[]\": np.nan, \"['']\": np.nan, '[\"]': np.nan}, inplace=True)\n",
        "df.drop(columns=['ManufacturePartNumber'], inplace=True)\n",
        "\n",
        "over_train_indices = np.load(over_train_indices_path)\n",
        "val_indices = np.load(val_indices_path)\n",
        "test_indices = np.load(test_indices_path)\n",
        "\n",
        "train_df = df.take(over_train_indices)\n",
        "val_df = df.take(val_indices)\n",
        "test_df = df.take(test_indices)\n",
        "\n",
        "desc = pd.read_csv(desc_file).drop(columns=['Decoded_DESC'])\n",
        "fitment = pd.read_csv(fitment_file)\n",
        "\n",
        "def merge_data(fitment, df, desc):\n",
        "    return fitment.merge(df, on='RECORD_ID', how='right').merge(desc, on='RECORD_ID', how='left')\n",
        "\n",
        "merged_train_df = merge_data(fitment, train_df, desc)\n",
        "merged_val_df = merge_data(fitment, val_df, desc)\n",
        "merged_test_df = merge_data(fitment, test_df, desc)\n",
        "\n",
        "X_train = merged_train_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_val = merged_val_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_test = merged_test_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "\n",
        "# Function to create the concatenated string for each row\n",
        "def create_x_string(row):\n",
        "    return ', '.join([f\"{col}: {val}\" for col, val in row.items() if pd.notnull(val)])\n",
        "\n",
        "# Apply the function to each row in X and create a new column 'x_string'\n",
        "merged_train_df['processed_x_string'] = X_train.apply(create_x_string, axis=1).str.lower()\n",
        "merged_val_df['processed_x_string'] = X_val.apply(create_x_string, axis=1).str.lower()\n",
        "merged_test_df['processed_x_string'] = X_test.apply(create_x_string, axis=1).str.lower()\n",
        "\n",
        "merged_train_df['FTMNT_MAKE'] = merged_train_df['FTMNT_MAKE'].str.lower()\n",
        "merged_val_df['FTMNT_MAKE'] = merged_val_df['FTMNT_MAKE'].str.lower()\n",
        "merged_test_df['FTMNT_MAKE'] = merged_test_df['FTMNT_MAKE'].str.lower()\n",
        "\n",
        "# Group the data by description and aggregate the compatible makes into lists\n",
        "def group_by_make(df):\n",
        "    grouped = df.groupby('processed_x_string')['FTMNT_MAKE'].apply(lambda x: list(set(x))).reset_index()\n",
        "    return grouped[['FTMNT_MAKE', 'processed_x_string']]\n",
        "\n",
        "grouped_train = group_by_make(merged_train_df)\n",
        "grouped_val = group_by_make(merged_val_df)\n",
        "grouped_test = group_by_make(merged_test_df)\n",
        "\n",
        "grouped_train.to_csv('merged_train_df.csv', index=False)\n",
        "grouped_val.to_csv('merged_val_df.csv', index=False)\n",
        "grouped_test.to_csv('merged_test_df.csv', index=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Read the CSV file\n",
        "file_path = '/Users/cyrusaghaee/DS 207/Final Project/tabular_base_model_comparison.csv'\n",
        "tab_preds_df = pd.read_csv(file_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {},
      "outputs": [],
      "source": [
        "tab_preds_df.rename(columns={'Index': 'RECORD_ID'}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [],
      "source": [
        "tab_preds_df = tab_preds_df.merge(df[['RECORD_ID', 'Make']], on='RECORD_ID', how='left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>RECORD_ID</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>Target</th>\n",
              "      <th>Equal</th>\n",
              "      <th>Make</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NISSAN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>[['International Harvester']]</td>\n",
              "      <td>[['International Harvester']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>22</td>\n",
              "      <td>[['Buick'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Oldsmobile'], ['Saab']]</td>\n",
              "      <td>[['Buick'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Oldsmobile'], ['Saab']]</td>\n",
              "      <td>True</td>\n",
              "      <td>GMC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>34</td>\n",
              "      <td>[['Buick']]</td>\n",
              "      <td>[['Buick']]</td>\n",
              "      <td>True</td>\n",
              "      <td>BUICK</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>37</td>\n",
              "      <td>[['Volkswagen']]</td>\n",
              "      <td>[['Volkswagen']]</td>\n",
              "      <td>True</td>\n",
              "      <td>VOLKSWAGEN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6</td>\n",
              "      <td>60</td>\n",
              "      <td>[['Honda']]</td>\n",
              "      <td>[['Honda']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>7</td>\n",
              "      <td>70</td>\n",
              "      <td>[['INFINITI'], ['Nissan']]</td>\n",
              "      <td>[['INFINITI'], ['Nissan']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>8</td>\n",
              "      <td>71</td>\n",
              "      <td>[['Buick'], ['Chevrolet']]</td>\n",
              "      <td>[['Buick'], ['Chevrolet']]</td>\n",
              "      <td>True</td>\n",
              "      <td>CHEVROLET</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>9</td>\n",
              "      <td>90</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>True</td>\n",
              "      <td>MITSUBISHI</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>10</td>\n",
              "      <td>96</td>\n",
              "      <td>[['Cadillac'], ['Chevrolet'], ['GMC']]</td>\n",
              "      <td>[['Cadillac'], ['Chevrolet'], ['GMC']]</td>\n",
              "      <td>True</td>\n",
              "      <td>CHEVROLET TRUCK</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>11</td>\n",
              "      <td>111</td>\n",
              "      <td>[['Subaru']]</td>\n",
              "      <td>[['Subaru']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>12</td>\n",
              "      <td>161</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>True</td>\n",
              "      <td>MITSUBISHI MONTERO</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>13</td>\n",
              "      <td>166</td>\n",
              "      <td>[['Chevrolet'], ['GMC'], ['Oldsmobile']]</td>\n",
              "      <td>[['Chevrolet'], ['GMC'], ['Oldsmobile']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>14</td>\n",
              "      <td>194</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>15</td>\n",
              "      <td>201</td>\n",
              "      <td>[['Volkswagen']]</td>\n",
              "      <td>[['Volkswagen']]</td>\n",
              "      <td>True</td>\n",
              "      <td>VOLKSWAGEN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>16</td>\n",
              "      <td>255</td>\n",
              "      <td>[['Honda']]</td>\n",
              "      <td>[['Honda']]</td>\n",
              "      <td>True</td>\n",
              "      <td>HONDA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>17</td>\n",
              "      <td>262</td>\n",
              "      <td>[['Hyundai'], ['Kia']]</td>\n",
              "      <td>[['Hyundai'], ['Kia']]</td>\n",
              "      <td>True</td>\n",
              "      <td>HYUNDAI</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>18</td>\n",
              "      <td>271</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>True</td>\n",
              "      <td>TOYOTA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>19</td>\n",
              "      <td>281</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>True</td>\n",
              "      <td>TOYOTA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>20</td>\n",
              "      <td>282</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NISSAN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>21</td>\n",
              "      <td>291</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>22</td>\n",
              "      <td>293</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>True</td>\n",
              "      <td>BMW</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>23</td>\n",
              "      <td>295</td>\n",
              "      <td>[['Ford'], ['Lincoln']]</td>\n",
              "      <td>[['Ford'], ['Lincoln']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>24</td>\n",
              "      <td>300</td>\n",
              "      <td>[['Ford'], ['Lincoln'], ['Mercury']]</td>\n",
              "      <td>[['Ford'], ['Lincoln'], ['Mercury']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>25</td>\n",
              "      <td>321</td>\n",
              "      <td>[['Lincoln']]</td>\n",
              "      <td>[['Lincoln']]</td>\n",
              "      <td>True</td>\n",
              "      <td>LINCOLN NAVIGATOR</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>26</td>\n",
              "      <td>329</td>\n",
              "      <td>[['Chevrolet']]</td>\n",
              "      <td>[['Chevrolet']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>27</td>\n",
              "      <td>330</td>\n",
              "      <td>[['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu']]</td>\n",
              "      <td>[['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu']]</td>\n",
              "      <td>True</td>\n",
              "      <td>GMC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>28</td>\n",
              "      <td>359</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>True</td>\n",
              "      <td>BMW</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>29</td>\n",
              "      <td>371</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC']]</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC']]</td>\n",
              "      <td>True</td>\n",
              "      <td>CHEVY</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>30</td>\n",
              "      <td>376</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Pontiac'], ['Saab']]</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Pontiac'], ['Saab']]</td>\n",
              "      <td>True</td>\n",
              "      <td>CADILLAC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>31</td>\n",
              "      <td>389</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>[['Nissan']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>32</td>\n",
              "      <td>407</td>\n",
              "      <td>[['Fargo']]</td>\n",
              "      <td>[['Fargo']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FARGO</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>33</td>\n",
              "      <td>413</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['Ford'], ['GMC'], ['Mercury'], ['Pontiac']]</td>\n",
              "      <td>[['Buick'], ['Cadillac'], ['Chevrolet'], ['Ford'], ['GMC'], ['Mercury'], ['Pontiac']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>34</td>\n",
              "      <td>416</td>\n",
              "      <td>[['Mazda']]</td>\n",
              "      <td>[['Mazda']]</td>\n",
              "      <td>True</td>\n",
              "      <td>MAZDA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>35</td>\n",
              "      <td>424</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>[['BMW']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>36</td>\n",
              "      <td>432</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>[['Mitsubishi']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>37</td>\n",
              "      <td>434</td>\n",
              "      <td>[['Mini']]</td>\n",
              "      <td>[['Mini']]</td>\n",
              "      <td>True</td>\n",
              "      <td>MINI COOPER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>38</td>\n",
              "      <td>438</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>39</td>\n",
              "      <td>441</td>\n",
              "      <td>[['Plymouth']]</td>\n",
              "      <td>[['Plymouth']]</td>\n",
              "      <td>True</td>\n",
              "      <td>PLYMOUTH</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>40</td>\n",
              "      <td>442</td>\n",
              "      <td>[['Ford'], ['Lincoln']]</td>\n",
              "      <td>[['Ford'], ['Lincoln']]</td>\n",
              "      <td>True</td>\n",
              "      <td>LINCOLN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>41</td>\n",
              "      <td>460</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>[['Toyota']]</td>\n",
              "      <td>True</td>\n",
              "      <td>TOYOTA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>42</td>\n",
              "      <td>462</td>\n",
              "      <td>[['Cadillac'], ['Saturn']]</td>\n",
              "      <td>[['Cadillac'], ['Saturn']]</td>\n",
              "      <td>True</td>\n",
              "      <td>CADILLAC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>43</td>\n",
              "      <td>464</td>\n",
              "      <td>[['Dodge'], ['Jeep'], ['Ram']]</td>\n",
              "      <td>[['Dodge'], ['Jeep'], ['Ram']]</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>44</td>\n",
              "      <td>480</td>\n",
              "      <td>[['Ford'], ['Mercury']]</td>\n",
              "      <td>[['Ford']]</td>\n",
              "      <td>False</td>\n",
              "      <td>FORD TRUCK</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>45</td>\n",
              "      <td>496</td>\n",
              "      <td>[['Lincoln']]</td>\n",
              "      <td>[['Lincoln']]</td>\n",
              "      <td>True</td>\n",
              "      <td>LINCOLN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>46</td>\n",
              "      <td>505</td>\n",
              "      <td>[['Dodge']]</td>\n",
              "      <td>[['Dodge']]</td>\n",
              "      <td>True</td>\n",
              "      <td>DODGE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>47</td>\n",
              "      <td>508</td>\n",
              "      <td>[['Hyundai']]</td>\n",
              "      <td>[['Hyundai']]</td>\n",
              "      <td>True</td>\n",
              "      <td>HYUNDAI KIA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>48</td>\n",
              "      <td>534</td>\n",
              "      <td>[['Chevrolet'], ['Ford'], ['Volkswagen']]</td>\n",
              "      <td>[['Chevrolet'], ['Ford'], ['Volkswagen']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>49</td>\n",
              "      <td>538</td>\n",
              "      <td>[['Dodge'], ['Ford']]</td>\n",
              "      <td>[['Dodge'], ['Ford']]</td>\n",
              "      <td>True</td>\n",
              "      <td>FORD</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Unnamed: 0  RECORD_ID                                                                                        Predicted                                                                                           Target  Equal                Make\n",
              "0            0          2                                                                                     [['Nissan']]                                                                                     [['Nissan']]   True              NISSAN\n",
              "1            1          3                                                                                       [['Ford']]                                                                                       [['Ford']]   True                FORD\n",
              "2            2          8                                                                    [['International Harvester']]                                                                    [['International Harvester']]   True                 NaN\n",
              "3            3         22             [['Buick'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Oldsmobile'], ['Saab']]             [['Buick'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Oldsmobile'], ['Saab']]   True                 GMC\n",
              "4            4         34                                                                                      [['Buick']]                                                                                      [['Buick']]   True               BUICK\n",
              "5            5         37                                                                                 [['Volkswagen']]                                                                                 [['Volkswagen']]   True          VOLKSWAGEN\n",
              "6            6         60                                                                                      [['Honda']]                                                                                      [['Honda']]   True                 NaN\n",
              "7            7         70                                                                       [['INFINITI'], ['Nissan']]                                                                       [['INFINITI'], ['Nissan']]   True                 NaN\n",
              "8            8         71                                                                       [['Buick'], ['Chevrolet']]                                                                       [['Buick'], ['Chevrolet']]   True           CHEVROLET\n",
              "9            9         90                                                                                 [['Mitsubishi']]                                                                                 [['Mitsubishi']]   True          MITSUBISHI\n",
              "10          10         96                                                           [['Cadillac'], ['Chevrolet'], ['GMC']]                                                           [['Cadillac'], ['Chevrolet'], ['GMC']]   True     CHEVROLET TRUCK\n",
              "11          11        111                                                                                     [['Subaru']]                                                                                     [['Subaru']]   True                 NaN\n",
              "12          12        161                                                                                 [['Mitsubishi']]                                                                                 [['Mitsubishi']]   True  MITSUBISHI MONTERO\n",
              "13          13        166                                                         [['Chevrolet'], ['GMC'], ['Oldsmobile']]                                                         [['Chevrolet'], ['GMC'], ['Oldsmobile']]   True                 NaN\n",
              "14          14        194                                                                                       [['Ford']]                                                                                       [['Ford']]   True                FORD\n",
              "15          15        201                                                                                 [['Volkswagen']]                                                                                 [['Volkswagen']]   True          VOLKSWAGEN\n",
              "16          16        255                                                                                      [['Honda']]                                                                                      [['Honda']]   True               HONDA\n",
              "17          17        262                                                                           [['Hyundai'], ['Kia']]                                                                           [['Hyundai'], ['Kia']]   True             HYUNDAI\n",
              "18          18        271                                                                                     [['Toyota']]                                                                                     [['Toyota']]   True              TOYOTA\n",
              "19          19        281                                                                                     [['Toyota']]                                                                                     [['Toyota']]   True              TOYOTA\n",
              "20          20        282                                                                                     [['Nissan']]                                                                                     [['Nissan']]   True              NISSAN\n",
              "21          21        291                                                                                     [['Nissan']]                                                                                     [['Nissan']]   True                 NaN\n",
              "22          22        293                                                                                        [['BMW']]                                                                                        [['BMW']]   True                 BMW\n",
              "23          23        295                                                                          [['Ford'], ['Lincoln']]                                                                          [['Ford'], ['Lincoln']]   True                FORD\n",
              "24          24        300                                                             [['Ford'], ['Lincoln'], ['Mercury']]                                                             [['Ford'], ['Lincoln'], ['Mercury']]   True                 NaN\n",
              "25          25        321                                                                                    [['Lincoln']]                                                                                    [['Lincoln']]   True   LINCOLN NAVIGATOR\n",
              "26          26        329                                                                                  [['Chevrolet']]                                                                                  [['Chevrolet']]   True                 NaN\n",
              "27          27        330                                                  [['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu']]                                                  [['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu']]   True                 GMC\n",
              "28          28        359                                                                                        [['BMW']]                                                                                        [['BMW']]   True                 BMW\n",
              "29          29        371                                                [['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC']]                                                [['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC']]   True               CHEVY\n",
              "30          30        376  [['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Pontiac'], ['Saab']]  [['Buick'], ['Cadillac'], ['Chevrolet'], ['GMC'], ['Hummer'], ['Isuzu'], ['Pontiac'], ['Saab']]   True            CADILLAC\n",
              "31          31        389                                                                                     [['Nissan']]                                                                                     [['Nissan']]   True                 NaN\n",
              "32          32        407                                                                                      [['Fargo']]                                                                                      [['Fargo']]   True               FARGO\n",
              "33          33        413            [['Buick'], ['Cadillac'], ['Chevrolet'], ['Ford'], ['GMC'], ['Mercury'], ['Pontiac']]            [['Buick'], ['Cadillac'], ['Chevrolet'], ['Ford'], ['GMC'], ['Mercury'], ['Pontiac']]   True                 NaN\n",
              "34          34        416                                                                                      [['Mazda']]                                                                                      [['Mazda']]   True               MAZDA\n",
              "35          35        424                                                                                        [['BMW']]                                                                                        [['BMW']]   True                 NaN\n",
              "36          36        432                                                                                 [['Mitsubishi']]                                                                                 [['Mitsubishi']]   True                 NaN\n",
              "37          37        434                                                                                       [['Mini']]                                                                                       [['Mini']]   True         MINI COOPER\n",
              "38          38        438                                                                                       [['Ford']]                                                                                       [['Ford']]   True                FORD\n",
              "39          39        441                                                                                   [['Plymouth']]                                                                                   [['Plymouth']]   True            PLYMOUTH\n",
              "40          40        442                                                                          [['Ford'], ['Lincoln']]                                                                          [['Ford'], ['Lincoln']]   True             LINCOLN\n",
              "41          41        460                                                                                     [['Toyota']]                                                                                     [['Toyota']]   True              TOYOTA\n",
              "42          42        462                                                                       [['Cadillac'], ['Saturn']]                                                                       [['Cadillac'], ['Saturn']]   True            CADILLAC\n",
              "43          43        464                                                                   [['Dodge'], ['Jeep'], ['Ram']]                                                                   [['Dodge'], ['Jeep'], ['Ram']]   True                 NaN\n",
              "44          44        480                                                                          [['Ford'], ['Mercury']]                                                                                       [['Ford']]  False          FORD TRUCK\n",
              "45          45        496                                                                                    [['Lincoln']]                                                                                    [['Lincoln']]   True             LINCOLN\n",
              "46          46        505                                                                                      [['Dodge']]                                                                                      [['Dodge']]   True               DODGE\n",
              "47          47        508                                                                                    [['Hyundai']]                                                                                    [['Hyundai']]   True         HYUNDAI KIA\n",
              "48          48        534                                                        [['Chevrolet'], ['Ford'], ['Volkswagen']]                                                        [['Chevrolet'], ['Ford'], ['Volkswagen']]   True                FORD\n",
              "49          49        538                                                                            [['Dodge'], ['Ford']]                                                                            [['Dodge'], ['Ford']]   True                FORD"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tab_pred_df = tab_preds_df.copy()\n",
        "tab_pred_df.head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overall percentage of correct predictions: 91.06%\n",
            "Percentage of correct predictions when 'Make' is not NaN: 94.03%\n",
            "Percentage of correct predictions when 'Make' is NaN: 85.52%\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import ast\n",
        "\n",
        "tab_pred_df = tab_preds_df.copy()\n",
        "\n",
        "# Convert string representations of lists into actual lists\n",
        "tab_pred_df['Predicted'] = tab_pred_df['Predicted'].apply(ast.literal_eval)\n",
        "tab_pred_df['Target'] = tab_pred_df['Target'].apply(ast.literal_eval)\n",
        "\n",
        "# Initialize an empty list to store the new rows\n",
        "new_rows = []\n",
        "\n",
        "# Iterate through each row\n",
        "for idx, row in tab_pred_df.iterrows():\n",
        "    predictions = [item[0] for item in row['Predicted']]  # Flatten the list\n",
        "    targets = [item[0] for item in row['Target']]  # Flatten the list\n",
        "    \n",
        "    for target in targets:\n",
        "        correct = 1 if target in predictions else 0\n",
        "        new_rows.append({'Target': target, 'Correct': correct, 'Make': row['Make']})\n",
        "\n",
        "# Create a new dataframe with the new rows\n",
        "new_df = pd.DataFrame(new_rows)\n",
        "\n",
        "# Calculate the overall percentage of correct predictions\n",
        "overall_correct_percentage = new_df['Correct'].mean() * 100\n",
        "\n",
        "# Calculate the percentage of correct predictions when 'Make' is not NaN\n",
        "not_nan_correct_percentage = new_df[new_df['Make'].notna()]['Correct'].mean() * 100\n",
        "\n",
        "# Calculate the percentage of correct predictions when 'Make' is NaN\n",
        "nan_correct_percentage = new_df[new_df['Make'].isna()]['Correct'].mean() * 100\n",
        "\n",
        "# Display the results\n",
        "print(f\"Overall percentage of correct predictions: {overall_correct_percentage:.2f}%\")\n",
        "print(f\"Percentage of correct predictions when 'Make' is not NaN: {not_nan_correct_percentage:.2f}%\")\n",
        "print(f\"Percentage of correct predictions when 'Make' is NaN: {nan_correct_percentage:.2f}%\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overall percentage of correct predictions: 91.06%\n",
            "Percentage of correct predictions when 'Make' is not NaN: 94.03%\n",
            "Percentage of correct predictions when 'Make' is NaN: 85.52%\n",
            "\n",
            "Overall count: 850\n",
            "Count when 'Make' is not NaN: 553\n",
            "Count when 'Make' is NaN: 297\n",
            "\n",
            "Sample of overall dataframe:\n",
            "       Target  Correct    Make\n",
            "464      Jeep        1   DODGE\n",
            "249    Suzuki        1  SUZUKI\n",
            "629  Cadillac        1     NaN\n",
            "605    Toyota        1  TOYOTA\n",
            "104   Mercury        1     NaN\n",
            "\n",
            "Sample when 'Make' is not NaN:\n",
            "        Target  Correct                    Make\n",
            "511        GMC        1                    CASE\n",
            "497     Datsun        1                  NISSAN\n",
            "360  Chevrolet        1  CHEVROLET C6500 KODIAK\n",
            "488      Dodge        1                CHRYSLER\n",
            "677       Ford        1                    FORD\n",
            "\n",
            "Sample when 'Make' is NaN:\n",
            "            Target  Correct Make\n",
            "518           Ford        1  NaN\n",
            "184        Pontiac        1  NaN\n",
            "23       Chevrolet        1  NaN\n",
            "773           Ford        1  NaN\n",
            "565  Mercedes-Benz        1  NaN\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import ast\n",
        "\n",
        "tab_pred_df = tab_preds_df.copy()\n",
        "\n",
        "# Convert string representations of lists into actual lists\n",
        "tab_pred_df['Predicted'] = tab_pred_df['Predicted'].apply(ast.literal_eval)\n",
        "tab_pred_df['Target'] = tab_pred_df['Target'].apply(ast.literal_eval)\n",
        "\n",
        "# Initialize an empty list to store the new rows\n",
        "new_rows = []\n",
        "\n",
        "# Iterate through each row\n",
        "for idx, row in tab_pred_df.iterrows():\n",
        "    predictions = [item[0] for item in row['Predicted']]  # Flatten the list\n",
        "    targets = [item[0] for item in row['Target']]  # Flatten the list\n",
        "    \n",
        "    for target in targets:\n",
        "        correct = 1 if target in predictions else 0\n",
        "        new_rows.append({'Target': target, 'Correct': correct, 'Make': row['Make']})\n",
        "\n",
        "# Create a new dataframe with the new rows\n",
        "new_df = pd.DataFrame(new_rows)\n",
        "\n",
        "# Calculate the overall percentage of correct predictions\n",
        "overall_correct_percentage = new_df['Correct'].mean() * 100\n",
        "\n",
        "# Calculate the percentage of correct predictions when 'Make' is not NaN\n",
        "not_nan_df = new_df[new_df['Make'].notna()]\n",
        "not_nan_correct_percentage = not_nan_df['Correct'].mean() * 100\n",
        "\n",
        "# Calculate the percentage of correct predictions when 'Make' is NaN\n",
        "nan_df = new_df[new_df['Make'].isna()]\n",
        "nan_correct_percentage = nan_df['Correct'].mean() * 100\n",
        "\n",
        "# Count of each group\n",
        "overall_count = len(new_df)\n",
        "not_nan_count = len(not_nan_df)\n",
        "nan_count = len(nan_df)\n",
        "\n",
        "# Display the results\n",
        "print(f\"Overall percentage of correct predictions: {overall_correct_percentage:.2f}%\")\n",
        "print(f\"Percentage of correct predictions when 'Make' is not NaN: {not_nan_correct_percentage:.2f}%\")\n",
        "print(f\"Percentage of correct predictions when 'Make' is NaN: {nan_correct_percentage:.2f}%\")\n",
        "\n",
        "print(f\"\\nOverall count: {overall_count}\")\n",
        "print(f\"Count when 'Make' is not NaN: {not_nan_count}\")\n",
        "print(f\"Count when 'Make' is NaN: {nan_count}\")\n",
        "\n",
        "# Show a sample of each group\n",
        "print(\"\\nSample of overall dataframe:\")\n",
        "print(new_df.sample(5))\n",
        "\n",
        "print(\"\\nSample when 'Make' is not NaN:\")\n",
        "print(not_nan_df.sample(5))\n",
        "\n",
        "print(\"\\nSample when 'Make' is NaN:\")\n",
        "print(nan_df.sample(5))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "#import nltk\n",
        "#from nltk.corpus import stopwords\n",
        "#from nltk.tokenize import word_tokenize\n",
        "#from nltk.stem import WordNetLemmatizer\n",
        "#import re\n",
        "\n",
        "# Download required NLTK data\n",
        "#nltk.download('punkt')\n",
        "#nltk.download('stopwords')\n",
        "#nltk.download('wordnet')\n",
        "\n",
        "# Display the entire sorted DataFrame\n",
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', 500)\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "\n",
        "# File paths\n",
        "df_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/final_df.csv'\n",
        "desc_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/desc.csv'\n",
        "fitment_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/ftmnt_train.csv'\n",
        "\n",
        "over_train_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/over_train_indices.npy'\n",
        "val_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/val_indices.npy'\n",
        "test_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/test_indices.npy'\n",
        "\n",
        "# Read the CSV files\n",
        "df = pd.read_csv(df_file)\n",
        "df.replace({\"[]\": np.nan, \"['']\": np.nan, '[\"]': np.nan}, inplace=True)\n",
        "df.drop(columns=['ManufacturePartNumber'], inplace=True)\n",
        "\n",
        "over_train_indices = np.load(over_train_indices_path)\n",
        "val_indices = np.load(val_indices_path)\n",
        "test_indices = np.load(test_indices_path)\n",
        "\n",
        "train_df = df.take(over_train_indices)\n",
        "val_df = df.take(val_indices)\n",
        "test_df = df.take(test_indices)\n",
        "\n",
        "desc = pd.read_csv(desc_file).drop(columns=['Decoded_DESC'])\n",
        "fitment = pd.read_csv(fitment_file)\n",
        "\n",
        "def merge_data(fitment, df, desc):\n",
        "    return fitment.merge(df, on='RECORD_ID', how='right').merge(desc, on='RECORD_ID', how='left')\n",
        "\n",
        "merged_train_df = merge_data(fitment, train_df, desc)\n",
        "merged_val_df = merge_data(fitment, val_df, desc)\n",
        "merged_test_df = merge_data(fitment, test_df, desc)\n",
        "\n",
        "X_train = merged_train_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_val = merged_val_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_test = merged_test_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "\n",
        "# Function to create the concatenated string for each row\n",
        "def create_x_string(row):\n",
        "    return ', '.join([f\"{col}: {val}\" for col, val in row.items() if pd.notnull(val)])\n",
        "\n",
        "# Apply the function to each row in X and create a new column 'x_string'\n",
        "merged_train_df['processed_x_string'] = X_train.apply(create_x_string, axis=1).str.lower()\n",
        "merged_val_df['processed_x_string'] = X_val.apply(create_x_string, axis=1).str.lower()\n",
        "merged_test_df['processed_x_string'] = X_test.apply(create_x_string, axis=1).str.lower()\n",
        "\n",
        "merged_train_df['FTMNT_MAKE'] = merged_train_df['FTMNT_MAKE'].str.lower()\n",
        "merged_val_df['FTMNT_MAKE'] = merged_val_df['FTMNT_MAKE'].str.lower()\n",
        "merged_test_df['FTMNT_MAKE'] = merged_test_df['FTMNT_MAKE'].str.lower()\n",
        "\n",
        "# Group the data by description and aggregate the compatible makes into lists\n",
        "def group_by_make(df):\n",
        "    grouped = df.groupby('processed_x_string')['FTMNT_MAKE'].apply(lambda x: list(set(x))).reset_index()\n",
        "    return grouped[['FTMNT_MAKE', 'processed_x_string']]\n",
        "\n",
        "grouped_train = group_by_make(merged_train_df)\n",
        "grouped_val = group_by_make(merged_val_df)\n",
        "grouped_test = group_by_make(merged_test_df)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [],
      "source": [
        "pred_test_df = test_df.merge(desc, on='RECORD_ID', how='left')[['RECORD_ID', 'Cleaned_DESC']]\n",
        "pred_test_df.to_csv('pred_test_df.csv', index=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(7066, 19)\n",
            "RECORD_ID\n",
            "3423    68\n",
            "1727    64\n",
            "1668    57\n",
            "3801    57\n",
            "3009    57\n",
            "1110    57\n",
            "3622    56\n",
            "2284    55\n",
            "1847    55\n",
            "3611    54\n",
            "2423    52\n",
            "1585    51\n",
            "3970    51\n",
            "3939    50\n",
            "2880    50\n",
            "768     50\n",
            "2855    50\n",
            "2846    49\n",
            "3121    48\n",
            "2716    48\n",
            "2899    43\n",
            "1732    43\n",
            "3834    43\n",
            "928     42\n",
            "1048    42\n",
            "1904    41\n",
            "3004    41\n",
            "1522    41\n",
            "645     41\n",
            "855     41\n",
            "2931    41\n",
            "2774    41\n",
            "1506    40\n",
            "1169    39\n",
            "2358    38\n",
            "3955    38\n",
            "3727    31\n",
            "3763    30\n",
            "2202    29\n",
            "2056    29\n",
            "572     28\n",
            "1372    27\n",
            "3219    27\n",
            "3380    26\n",
            "3914    26\n",
            "936     25\n",
            "3930    25\n",
            "553     25\n",
            "2176    25\n",
            "2858    23\n",
            "749     22\n",
            "2823    21\n",
            "96      20\n",
            "3884    20\n",
            "651     20\n",
            "3188    19\n",
            "2064    19\n",
            "1667    18\n",
            "2557    18\n",
            "3328    17\n",
            "430     17\n",
            "3226    16\n",
            "1252    16\n",
            "3665    16\n",
            "210     15\n",
            "761     15\n",
            "2950    15\n",
            "488     14\n",
            "2702    14\n",
            "76      14\n",
            "511     13\n",
            "2292    13\n",
            "3187    12\n",
            "2103    12\n",
            "2717    11\n",
            "3566    11\n",
            "2859    11\n",
            "1134    11\n",
            "3133    11\n",
            "3558    11\n",
            "3261    10\n",
            "836     10\n",
            "3232    10\n",
            "2802     9\n",
            "3810     9\n",
            "1610     9\n",
            "2379     9\n",
            "727      9\n",
            "3995     8\n",
            "1731     8\n",
            "2097     8\n",
            "1057     8\n",
            "701      8\n",
            "1744     8\n",
            "3664     8\n",
            "547      8\n",
            "827      7\n",
            "2695     7\n",
            "1709     7\n",
            "2727     7\n",
            "507      7\n",
            "2673     6\n",
            "570      6\n",
            "2611     6\n",
            "2906     6\n",
            "1809     6\n",
            "1007     6\n",
            "3021     6\n",
            "2139     6\n",
            "1548     6\n",
            "454      6\n",
            "1142     6\n",
            "776      6\n",
            "1380     6\n",
            "2987     6\n",
            "2072     6\n",
            "1184     5\n",
            "1428     5\n",
            "2870     5\n",
            "2249     5\n",
            "2676     5\n",
            "3024     5\n",
            "1985     5\n",
            "1781     5\n",
            "1303     5\n",
            "3011     5\n",
            "374      5\n",
            "1992     5\n",
            "2135     5\n",
            "2147     5\n",
            "1785     5\n",
            "2574     5\n",
            "3950     5\n",
            "2019     4\n",
            "789      4\n",
            "3503     4\n",
            "3381     4\n",
            "1587     4\n",
            "2758     4\n",
            "3119     4\n",
            "1764     4\n",
            "1542     4\n",
            "224      4\n",
            "3155     4\n",
            "3201     4\n",
            "2089     4\n",
            "2130     4\n",
            "2828     4\n",
            "2198     4\n",
            "1607     4\n",
            "1223     4\n",
            "3012     4\n",
            "3623     4\n",
            "1946     4\n",
            "1290     4\n",
            "1397     4\n",
            "2168     4\n",
            "2298     4\n",
            "2154     4\n",
            "2324     4\n",
            "269      4\n",
            "3805     4\n",
            "3302     4\n",
            "1445     4\n",
            "3518     4\n",
            "971      4\n",
            "1504     4\n",
            "173      4\n",
            "826      3\n",
            "794      3\n",
            "2836     3\n",
            "2241     3\n",
            "411      3\n",
            "2648     3\n",
            "3714     3\n",
            "2371     3\n",
            "1855     3\n",
            "3549     3\n",
            "2806     3\n",
            "1311     3\n",
            "771      3\n",
            "1313     3\n",
            "254      3\n",
            "2440     3\n",
            "854      3\n",
            "2829     3\n",
            "235      3\n",
            "2355     3\n",
            "3568     3\n",
            "354      3\n",
            "942      3\n",
            "2366     3\n",
            "3111     3\n",
            "3663     3\n",
            "2408     3\n",
            "533      3\n",
            "2869     3\n",
            "1474     3\n",
            "3974     3\n",
            "2085     3\n",
            "3447     3\n",
            "3833     3\n",
            "2513     3\n",
            "3366     3\n",
            "3979     3\n",
            "3427     3\n",
            "86       3\n",
            "1488     3\n",
            "1545     3\n",
            "1190     3\n",
            "1149     3\n",
            "1589     3\n",
            "3924     3\n",
            "641      3\n",
            "11       3\n",
            "2170     3\n",
            "2643     3\n",
            "10       3\n",
            "1902     3\n",
            "2327     2\n",
            "1035     2\n",
            "2159     2\n",
            "2550     2\n",
            "1637     2\n",
            "1955     2\n",
            "3034     2\n",
            "2299     2\n",
            "2341     2\n",
            "1405     2\n",
            "588      2\n",
            "752      2\n",
            "861      2\n",
            "1221     2\n",
            "3018     2\n",
            "666      2\n",
            "3020     2\n",
            "2125     2\n",
            "3883     2\n",
            "2688     2\n",
            "2115     2\n",
            "1321     2\n",
            "1074     2\n",
            "3601     2\n",
            "634      2\n",
            "3399     2\n",
            "2402     2\n",
            "807      2\n",
            "3609     2\n",
            "2398     2\n",
            "2396     2\n",
            "3402     2\n",
            "2728     2\n",
            "1346     2\n",
            "3414     2\n",
            "652      2\n",
            "2352     2\n",
            "1740     2\n",
            "2744     2\n",
            "1069     2\n",
            "2107     2\n",
            "52       2\n",
            "2017     2\n",
            "3570     2\n",
            "2114     2\n",
            "3046     2\n",
            "2007     2\n",
            "2697     2\n",
            "449      2\n",
            "1828     2\n",
            "2686     2\n",
            "1419     2\n",
            "143      2\n",
            "3837     2\n",
            "923      2\n",
            "2866     2\n",
            "2279     2\n",
            "1256     2\n",
            "3757     2\n",
            "1207     2\n",
            "2210     2\n",
            "2250     2\n",
            "1431     2\n",
            "3859     2\n",
            "726      2\n",
            "3138     2\n",
            "1624     2\n",
            "3851     2\n",
            "3140     2\n",
            "1594     2\n",
            "934      2\n",
            "1930     2\n",
            "197      2\n",
            "1605     2\n",
            "2597     1\n",
            "2703     1\n",
            "2704     1\n",
            "2705     1\n",
            "2706     1\n",
            "2707     1\n",
            "2596     1\n",
            "2667     1\n",
            "2598     1\n",
            "2638     1\n",
            "2599     1\n",
            "2600     1\n",
            "2701     1\n",
            "2700     1\n",
            "2699     1\n",
            "2656     1\n",
            "2657     1\n",
            "2637     1\n",
            "2698     1\n",
            "2601     1\n",
            "2639     1\n",
            "2654     1\n",
            "2595     1\n",
            "2647     1\n",
            "2587     1\n",
            "2651     1\n",
            "2713     1\n",
            "2719     1\n",
            "2649     1\n",
            "2642     1\n",
            "2714     1\n",
            "2640     1\n",
            "2715     1\n",
            "2646     1\n",
            "2645     1\n",
            "2586     1\n",
            "2644     1\n",
            "2718     1\n",
            "2652     1\n",
            "2641     1\n",
            "2712     1\n",
            "2588     1\n",
            "2711     1\n",
            "2710     1\n",
            "2589     1\n",
            "2653     1\n",
            "2709     1\n",
            "2590     1\n",
            "2591     1\n",
            "2592     1\n",
            "2708     1\n",
            "2636     1\n",
            "2655     1\n",
            "2593     1\n",
            "2594     1\n",
            "2658     1\n",
            "2604     1\n",
            "2635     1\n",
            "2687     1\n",
            "2626     1\n",
            "2685     1\n",
            "2684     1\n",
            "2683     1\n",
            "2682     1\n",
            "2625     1\n",
            "2681     1\n",
            "2665     1\n",
            "2680     1\n",
            "2615     1\n",
            "2679     1\n",
            "2616     1\n",
            "2678     1\n",
            "2677     1\n",
            "2617     1\n",
            "2618     1\n",
            "2675     1\n",
            "2619     1\n",
            "2666     1\n",
            "2674     1\n",
            "2624     1\n",
            "2620     1\n",
            "2672     1\n",
            "2621     1\n",
            "2671     1\n",
            "2670     1\n",
            "2623     1\n",
            "2622     1\n",
            "2669     1\n",
            "2614     1\n",
            "2664     1\n",
            "2602     1\n",
            "2613     1\n",
            "2603     1\n",
            "2634     1\n",
            "2633     1\n",
            "2668     1\n",
            "2659     1\n",
            "2605     1\n",
            "2696     1\n",
            "2632     1\n",
            "2631     1\n",
            "2630     1\n",
            "2629     1\n",
            "2606     1\n",
            "2628     1\n",
            "2660     1\n",
            "2661     1\n",
            "2662     1\n",
            "2607     1\n",
            "2694     1\n",
            "2608     1\n",
            "2693     1\n",
            "2609     1\n",
            "2610     1\n",
            "2692     1\n",
            "2691     1\n",
            "2627     1\n",
            "2690     1\n",
            "2612     1\n",
            "2663     1\n",
            "2689     1\n",
            "2650     1\n",
            "2771     1\n",
            "2720     1\n",
            "2887     1\n",
            "2875     1\n",
            "2876     1\n",
            "2877     1\n",
            "2878     1\n",
            "2879     1\n",
            "2881     1\n",
            "2882     1\n",
            "2883     1\n",
            "2884     1\n",
            "2885     1\n",
            "2886     1\n",
            "2888     1\n",
            "2721     1\n",
            "2889     1\n",
            "2890     1\n",
            "2891     1\n",
            "2892     1\n",
            "2893     1\n",
            "2894     1\n",
            "2895     1\n",
            "2896     1\n",
            "2897     1\n",
            "2898     1\n",
            "2900     1\n",
            "2874     1\n",
            "2873     1\n",
            "2872     1\n",
            "2871     1\n",
            "2841     1\n",
            "2842     1\n",
            "2843     1\n",
            "2844     1\n",
            "2845     1\n",
            "2847     1\n",
            "2848     1\n",
            "2849     1\n",
            "2850     1\n",
            "2851     1\n",
            "2852     1\n",
            "2853     1\n",
            "2854     1\n",
            "2856     1\n",
            "2857     1\n",
            "2860     1\n",
            "2861     1\n",
            "2862     1\n",
            "2863     1\n",
            "2864     1\n",
            "2865     1\n",
            "2867     1\n",
            "2868     1\n",
            "2901     1\n",
            "2902     1\n",
            "2903     1\n",
            "2932     1\n",
            "2934     1\n",
            "2935     1\n",
            "2936     1\n",
            "2937     1\n",
            "2938     1\n",
            "2939     1\n",
            "2940     1\n",
            "2941     1\n",
            "2942     1\n",
            "2943     1\n",
            "2944     1\n",
            "2945     1\n",
            "2946     1\n",
            "2947     1\n",
            "2948     1\n",
            "2949     1\n",
            "2951     1\n",
            "2952     1\n",
            "2953     1\n",
            "2954     1\n",
            "2955     1\n",
            "2956     1\n",
            "2957     1\n",
            "2933     1\n",
            "2930     1\n",
            "2904     1\n",
            "2929     1\n",
            "2905     1\n",
            "2907     1\n",
            "2908     1\n",
            "2909     1\n",
            "2910     1\n",
            "2911     1\n",
            "2912     1\n",
            "2913     1\n",
            "2914     1\n",
            "2915     1\n",
            "2916     1\n",
            "2917     1\n",
            "2918     1\n",
            "2919     1\n",
            "2920     1\n",
            "2921     1\n",
            "2922     1\n",
            "2923     1\n",
            "2924     1\n",
            "2925     1\n",
            "2926     1\n",
            "2927     1\n",
            "2928     1\n",
            "2840     1\n",
            "2839     1\n",
            "2838     1\n",
            "2750     1\n",
            "2752     1\n",
            "2753     1\n",
            "2754     1\n",
            "2755     1\n",
            "2756     1\n",
            "2757     1\n",
            "2759     1\n",
            "2760     1\n",
            "2761     1\n",
            "2762     1\n",
            "2763     1\n",
            "2764     1\n",
            "2765     1\n",
            "2766     1\n",
            "2767     1\n",
            "2768     1\n",
            "2769     1\n",
            "2770     1\n",
            "2584     1\n",
            "2772     1\n",
            "2773     1\n",
            "2775     1\n",
            "2776     1\n",
            "2751     1\n",
            "2749     1\n",
            "2778     1\n",
            "2748     1\n",
            "2722     1\n",
            "2723     1\n",
            "2724     1\n",
            "2725     1\n",
            "2726     1\n",
            "2729     1\n",
            "2730     1\n",
            "2731     1\n",
            "2732     1\n",
            "2733     1\n",
            "2734     1\n",
            "2735     1\n",
            "2736     1\n",
            "2737     1\n",
            "2738     1\n",
            "2739     1\n",
            "2740     1\n",
            "2741     1\n",
            "2742     1\n",
            "2743     1\n",
            "2745     1\n",
            "2746     1\n",
            "2747     1\n",
            "2777     1\n",
            "2779     1\n",
            "2837     1\n",
            "2808     1\n",
            "2810     1\n",
            "2811     1\n",
            "2812     1\n",
            "2813     1\n",
            "2814     1\n",
            "2815     1\n",
            "2816     1\n",
            "2817     1\n",
            "2818     1\n",
            "2819     1\n",
            "2820     1\n",
            "2821     1\n",
            "2822     1\n",
            "2824     1\n",
            "2825     1\n",
            "2826     1\n",
            "2827     1\n",
            "2830     1\n",
            "2831     1\n",
            "2832     1\n",
            "2833     1\n",
            "2834     1\n",
            "2835     1\n",
            "2809     1\n",
            "2807     1\n",
            "2780     1\n",
            "2805     1\n",
            "2781     1\n",
            "2782     1\n",
            "2783     1\n",
            "2784     1\n",
            "2785     1\n",
            "2786     1\n",
            "2787     1\n",
            "2788     1\n",
            "2789     1\n",
            "2790     1\n",
            "2791     1\n",
            "2792     1\n",
            "2793     1\n",
            "2794     1\n",
            "2795     1\n",
            "2796     1\n",
            "2797     1\n",
            "2798     1\n",
            "2799     1\n",
            "2800     1\n",
            "2801     1\n",
            "2803     1\n",
            "2804     1\n",
            "2585     1\n",
            "2517     1\n",
            "2583     1\n",
            "2248     1\n",
            "2235     1\n",
            "2236     1\n",
            "2237     1\n",
            "2238     1\n",
            "2239     1\n",
            "2240     1\n",
            "2242     1\n",
            "2243     1\n",
            "2244     1\n",
            "2245     1\n",
            "2246     1\n",
            "2247     1\n",
            "2251     1\n",
            "2265     1\n",
            "2252     1\n",
            "2253     1\n",
            "2254     1\n",
            "2255     1\n",
            "2256     1\n",
            "2257     1\n",
            "2258     1\n",
            "2259     1\n",
            "2260     1\n",
            "2261     1\n",
            "2262     1\n",
            "2263     1\n",
            "2234     1\n",
            "2233     1\n",
            "2232     1\n",
            "2231     1\n",
            "2205     1\n",
            "2206     1\n",
            "2207     1\n",
            "2208     1\n",
            "2209     1\n",
            "2211     1\n",
            "2212     1\n",
            "2213     1\n",
            "2214     1\n",
            "2215     1\n",
            "2216     1\n",
            "2217     1\n",
            "2218     1\n",
            "2219     1\n",
            "2220     1\n",
            "2221     1\n",
            "2222     1\n",
            "2223     1\n",
            "2224     1\n",
            "2225     1\n",
            "2226     1\n",
            "2227     1\n",
            "2228     1\n",
            "2229     1\n",
            "2230     1\n",
            "2264     1\n",
            "2266     1\n",
            "2331     1\n",
            "2314     1\n",
            "2302     1\n",
            "2303     1\n",
            "2304     1\n",
            "2305     1\n",
            "2306     1\n",
            "2307     1\n",
            "2308     1\n",
            "2309     1\n",
            "2310     1\n",
            "2311     1\n",
            "2312     1\n",
            "2313     1\n",
            "2315     1\n",
            "2267     1\n",
            "2316     1\n",
            "2317     1\n",
            "2318     1\n",
            "2319     1\n",
            "2320     1\n",
            "2321     1\n",
            "2322     1\n",
            "2323     1\n",
            "2325     1\n",
            "2326     1\n",
            "2328     1\n",
            "2329     1\n",
            "2301     1\n",
            "2300     1\n",
            "2297     1\n",
            "2296     1\n",
            "2268     1\n",
            "2269     1\n",
            "2270     1\n",
            "2271     1\n",
            "2272     1\n",
            "2273     1\n",
            "2274     1\n",
            "2275     1\n",
            "2276     1\n",
            "2277     1\n",
            "2278     1\n",
            "2280     1\n",
            "2281     1\n",
            "2282     1\n",
            "2283     1\n",
            "2285     1\n",
            "2286     1\n",
            "2287     1\n",
            "2288     1\n",
            "2289     1\n",
            "2290     1\n",
            "2291     1\n",
            "2293     1\n",
            "2294     1\n",
            "2295     1\n",
            "2204     1\n",
            "2203     1\n",
            "2201     1\n",
            "2117     1\n",
            "2101     1\n",
            "2102     1\n",
            "2104     1\n",
            "2105     1\n",
            "2106     1\n",
            "2108     1\n",
            "2109     1\n",
            "2110     1\n",
            "2111     1\n",
            "2112     1\n",
            "2113     1\n",
            "2116     1\n",
            "2118     1\n",
            "2200     1\n",
            "2119     1\n",
            "2120     1\n",
            "2121     1\n",
            "2122     1\n",
            "2123     1\n",
            "2124     1\n",
            "2126     1\n",
            "2127     1\n",
            "2128     1\n",
            "2129     1\n",
            "2131     1\n",
            "2132     1\n",
            "2100     1\n",
            "2099     1\n",
            "2098     1\n",
            "2096     1\n",
            "2068     1\n",
            "2069     1\n",
            "2070     1\n",
            "2071     1\n",
            "2073     1\n",
            "2074     1\n",
            "2075     1\n",
            "2076     1\n",
            "2077     1\n",
            "2078     1\n",
            "2079     1\n",
            "2080     1\n",
            "2081     1\n",
            "2082     1\n",
            "2083     1\n",
            "2084     1\n",
            "2086     1\n",
            "2087     1\n",
            "2088     1\n",
            "2090     1\n",
            "2091     1\n",
            "2092     1\n",
            "2093     1\n",
            "2094     1\n",
            "2095     1\n",
            "2133     1\n",
            "2134     1\n",
            "2136     1\n",
            "2171     1\n",
            "2173     1\n",
            "2174     1\n",
            "2175     1\n",
            "2177     1\n",
            "2178     1\n",
            "2179     1\n",
            "2180     1\n",
            "2181     1\n",
            "2182     1\n",
            "2183     1\n",
            "2184     1\n",
            "2185     1\n",
            "2186     1\n",
            "2187     1\n",
            "2188     1\n",
            "2189     1\n",
            "2190     1\n",
            "2191     1\n",
            "2192     1\n",
            "2193     1\n",
            "2194     1\n",
            "2195     1\n",
            "2196     1\n",
            "2197     1\n",
            "2199     1\n",
            "2172     1\n",
            "2169     1\n",
            "2137     1\n",
            "2167     1\n",
            "2138     1\n",
            "2140     1\n",
            "2141     1\n",
            "2142     1\n",
            "2143     1\n",
            "2144     1\n",
            "2145     1\n",
            "2146     1\n",
            "2148     1\n",
            "2149     1\n",
            "2150     1\n",
            "2151     1\n",
            "2152     1\n",
            "2153     1\n",
            "2155     1\n",
            "2156     1\n",
            "2157     1\n",
            "2158     1\n",
            "2160     1\n",
            "2161     1\n",
            "2162     1\n",
            "2163     1\n",
            "2164     1\n",
            "2165     1\n",
            "2166     1\n",
            "2330     1\n",
            "2332     1\n",
            "2582     1\n",
            "2505     1\n",
            "2493     1\n",
            "2494     1\n",
            "2495     1\n",
            "2496     1\n",
            "2497     1\n",
            "2498     1\n",
            "2499     1\n",
            "2500     1\n",
            "2501     1\n",
            "2502     1\n",
            "2503     1\n",
            "2504     1\n",
            "2506     1\n",
            "2521     1\n",
            "2507     1\n",
            "2508     1\n",
            "2509     1\n",
            "2510     1\n",
            "2511     1\n",
            "2512     1\n",
            "2514     1\n",
            "2515     1\n",
            "2516     1\n",
            "2959     1\n",
            "2518     1\n",
            "2519     1\n",
            "2492     1\n",
            "2491     1\n",
            "2490     1\n",
            "2489     1\n",
            "2464     1\n",
            "2465     1\n",
            "2466     1\n",
            "2467     1\n",
            "2468     1\n",
            "2469     1\n",
            "2470     1\n",
            "2471     1\n",
            "2472     1\n",
            "2473     1\n",
            "2474     1\n",
            "2475     1\n",
            "2476     1\n",
            "2477     1\n",
            "2478     1\n",
            "2479     1\n",
            "2480     1\n",
            "2481     1\n",
            "2482     1\n",
            "2483     1\n",
            "2484     1\n",
            "2485     1\n",
            "2486     1\n",
            "2487     1\n",
            "2488     1\n",
            "2520     1\n",
            "2522     1\n",
            "2333     1\n",
            "2567     1\n",
            "2554     1\n",
            "2555     1\n",
            "2556     1\n",
            "2558     1\n",
            "2559     1\n",
            "2560     1\n",
            "2561     1\n",
            "2562     1\n",
            "2563     1\n",
            "2564     1\n",
            "2565     1\n",
            "2566     1\n",
            "2568     1\n",
            "2523     1\n",
            "2569     1\n",
            "2570     1\n",
            "2571     1\n",
            "2572     1\n",
            "2573     1\n",
            "2575     1\n",
            "2576     1\n",
            "2577     1\n",
            "2578     1\n",
            "2579     1\n",
            "2580     1\n",
            "2581     1\n",
            "2553     1\n",
            "2552     1\n",
            "2551     1\n",
            "2549     1\n",
            "2524     1\n",
            "2525     1\n",
            "2526     1\n",
            "2527     1\n",
            "2528     1\n",
            "2529     1\n",
            "2530     1\n",
            "2531     1\n",
            "2532     1\n",
            "2533     1\n",
            "2534     1\n",
            "2535     1\n",
            "2536     1\n",
            "2537     1\n",
            "2538     1\n",
            "2539     1\n",
            "2540     1\n",
            "2541     1\n",
            "2542     1\n",
            "2543     1\n",
            "2544     1\n",
            "2545     1\n",
            "2546     1\n",
            "2547     1\n",
            "2548     1\n",
            "2463     1\n",
            "2462     1\n",
            "2461     1\n",
            "2382     1\n",
            "2368     1\n",
            "2369     1\n",
            "2370     1\n",
            "2372     1\n",
            "2373     1\n",
            "2374     1\n",
            "2375     1\n",
            "2376     1\n",
            "2377     1\n",
            "2378     1\n",
            "2380     1\n",
            "2381     1\n",
            "2383     1\n",
            "2460     1\n",
            "2384     1\n",
            "2385     1\n",
            "2386     1\n",
            "2387     1\n",
            "2388     1\n",
            "2389     1\n",
            "2390     1\n",
            "2391     1\n",
            "2392     1\n",
            "2393     1\n",
            "2394     1\n",
            "2395     1\n",
            "2367     1\n",
            "2365     1\n",
            "2364     1\n",
            "2363     1\n",
            "2334     1\n",
            "2335     1\n",
            "2336     1\n",
            "2337     1\n",
            "2338     1\n",
            "2339     1\n",
            "2340     1\n",
            "2342     1\n",
            "2343     1\n",
            "2344     1\n",
            "2345     1\n",
            "2346     1\n",
            "2347     1\n",
            "2348     1\n",
            "2349     1\n",
            "2350     1\n",
            "2351     1\n",
            "2353     1\n",
            "2354     1\n",
            "2356     1\n",
            "2357     1\n",
            "2359     1\n",
            "2360     1\n",
            "2361     1\n",
            "2362     1\n",
            "2397     1\n",
            "2399     1\n",
            "2400     1\n",
            "2432     1\n",
            "2434     1\n",
            "2435     1\n",
            "2436     1\n",
            "2437     1\n",
            "2438     1\n",
            "2439     1\n",
            "2441     1\n",
            "2442     1\n",
            "2443     1\n",
            "2444     1\n",
            "2445     1\n",
            "2446     1\n",
            "2447     1\n",
            "2448     1\n",
            "2449     1\n",
            "2450     1\n",
            "2451     1\n",
            "2452     1\n",
            "2453     1\n",
            "2454     1\n",
            "2455     1\n",
            "2456     1\n",
            "2457     1\n",
            "2458     1\n",
            "2459     1\n",
            "2433     1\n",
            "2431     1\n",
            "2401     1\n",
            "2430     1\n",
            "2403     1\n",
            "2404     1\n",
            "2405     1\n",
            "2406     1\n",
            "2407     1\n",
            "2409     1\n",
            "2410     1\n",
            "2411     1\n",
            "2412     1\n",
            "2413     1\n",
            "2414     1\n",
            "2415     1\n",
            "2416     1\n",
            "2417     1\n",
            "2418     1\n",
            "2419     1\n",
            "2420     1\n",
            "2421     1\n",
            "2422     1\n",
            "2424     1\n",
            "2425     1\n",
            "2426     1\n",
            "2427     1\n",
            "2428     1\n",
            "2429     1\n",
            "2958     1\n",
            "0        1\n",
            "2960     1\n",
            "3734     1\n",
            "3641     1\n",
            "3642     1\n",
            "3643     1\n",
            "3644     1\n",
            "3645     1\n",
            "3646     1\n",
            "3647     1\n",
            "3648     1\n",
            "3649     1\n",
            "3650     1\n",
            "3651     1\n",
            "3652     1\n",
            "3653     1\n",
            "3654     1\n",
            "3655     1\n",
            "3656     1\n",
            "3657     1\n",
            "3658     1\n",
            "3659     1\n",
            "3660     1\n",
            "3661     1\n",
            "3662     1\n",
            "3666     1\n",
            "3667     1\n",
            "3668     1\n",
            "3669     1\n",
            "3670     1\n",
            "3640     1\n",
            "3639     1\n",
            "3638     1\n",
            "3621     1\n",
            "3607     1\n",
            "3608     1\n",
            "3610     1\n",
            "3612     1\n",
            "3613     1\n",
            "3614     1\n",
            "3615     1\n",
            "3616     1\n",
            "3617     1\n",
            "3618     1\n",
            "3619     1\n",
            "3620     1\n",
            "3624     1\n",
            "3637     1\n",
            "3625     1\n",
            "3626     1\n",
            "3627     1\n",
            "3628     1\n",
            "3629     1\n",
            "3630     1\n",
            "3631     1\n",
            "3632     1\n",
            "3633     1\n",
            "3634     1\n",
            "3635     1\n",
            "3636     1\n",
            "3671     1\n",
            "3672     1\n",
            "3673     1\n",
            "3718     1\n",
            "3705     1\n",
            "3706     1\n",
            "3707     1\n",
            "3708     1\n",
            "3709     1\n",
            "3710     1\n",
            "3711     1\n",
            "3712     1\n",
            "3713     1\n",
            "3715     1\n",
            "3716     1\n",
            "3717     1\n",
            "3719     1\n",
            "3703     1\n",
            "3720     1\n",
            "3721     1\n",
            "3722     1\n",
            "3723     1\n",
            "3724     1\n",
            "3725     1\n",
            "3726     1\n",
            "3728     1\n",
            "3729     1\n",
            "3730     1\n",
            "3731     1\n",
            "3732     1\n",
            "3704     1\n",
            "3702     1\n",
            "3674     1\n",
            "3687     1\n",
            "3675     1\n",
            "3676     1\n",
            "3677     1\n",
            "3678     1\n",
            "3679     1\n",
            "3680     1\n",
            "3681     1\n",
            "3682     1\n",
            "3683     1\n",
            "3684     1\n",
            "3685     1\n",
            "3686     1\n",
            "3688     1\n",
            "3701     1\n",
            "3689     1\n",
            "3690     1\n",
            "3691     1\n",
            "3692     1\n",
            "3693     1\n",
            "3694     1\n",
            "3695     1\n",
            "3696     1\n",
            "3697     1\n",
            "3698     1\n",
            "3699     1\n",
            "3700     1\n",
            "3606     1\n",
            "3605     1\n",
            "3604     1\n",
            "3523     1\n",
            "3510     1\n",
            "3511     1\n",
            "3512     1\n",
            "3513     1\n",
            "3514     1\n",
            "3515     1\n",
            "3516     1\n",
            "3517     1\n",
            "3519     1\n",
            "3520     1\n",
            "3521     1\n",
            "3522     1\n",
            "3524     1\n",
            "3508     1\n",
            "3525     1\n",
            "3526     1\n",
            "3527     1\n",
            "3528     1\n",
            "3529     1\n",
            "3530     1\n",
            "3531     1\n",
            "3532     1\n",
            "3533     1\n",
            "3534     1\n",
            "3535     1\n",
            "3536     1\n",
            "3509     1\n",
            "3507     1\n",
            "3538     1\n",
            "3491     1\n",
            "3479     1\n",
            "3480     1\n",
            "3481     1\n",
            "3482     1\n",
            "3483     1\n",
            "3484     1\n",
            "3485     1\n",
            "3486     1\n",
            "3487     1\n",
            "3488     1\n",
            "3489     1\n",
            "3490     1\n",
            "3492     1\n",
            "3506     1\n",
            "3493     1\n",
            "3494     1\n",
            "3495     1\n",
            "3496     1\n",
            "3497     1\n",
            "3498     1\n",
            "3499     1\n",
            "3500     1\n",
            "3501     1\n",
            "3502     1\n",
            "3504     1\n",
            "3505     1\n",
            "3537     1\n",
            "3539     1\n",
            "3603     1\n",
            "3588     1\n",
            "3576     1\n",
            "3577     1\n",
            "3578     1\n",
            "3579     1\n",
            "3580     1\n",
            "3581     1\n",
            "3582     1\n",
            "3583     1\n",
            "3584     1\n",
            "3585     1\n",
            "3586     1\n",
            "3587     1\n",
            "3589     1\n",
            "3574     1\n",
            "3590     1\n",
            "3591     1\n",
            "3592     1\n",
            "3593     1\n",
            "3594     1\n",
            "3595     1\n",
            "3596     1\n",
            "3597     1\n",
            "3598     1\n",
            "3599     1\n",
            "3600     1\n",
            "3602     1\n",
            "3575     1\n",
            "3573     1\n",
            "3540     1\n",
            "3554     1\n",
            "3541     1\n",
            "3542     1\n",
            "3543     1\n",
            "3544     1\n",
            "3545     1\n",
            "3546     1\n",
            "3547     1\n",
            "3548     1\n",
            "3550     1\n",
            "3551     1\n",
            "3552     1\n",
            "3553     1\n",
            "3555     1\n",
            "3572     1\n",
            "3556     1\n",
            "3557     1\n",
            "3559     1\n",
            "3560     1\n",
            "3561     1\n",
            "3562     1\n",
            "3563     1\n",
            "3564     1\n",
            "3565     1\n",
            "3567     1\n",
            "3569     1\n",
            "3571     1\n",
            "3733     1\n",
            "3735     1\n",
            "3477     1\n",
            "3736     1\n",
            "3899     1\n",
            "3900     1\n",
            "3901     1\n",
            "3902     1\n",
            "3903     1\n",
            "3904     1\n",
            "3905     1\n",
            "3906     1\n",
            "3907     1\n",
            "3908     1\n",
            "3909     1\n",
            "3910     1\n",
            "3911     1\n",
            "3912     1\n",
            "3913     1\n",
            "3915     1\n",
            "3916     1\n",
            "3917     1\n",
            "3918     1\n",
            "3919     1\n",
            "3920     1\n",
            "3921     1\n",
            "3922     1\n",
            "3923     1\n",
            "3925     1\n",
            "3926     1\n",
            "3927     1\n",
            "3898     1\n",
            "3897     1\n",
            "3896     1\n",
            "3879     1\n",
            "3867     1\n",
            "3868     1\n",
            "3869     1\n",
            "3870     1\n",
            "3871     1\n",
            "3872     1\n",
            "3873     1\n",
            "3874     1\n",
            "3875     1\n",
            "3876     1\n",
            "3877     1\n",
            "3878     1\n",
            "3880     1\n",
            "3895     1\n",
            "3881     1\n",
            "3882     1\n",
            "3885     1\n",
            "3886     1\n",
            "3887     1\n",
            "3888     1\n",
            "3889     1\n",
            "3890     1\n",
            "3891     1\n",
            "3892     1\n",
            "3893     1\n",
            "3894     1\n",
            "3928     1\n",
            "3929     1\n",
            "3931     1\n",
            "3981     1\n",
            "3966     1\n",
            "3967     1\n",
            "3968     1\n",
            "3969     1\n",
            "3971     1\n",
            "3972     1\n",
            "3973     1\n",
            "3975     1\n",
            "3976     1\n",
            "3977     1\n",
            "3978     1\n",
            "3980     1\n",
            "3982     1\n",
            "3964     1\n",
            "3983     1\n",
            "3984     1\n",
            "3985     1\n",
            "3986     1\n",
            "3987     1\n",
            "3988     1\n",
            "3989     1\n",
            "3990     1\n",
            "3991     1\n",
            "3992     1\n",
            "3993     1\n",
            "3994     1\n",
            "3965     1\n",
            "3963     1\n",
            "3932     1\n",
            "3946     1\n",
            "3933     1\n",
            "3934     1\n",
            "3935     1\n",
            "3936     1\n",
            "3937     1\n",
            "3938     1\n",
            "3940     1\n",
            "3941     1\n",
            "3942     1\n",
            "3943     1\n",
            "3944     1\n",
            "3945     1\n",
            "3947     1\n",
            "3962     1\n",
            "3948     1\n",
            "3949     1\n",
            "3951     1\n",
            "3952     1\n",
            "3953     1\n",
            "3954     1\n",
            "3956     1\n",
            "3957     1\n",
            "3958     1\n",
            "3959     1\n",
            "3960     1\n",
            "3961     1\n",
            "3866     1\n",
            "3865     1\n",
            "3864     1\n",
            "3781     1\n",
            "3769     1\n",
            "3770     1\n",
            "3771     1\n",
            "3772     1\n",
            "3773     1\n",
            "3774     1\n",
            "3775     1\n",
            "3776     1\n",
            "3777     1\n",
            "3778     1\n",
            "3779     1\n",
            "3780     1\n",
            "3782     1\n",
            "3767     1\n",
            "3783     1\n",
            "3784     1\n",
            "3785     1\n",
            "3786     1\n",
            "3787     1\n",
            "3788     1\n",
            "3789     1\n",
            "3790     1\n",
            "3791     1\n",
            "3792     1\n",
            "3793     1\n",
            "3794     1\n",
            "3768     1\n",
            "3766     1\n",
            "3796     1\n",
            "3749     1\n",
            "3737     1\n",
            "3738     1\n",
            "3739     1\n",
            "3740     1\n",
            "3741     1\n",
            "3742     1\n",
            "3743     1\n",
            "3744     1\n",
            "3745     1\n",
            "3746     1\n",
            "3747     1\n",
            "3748     1\n",
            "3750     1\n",
            "3765     1\n",
            "3751     1\n",
            "3752     1\n",
            "3753     1\n",
            "3754     1\n",
            "3755     1\n",
            "3756     1\n",
            "3758     1\n",
            "3759     1\n",
            "3760     1\n",
            "3761     1\n",
            "3762     1\n",
            "3764     1\n",
            "3795     1\n",
            "3797     1\n",
            "3863     1\n",
            "3847     1\n",
            "3832     1\n",
            "3835     1\n",
            "3836     1\n",
            "3838     1\n",
            "3839     1\n",
            "3840     1\n",
            "3841     1\n",
            "3842     1\n",
            "3843     1\n",
            "3844     1\n",
            "3845     1\n",
            "3846     1\n",
            "3848     1\n",
            "3830     1\n",
            "3849     1\n",
            "3850     1\n",
            "3852     1\n",
            "3853     1\n",
            "3854     1\n",
            "3855     1\n",
            "3856     1\n",
            "3857     1\n",
            "3858     1\n",
            "3860     1\n",
            "3861     1\n",
            "3862     1\n",
            "3831     1\n",
            "3829     1\n",
            "3798     1\n",
            "3814     1\n",
            "3799     1\n",
            "3800     1\n",
            "3802     1\n",
            "3803     1\n",
            "3804     1\n",
            "3806     1\n",
            "3807     1\n",
            "3808     1\n",
            "3809     1\n",
            "3811     1\n",
            "3812     1\n",
            "3813     1\n",
            "3815     1\n",
            "3828     1\n",
            "3816     1\n",
            "3817     1\n",
            "3818     1\n",
            "3819     1\n",
            "3820     1\n",
            "3821     1\n",
            "3822     1\n",
            "3823     1\n",
            "3824     1\n",
            "3825     1\n",
            "3826     1\n",
            "3827     1\n",
            "3478     1\n",
            "3476     1\n",
            "2961     1\n",
            "3221     1\n",
            "3125     1\n",
            "3126     1\n",
            "3127     1\n",
            "3128     1\n",
            "3129     1\n",
            "3130     1\n",
            "3131     1\n",
            "3132     1\n",
            "3134     1\n",
            "3135     1\n",
            "3136     1\n",
            "3137     1\n",
            "3139     1\n",
            "3141     1\n",
            "3142     1\n",
            "3143     1\n",
            "3144     1\n",
            "3145     1\n",
            "3146     1\n",
            "3147     1\n",
            "3148     1\n",
            "3149     1\n",
            "3150     1\n",
            "3151     1\n",
            "3152     1\n",
            "3153     1\n",
            "3154     1\n",
            "3124     1\n",
            "3123     1\n",
            "3122     1\n",
            "3104     1\n",
            "3092     1\n",
            "3093     1\n",
            "3094     1\n",
            "3095     1\n",
            "3096     1\n",
            "3097     1\n",
            "3098     1\n",
            "3099     1\n",
            "3100     1\n",
            "3101     1\n",
            "3102     1\n",
            "3103     1\n",
            "3105     1\n",
            "3120     1\n",
            "3106     1\n",
            "3107     1\n",
            "3108     1\n",
            "3109     1\n",
            "3110     1\n",
            "3112     1\n",
            "3113     1\n",
            "3114     1\n",
            "3115     1\n",
            "3116     1\n",
            "3117     1\n",
            "3118     1\n",
            "3156     1\n",
            "3157     1\n",
            "3158     1\n",
            "3205     1\n",
            "3192     1\n",
            "3193     1\n",
            "3194     1\n",
            "3195     1\n",
            "3196     1\n",
            "3197     1\n",
            "3198     1\n",
            "3199     1\n",
            "3200     1\n",
            "3202     1\n",
            "3203     1\n",
            "3204     1\n",
            "3206     1\n",
            "3190     1\n",
            "3207     1\n",
            "3208     1\n",
            "3209     1\n",
            "3210     1\n",
            "3211     1\n",
            "3212     1\n",
            "3213     1\n",
            "3214     1\n",
            "3215     1\n",
            "3216     1\n",
            "3217     1\n",
            "3218     1\n",
            "3191     1\n",
            "3189     1\n",
            "3159     1\n",
            "3172     1\n",
            "3160     1\n",
            "3161     1\n",
            "3162     1\n",
            "3163     1\n",
            "3164     1\n",
            "3165     1\n",
            "3166     1\n",
            "3167     1\n",
            "3168     1\n",
            "3169     1\n",
            "3170     1\n",
            "3171     1\n",
            "3173     1\n",
            "3186     1\n",
            "3174     1\n",
            "3175     1\n",
            "3176     1\n",
            "3177     1\n",
            "3178     1\n",
            "3179     1\n",
            "3180     1\n",
            "3181     1\n",
            "3182     1\n",
            "3183     1\n",
            "3184     1\n",
            "3185     1\n",
            "3091     1\n",
            "3090     1\n",
            "3089     1\n",
            "3006     1\n",
            "2993     1\n",
            "2994     1\n",
            "2995     1\n",
            "2996     1\n",
            "2997     1\n",
            "2998     1\n",
            "2999     1\n",
            "3000     1\n",
            "3001     1\n",
            "3002     1\n",
            "3003     1\n",
            "3005     1\n",
            "3007     1\n",
            "2991     1\n",
            "3008     1\n",
            "3010     1\n",
            "3013     1\n",
            "3014     1\n",
            "3015     1\n",
            "3016     1\n",
            "3017     1\n",
            "3019     1\n",
            "3022     1\n",
            "3023     1\n",
            "2066     1\n",
            "3025     1\n",
            "2992     1\n",
            "2990     1\n",
            "3027     1\n",
            "2974     1\n",
            "2962     1\n",
            "2963     1\n",
            "2964     1\n",
            "2965     1\n",
            "2966     1\n",
            "2967     1\n",
            "2968     1\n",
            "2969     1\n",
            "2970     1\n",
            "2971     1\n",
            "2972     1\n",
            "2973     1\n",
            "2975     1\n",
            "2989     1\n",
            "2976     1\n",
            "2977     1\n",
            "2978     1\n",
            "2979     1\n",
            "2980     1\n",
            "2981     1\n",
            "2982     1\n",
            "2983     1\n",
            "2984     1\n",
            "2985     1\n",
            "2986     1\n",
            "2988     1\n",
            "3026     1\n",
            "3028     1\n",
            "3088     1\n",
            "3074     1\n",
            "3062     1\n",
            "3063     1\n",
            "3064     1\n",
            "3065     1\n",
            "3066     1\n",
            "3067     1\n",
            "3068     1\n",
            "3069     1\n",
            "3070     1\n",
            "3071     1\n",
            "3072     1\n",
            "3073     1\n",
            "3075     1\n",
            "3060     1\n",
            "3076     1\n",
            "3077     1\n",
            "3078     1\n",
            "3079     1\n",
            "3080     1\n",
            "3081     1\n",
            "3082     1\n",
            "3083     1\n",
            "3084     1\n",
            "3085     1\n",
            "3086     1\n",
            "3087     1\n",
            "3061     1\n",
            "3059     1\n",
            "3029     1\n",
            "3043     1\n",
            "3030     1\n",
            "3031     1\n",
            "3032     1\n",
            "3033     1\n",
            "3035     1\n",
            "3036     1\n",
            "3037     1\n",
            "3038     1\n",
            "3039     1\n",
            "3040     1\n",
            "3041     1\n",
            "3042     1\n",
            "3044     1\n",
            "3058     1\n",
            "3045     1\n",
            "3047     1\n",
            "3048     1\n",
            "3049     1\n",
            "3050     1\n",
            "3051     1\n",
            "3052     1\n",
            "3053     1\n",
            "3054     1\n",
            "3055     1\n",
            "3056     1\n",
            "3057     1\n",
            "3220     1\n",
            "3222     1\n",
            "3475     1\n",
            "3223     1\n",
            "3382     1\n",
            "3383     1\n",
            "3384     1\n",
            "3385     1\n",
            "3386     1\n",
            "3387     1\n",
            "3388     1\n",
            "3389     1\n",
            "3390     1\n",
            "3391     1\n",
            "3392     1\n",
            "3393     1\n",
            "3394     1\n",
            "3395     1\n",
            "3396     1\n",
            "3397     1\n",
            "3398     1\n",
            "3400     1\n",
            "3401     1\n",
            "3403     1\n",
            "3404     1\n",
            "3405     1\n",
            "3406     1\n",
            "3407     1\n",
            "3408     1\n",
            "3409     1\n",
            "3410     1\n",
            "3379     1\n",
            "3378     1\n",
            "3377     1\n",
            "3361     1\n",
            "3349     1\n",
            "3350     1\n",
            "3351     1\n",
            "3352     1\n",
            "3353     1\n",
            "3354     1\n",
            "3355     1\n",
            "3356     1\n",
            "3357     1\n",
            "3358     1\n",
            "3359     1\n",
            "3360     1\n",
            "3362     1\n",
            "3376     1\n",
            "3363     1\n",
            "3364     1\n",
            "3365     1\n",
            "3367     1\n",
            "3368     1\n",
            "3369     1\n",
            "3370     1\n",
            "3371     1\n",
            "3372     1\n",
            "3373     1\n",
            "3374     1\n",
            "3375     1\n",
            "3411     1\n",
            "3412     1\n",
            "3413     1\n",
            "3461     1\n",
            "3449     1\n",
            "3450     1\n",
            "3451     1\n",
            "3452     1\n",
            "3453     1\n",
            "3454     1\n",
            "3455     1\n",
            "3456     1\n",
            "3457     1\n",
            "3458     1\n",
            "3459     1\n",
            "3460     1\n",
            "3462     1\n",
            "3446     1\n",
            "3463     1\n",
            "3464     1\n",
            "3465     1\n",
            "3466     1\n",
            "3467     1\n",
            "3468     1\n",
            "3469     1\n",
            "3470     1\n",
            "3471     1\n",
            "3472     1\n",
            "3473     1\n",
            "3474     1\n",
            "3448     1\n",
            "3445     1\n",
            "3415     1\n",
            "3430     1\n",
            "3416     1\n",
            "3417     1\n",
            "3418     1\n",
            "3419     1\n",
            "3420     1\n",
            "3421     1\n",
            "3422     1\n",
            "3424     1\n",
            "3425     1\n",
            "3426     1\n",
            "3428     1\n",
            "3429     1\n",
            "3431     1\n",
            "3444     1\n",
            "3432     1\n",
            "3433     1\n",
            "3434     1\n",
            "3435     1\n",
            "3436     1\n",
            "3437     1\n",
            "3438     1\n",
            "3439     1\n",
            "3440     1\n",
            "3441     1\n",
            "3442     1\n",
            "3443     1\n",
            "3348     1\n",
            "3347     1\n",
            "3346     1\n",
            "3269     1\n",
            "3256     1\n",
            "3257     1\n",
            "3258     1\n",
            "3259     1\n",
            "3260     1\n",
            "3262     1\n",
            "3263     1\n",
            "3264     1\n",
            "3265     1\n",
            "3266     1\n",
            "3267     1\n",
            "3268     1\n",
            "3270     1\n",
            "3254     1\n",
            "3271     1\n",
            "3272     1\n",
            "3273     1\n",
            "3274     1\n",
            "3275     1\n",
            "3276     1\n",
            "3277     1\n",
            "3278     1\n",
            "3279     1\n",
            "3280     1\n",
            "3281     1\n",
            "3282     1\n",
            "3255     1\n",
            "3253     1\n",
            "3284     1\n",
            "3238     1\n",
            "3224     1\n",
            "3225     1\n",
            "3227     1\n",
            "3228     1\n",
            "3229     1\n",
            "3230     1\n",
            "3231     1\n",
            "3233     1\n",
            "3234     1\n",
            "3235     1\n",
            "3236     1\n",
            "3237     1\n",
            "3239     1\n",
            "3252     1\n",
            "3240     1\n",
            "3241     1\n",
            "3242     1\n",
            "3243     1\n",
            "3244     1\n",
            "3245     1\n",
            "3246     1\n",
            "3247     1\n",
            "3248     1\n",
            "3249     1\n",
            "3250     1\n",
            "3251     1\n",
            "3283     1\n",
            "3285     1\n",
            "3345     1\n",
            "3331     1\n",
            "3318     1\n",
            "3319     1\n",
            "3320     1\n",
            "3321     1\n",
            "3322     1\n",
            "3323     1\n",
            "3324     1\n",
            "3325     1\n",
            "3326     1\n",
            "3327     1\n",
            "3329     1\n",
            "3330     1\n",
            "3332     1\n",
            "3316     1\n",
            "3333     1\n",
            "3334     1\n",
            "3335     1\n",
            "3336     1\n",
            "3337     1\n",
            "3338     1\n",
            "3339     1\n",
            "3340     1\n",
            "3341     1\n",
            "3342     1\n",
            "3343     1\n",
            "3344     1\n",
            "3317     1\n",
            "3315     1\n",
            "3286     1\n",
            "3299     1\n",
            "3287     1\n",
            "3288     1\n",
            "3289     1\n",
            "3290     1\n",
            "3291     1\n",
            "3292     1\n",
            "3293     1\n",
            "3294     1\n",
            "3295     1\n",
            "3296     1\n",
            "3297     1\n",
            "3298     1\n",
            "3300     1\n",
            "3314     1\n",
            "3301     1\n",
            "3303     1\n",
            "3304     1\n",
            "3305     1\n",
            "3306     1\n",
            "3307     1\n",
            "3308     1\n",
            "3309     1\n",
            "3310     1\n",
            "3311     1\n",
            "3312     1\n",
            "3313     1\n",
            "2067     1\n",
            "1998     1\n",
            "2065     1\n",
            "633      1\n",
            "671      1\n",
            "672      1\n",
            "673      1\n",
            "674      1\n",
            "675      1\n",
            "676      1\n",
            "677      1\n",
            "678      1\n",
            "679      1\n",
            "680      1\n",
            "681      1\n",
            "682      1\n",
            "683      1\n",
            "684      1\n",
            "685      1\n",
            "686      1\n",
            "687      1\n",
            "688      1\n",
            "689      1\n",
            "690      1\n",
            "691      1\n",
            "692      1\n",
            "693      1\n",
            "694      1\n",
            "695      1\n",
            "696      1\n",
            "697      1\n",
            "670      1\n",
            "669      1\n",
            "668      1\n",
            "650      1\n",
            "636      1\n",
            "637      1\n",
            "638      1\n",
            "639      1\n",
            "640      1\n",
            "642      1\n",
            "643      1\n",
            "644      1\n",
            "646      1\n",
            "647      1\n",
            "648      1\n",
            "649      1\n",
            "653      1\n",
            "667      1\n",
            "654      1\n",
            "655      1\n",
            "656      1\n",
            "657      1\n",
            "658      1\n",
            "659      1\n",
            "660      1\n",
            "661      1\n",
            "662      1\n",
            "663      1\n",
            "664      1\n",
            "665      1\n",
            "698      1\n",
            "699      1\n",
            "700      1\n",
            "747      1\n",
            "735      1\n",
            "736      1\n",
            "737      1\n",
            "738      1\n",
            "739      1\n",
            "740      1\n",
            "741      1\n",
            "742      1\n",
            "743      1\n",
            "744      1\n",
            "745      1\n",
            "746      1\n",
            "748      1\n",
            "733      1\n",
            "750      1\n",
            "751      1\n",
            "753      1\n",
            "754      1\n",
            "755      1\n",
            "756      1\n",
            "757      1\n",
            "758      1\n",
            "759      1\n",
            "760      1\n",
            "762      1\n",
            "763      1\n",
            "734      1\n",
            "732      1\n",
            "702      1\n",
            "715      1\n",
            "703      1\n",
            "704      1\n",
            "705      1\n",
            "706      1\n",
            "707      1\n",
            "708      1\n",
            "709      1\n",
            "710      1\n",
            "711      1\n",
            "712      1\n",
            "713      1\n",
            "714      1\n",
            "716      1\n",
            "731      1\n",
            "717      1\n",
            "718      1\n",
            "719      1\n",
            "720      1\n",
            "721      1\n",
            "722      1\n",
            "723      1\n",
            "724      1\n",
            "725      1\n",
            "728      1\n",
            "729      1\n",
            "730      1\n",
            "635      1\n",
            "632      1\n",
            "504      1\n",
            "631      1\n",
            "539      1\n",
            "540      1\n",
            "541      1\n",
            "542      1\n",
            "543      1\n",
            "544      1\n",
            "545      1\n",
            "546      1\n",
            "548      1\n",
            "549      1\n",
            "550      1\n",
            "551      1\n",
            "552      1\n",
            "554      1\n",
            "555      1\n",
            "556      1\n",
            "557      1\n",
            "558      1\n",
            "559      1\n",
            "560      1\n",
            "561      1\n",
            "562      1\n",
            "563      1\n",
            "564      1\n",
            "565      1\n",
            "566      1\n",
            "567      1\n",
            "538      1\n",
            "537      1\n",
            "536      1\n",
            "520      1\n",
            "506      1\n",
            "508      1\n",
            "509      1\n",
            "510      1\n",
            "512      1\n",
            "513      1\n",
            "514      1\n",
            "515      1\n",
            "516      1\n",
            "517      1\n",
            "518      1\n",
            "519      1\n",
            "521      1\n",
            "535      1\n",
            "522      1\n",
            "523      1\n",
            "524      1\n",
            "525      1\n",
            "526      1\n",
            "527      1\n",
            "528      1\n",
            "529      1\n",
            "530      1\n",
            "531      1\n",
            "532      1\n",
            "534      1\n",
            "568      1\n",
            "569      1\n",
            "571      1\n",
            "617      1\n",
            "605      1\n",
            "606      1\n",
            "607      1\n",
            "608      1\n",
            "609      1\n",
            "610      1\n",
            "611      1\n",
            "612      1\n",
            "613      1\n",
            "614      1\n",
            "615      1\n",
            "616      1\n",
            "618      1\n",
            "603      1\n",
            "619      1\n",
            "620      1\n",
            "621      1\n",
            "622      1\n",
            "623      1\n",
            "624      1\n",
            "625      1\n",
            "626      1\n",
            "627      1\n",
            "628      1\n",
            "629      1\n",
            "630      1\n",
            "604      1\n",
            "602      1\n",
            "573      1\n",
            "586      1\n",
            "574      1\n",
            "575      1\n",
            "576      1\n",
            "577      1\n",
            "578      1\n",
            "579      1\n",
            "580      1\n",
            "581      1\n",
            "582      1\n",
            "583      1\n",
            "584      1\n",
            "585      1\n",
            "587      1\n",
            "601      1\n",
            "589      1\n",
            "590      1\n",
            "591      1\n",
            "592      1\n",
            "593      1\n",
            "594      1\n",
            "595      1\n",
            "596      1\n",
            "597      1\n",
            "598      1\n",
            "599      1\n",
            "600      1\n",
            "764      1\n",
            "765      1\n",
            "766      1\n",
            "767      1\n",
            "932      1\n",
            "933      1\n",
            "935      1\n",
            "937      1\n",
            "938      1\n",
            "939      1\n",
            "940      1\n",
            "941      1\n",
            "943      1\n",
            "944      1\n",
            "945      1\n",
            "946      1\n",
            "947      1\n",
            "948      1\n",
            "949      1\n",
            "950      1\n",
            "951      1\n",
            "952      1\n",
            "953      1\n",
            "954      1\n",
            "955      1\n",
            "956      1\n",
            "957      1\n",
            "958      1\n",
            "959      1\n",
            "960      1\n",
            "961      1\n",
            "931      1\n",
            "930      1\n",
            "929      1\n",
            "912      1\n",
            "900      1\n",
            "901      1\n",
            "902      1\n",
            "903      1\n",
            "904      1\n",
            "905      1\n",
            "906      1\n",
            "907      1\n",
            "908      1\n",
            "909      1\n",
            "910      1\n",
            "911      1\n",
            "913      1\n",
            "927      1\n",
            "914      1\n",
            "915      1\n",
            "916      1\n",
            "917      1\n",
            "918      1\n",
            "919      1\n",
            "920      1\n",
            "921      1\n",
            "922      1\n",
            "924      1\n",
            "925      1\n",
            "926      1\n",
            "962      1\n",
            "963      1\n",
            "964      1\n",
            "1010     1\n",
            "997      1\n",
            "998      1\n",
            "999      1\n",
            "1000     1\n",
            "1001     1\n",
            "1002     1\n",
            "1003     1\n",
            "1004     1\n",
            "1005     1\n",
            "1006     1\n",
            "1008     1\n",
            "1009     1\n",
            "1011     1\n",
            "995      1\n",
            "1012     1\n",
            "1013     1\n",
            "1014     1\n",
            "1015     1\n",
            "1016     1\n",
            "1017     1\n",
            "1018     1\n",
            "1019     1\n",
            "1020     1\n",
            "1021     1\n",
            "1022     1\n",
            "1023     1\n",
            "996      1\n",
            "994      1\n",
            "965      1\n",
            "979      1\n",
            "966      1\n",
            "967      1\n",
            "968      1\n",
            "969      1\n",
            "970      1\n",
            "972      1\n",
            "973      1\n",
            "974      1\n",
            "975      1\n",
            "976      1\n",
            "977      1\n",
            "978      1\n",
            "980      1\n",
            "993      1\n",
            "981      1\n",
            "982      1\n",
            "983      1\n",
            "984      1\n",
            "985      1\n",
            "986      1\n",
            "987      1\n",
            "988      1\n",
            "989      1\n",
            "990      1\n",
            "991      1\n",
            "992      1\n",
            "899      1\n",
            "898      1\n",
            "897      1\n",
            "816      1\n",
            "803      1\n",
            "804      1\n",
            "805      1\n",
            "806      1\n",
            "808      1\n",
            "809      1\n",
            "810      1\n",
            "811      1\n",
            "812      1\n",
            "813      1\n",
            "814      1\n",
            "815      1\n",
            "817      1\n",
            "801      1\n",
            "818      1\n",
            "819      1\n",
            "820      1\n",
            "821      1\n",
            "822      1\n",
            "823      1\n",
            "824      1\n",
            "825      1\n",
            "828      1\n",
            "829      1\n",
            "830      1\n",
            "831      1\n",
            "802      1\n",
            "800      1\n",
            "833      1\n",
            "783      1\n",
            "769      1\n",
            "770      1\n",
            "772      1\n",
            "773      1\n",
            "774      1\n",
            "775      1\n",
            "777      1\n",
            "778      1\n",
            "779      1\n",
            "780      1\n",
            "781      1\n",
            "782      1\n",
            "784      1\n",
            "799      1\n",
            "785      1\n",
            "786      1\n",
            "787      1\n",
            "788      1\n",
            "790      1\n",
            "791      1\n",
            "792      1\n",
            "793      1\n",
            "795      1\n",
            "796      1\n",
            "797      1\n",
            "798      1\n",
            "832      1\n",
            "834      1\n",
            "896      1\n",
            "882      1\n",
            "870      1\n",
            "871      1\n",
            "872      1\n",
            "873      1\n",
            "874      1\n",
            "875      1\n",
            "876      1\n",
            "877      1\n",
            "878      1\n",
            "879      1\n",
            "880      1\n",
            "881      1\n",
            "883      1\n",
            "868      1\n",
            "884      1\n",
            "885      1\n",
            "886      1\n",
            "887      1\n",
            "888      1\n",
            "889      1\n",
            "890      1\n",
            "891      1\n",
            "892      1\n",
            "893      1\n",
            "894      1\n",
            "895      1\n",
            "869      1\n",
            "867      1\n",
            "835      1\n",
            "849      1\n",
            "837      1\n",
            "838      1\n",
            "839      1\n",
            "840      1\n",
            "841      1\n",
            "842      1\n",
            "843      1\n",
            "844      1\n",
            "845      1\n",
            "846      1\n",
            "847      1\n",
            "848      1\n",
            "850      1\n",
            "866      1\n",
            "851      1\n",
            "852      1\n",
            "853      1\n",
            "856      1\n",
            "857      1\n",
            "858      1\n",
            "859      1\n",
            "860      1\n",
            "862      1\n",
            "863      1\n",
            "864      1\n",
            "865      1\n",
            "505      1\n",
            "503      1\n",
            "2063     1\n",
            "127      1\n",
            "160      1\n",
            "161      1\n",
            "162      1\n",
            "163      1\n",
            "164      1\n",
            "165      1\n",
            "166      1\n",
            "167      1\n",
            "168      1\n",
            "169      1\n",
            "170      1\n",
            "171      1\n",
            "172      1\n",
            "174      1\n",
            "175      1\n",
            "176      1\n",
            "177      1\n",
            "178      1\n",
            "179      1\n",
            "180      1\n",
            "181      1\n",
            "182      1\n",
            "183      1\n",
            "184      1\n",
            "185      1\n",
            "186      1\n",
            "187      1\n",
            "159      1\n",
            "158      1\n",
            "157      1\n",
            "141      1\n",
            "129      1\n",
            "130      1\n",
            "131      1\n",
            "132      1\n",
            "133      1\n",
            "134      1\n",
            "135      1\n",
            "136      1\n",
            "137      1\n",
            "138      1\n",
            "139      1\n",
            "140      1\n",
            "142      1\n",
            "156      1\n",
            "144      1\n",
            "145      1\n",
            "146      1\n",
            "147      1\n",
            "148      1\n",
            "149      1\n",
            "150      1\n",
            "151      1\n",
            "152      1\n",
            "153      1\n",
            "154      1\n",
            "155      1\n",
            "188      1\n",
            "189      1\n",
            "190      1\n",
            "238      1\n",
            "225      1\n",
            "226      1\n",
            "227      1\n",
            "228      1\n",
            "229      1\n",
            "230      1\n",
            "231      1\n",
            "232      1\n",
            "233      1\n",
            "234      1\n",
            "236      1\n",
            "237      1\n",
            "239      1\n",
            "222      1\n",
            "240      1\n",
            "241      1\n",
            "242      1\n",
            "243      1\n",
            "244      1\n",
            "245      1\n",
            "246      1\n",
            "247      1\n",
            "248      1\n",
            "249      1\n",
            "250      1\n",
            "251      1\n",
            "223      1\n",
            "221      1\n",
            "191      1\n",
            "205      1\n",
            "192      1\n",
            "193      1\n",
            "194      1\n",
            "195      1\n",
            "196      1\n",
            "198      1\n",
            "199      1\n",
            "200      1\n",
            "201      1\n",
            "202      1\n",
            "203      1\n",
            "204      1\n",
            "206      1\n",
            "220      1\n",
            "207      1\n",
            "208      1\n",
            "209      1\n",
            "211      1\n",
            "212      1\n",
            "213      1\n",
            "214      1\n",
            "215      1\n",
            "216      1\n",
            "217      1\n",
            "218      1\n",
            "219      1\n",
            "128      1\n",
            "126      1\n",
            "502      1\n",
            "125      1\n",
            "34       1\n",
            "35       1\n",
            "36       1\n",
            "37       1\n",
            "38       1\n",
            "39       1\n",
            "40       1\n",
            "41       1\n",
            "42       1\n",
            "43       1\n",
            "44       1\n",
            "45       1\n",
            "46       1\n",
            "47       1\n",
            "48       1\n",
            "49       1\n",
            "50       1\n",
            "51       1\n",
            "53       1\n",
            "54       1\n",
            "55       1\n",
            "56       1\n",
            "57       1\n",
            "58       1\n",
            "59       1\n",
            "60       1\n",
            "61       1\n",
            "33       1\n",
            "32       1\n",
            "31       1\n",
            "16       1\n",
            "2        1\n",
            "3        1\n",
            "4        1\n",
            "5        1\n",
            "6        1\n",
            "7        1\n",
            "8        1\n",
            "9        1\n",
            "12       1\n",
            "13       1\n",
            "14       1\n",
            "15       1\n",
            "17       1\n",
            "30       1\n",
            "18       1\n",
            "19       1\n",
            "20       1\n",
            "21       1\n",
            "22       1\n",
            "23       1\n",
            "24       1\n",
            "25       1\n",
            "26       1\n",
            "27       1\n",
            "28       1\n",
            "29       1\n",
            "62       1\n",
            "63       1\n",
            "64       1\n",
            "111      1\n",
            "99       1\n",
            "100      1\n",
            "101      1\n",
            "102      1\n",
            "103      1\n",
            "104      1\n",
            "105      1\n",
            "106      1\n",
            "107      1\n",
            "108      1\n",
            "109      1\n",
            "110      1\n",
            "112      1\n",
            "97       1\n",
            "113      1\n",
            "114      1\n",
            "115      1\n",
            "116      1\n",
            "117      1\n",
            "118      1\n",
            "119      1\n",
            "120      1\n",
            "121      1\n",
            "122      1\n",
            "123      1\n",
            "124      1\n",
            "98       1\n",
            "95       1\n",
            "65       1\n",
            "79       1\n",
            "66       1\n",
            "67       1\n",
            "68       1\n",
            "69       1\n",
            "70       1\n",
            "71       1\n",
            "72       1\n",
            "73       1\n",
            "74       1\n",
            "75       1\n",
            "77       1\n",
            "78       1\n",
            "80       1\n",
            "94       1\n",
            "81       1\n",
            "82       1\n",
            "83       1\n",
            "84       1\n",
            "85       1\n",
            "87       1\n",
            "88       1\n",
            "89       1\n",
            "90       1\n",
            "91       1\n",
            "92       1\n",
            "93       1\n",
            "252      1\n",
            "253      1\n",
            "255      1\n",
            "256      1\n",
            "410      1\n",
            "412      1\n",
            "413      1\n",
            "414      1\n",
            "415      1\n",
            "416      1\n",
            "417      1\n",
            "418      1\n",
            "419      1\n",
            "420      1\n",
            "421      1\n",
            "422      1\n",
            "423      1\n",
            "424      1\n",
            "425      1\n",
            "426      1\n",
            "427      1\n",
            "428      1\n",
            "429      1\n",
            "431      1\n",
            "432      1\n",
            "433      1\n",
            "434      1\n",
            "435      1\n",
            "436      1\n",
            "437      1\n",
            "438      1\n",
            "409      1\n",
            "408      1\n",
            "407      1\n",
            "392      1\n",
            "380      1\n",
            "381      1\n",
            "382      1\n",
            "383      1\n",
            "384      1\n",
            "385      1\n",
            "386      1\n",
            "387      1\n",
            "388      1\n",
            "389      1\n",
            "390      1\n",
            "391      1\n",
            "393      1\n",
            "406      1\n",
            "394      1\n",
            "395      1\n",
            "396      1\n",
            "397      1\n",
            "398      1\n",
            "399      1\n",
            "400      1\n",
            "401      1\n",
            "402      1\n",
            "403      1\n",
            "404      1\n",
            "405      1\n",
            "439      1\n",
            "440      1\n",
            "441      1\n",
            "487      1\n",
            "475      1\n",
            "476      1\n",
            "477      1\n",
            "478      1\n",
            "479      1\n",
            "480      1\n",
            "481      1\n",
            "482      1\n",
            "483      1\n",
            "484      1\n",
            "485      1\n",
            "486      1\n",
            "489      1\n",
            "473      1\n",
            "490      1\n",
            "491      1\n",
            "492      1\n",
            "493      1\n",
            "494      1\n",
            "495      1\n",
            "496      1\n",
            "497      1\n",
            "498      1\n",
            "499      1\n",
            "500      1\n",
            "501      1\n",
            "474      1\n",
            "472      1\n",
            "442      1\n",
            "457      1\n",
            "443      1\n",
            "444      1\n",
            "445      1\n",
            "446      1\n",
            "447      1\n",
            "448      1\n",
            "450      1\n",
            "451      1\n",
            "452      1\n",
            "453      1\n",
            "455      1\n",
            "456      1\n",
            "458      1\n",
            "471      1\n",
            "459      1\n",
            "460      1\n",
            "461      1\n",
            "462      1\n",
            "463      1\n",
            "464      1\n",
            "465      1\n",
            "466      1\n",
            "467      1\n",
            "468      1\n",
            "469      1\n",
            "470      1\n",
            "379      1\n",
            "378      1\n",
            "377      1\n",
            "300      1\n",
            "288      1\n",
            "289      1\n",
            "290      1\n",
            "291      1\n",
            "292      1\n",
            "293      1\n",
            "294      1\n",
            "295      1\n",
            "296      1\n",
            "297      1\n",
            "298      1\n",
            "299      1\n",
            "301      1\n",
            "286      1\n",
            "302      1\n",
            "303      1\n",
            "304      1\n",
            "305      1\n",
            "306      1\n",
            "307      1\n",
            "308      1\n",
            "309      1\n",
            "310      1\n",
            "311      1\n",
            "312      1\n",
            "313      1\n",
            "287      1\n",
            "285      1\n",
            "315      1\n",
            "270      1\n",
            "257      1\n",
            "258      1\n",
            "259      1\n",
            "260      1\n",
            "261      1\n",
            "262      1\n",
            "263      1\n",
            "264      1\n",
            "265      1\n",
            "266      1\n",
            "267      1\n",
            "268      1\n",
            "271      1\n",
            "284      1\n",
            "272      1\n",
            "273      1\n",
            "274      1\n",
            "275      1\n",
            "276      1\n",
            "277      1\n",
            "278      1\n",
            "279      1\n",
            "280      1\n",
            "281      1\n",
            "282      1\n",
            "283      1\n",
            "314      1\n",
            "316      1\n",
            "376      1\n",
            "361      1\n",
            "348      1\n",
            "349      1\n",
            "350      1\n",
            "351      1\n",
            "352      1\n",
            "353      1\n",
            "355      1\n",
            "356      1\n",
            "357      1\n",
            "358      1\n",
            "359      1\n",
            "360      1\n",
            "362      1\n",
            "346      1\n",
            "363      1\n",
            "364      1\n",
            "365      1\n",
            "366      1\n",
            "367      1\n",
            "368      1\n",
            "369      1\n",
            "370      1\n",
            "371      1\n",
            "372      1\n",
            "373      1\n",
            "375      1\n",
            "347      1\n",
            "345      1\n",
            "317      1\n",
            "330      1\n",
            "318      1\n",
            "319      1\n",
            "320      1\n",
            "321      1\n",
            "322      1\n",
            "323      1\n",
            "324      1\n",
            "325      1\n",
            "326      1\n",
            "327      1\n",
            "328      1\n",
            "329      1\n",
            "331      1\n",
            "344      1\n",
            "332      1\n",
            "333      1\n",
            "334      1\n",
            "335      1\n",
            "336      1\n",
            "337      1\n",
            "338      1\n",
            "339      1\n",
            "340      1\n",
            "341      1\n",
            "342      1\n",
            "343      1\n",
            "1024     1\n",
            "1025     1\n",
            "1026     1\n",
            "1679     1\n",
            "1712     1\n",
            "1713     1\n",
            "1714     1\n",
            "1715     1\n",
            "1716     1\n",
            "1717     1\n",
            "1718     1\n",
            "1719     1\n",
            "1720     1\n",
            "1721     1\n",
            "1722     1\n",
            "1723     1\n",
            "1724     1\n",
            "1725     1\n",
            "1726     1\n",
            "1728     1\n",
            "1729     1\n",
            "1730     1\n",
            "1733     1\n",
            "1734     1\n",
            "1735     1\n",
            "1736     1\n",
            "1737     1\n",
            "1738     1\n",
            "1739     1\n",
            "1741     1\n",
            "1742     1\n",
            "1711     1\n",
            "1710     1\n",
            "1708     1\n",
            "1693     1\n",
            "1681     1\n",
            "1682     1\n",
            "1683     1\n",
            "1684     1\n",
            "1685     1\n",
            "1686     1\n",
            "1687     1\n",
            "1688     1\n",
            "1689     1\n",
            "1690     1\n",
            "1691     1\n",
            "1692     1\n",
            "1694     1\n",
            "1707     1\n",
            "1695     1\n",
            "1696     1\n",
            "1697     1\n",
            "1698     1\n",
            "1699     1\n",
            "1700     1\n",
            "1701     1\n",
            "1702     1\n",
            "1703     1\n",
            "1704     1\n",
            "1705     1\n",
            "1706     1\n",
            "1743     1\n",
            "1745     1\n",
            "1746     1\n",
            "1793     1\n",
            "1779     1\n",
            "1780     1\n",
            "1782     1\n",
            "1783     1\n",
            "1784     1\n",
            "1786     1\n",
            "1787     1\n",
            "1788     1\n",
            "1789     1\n",
            "1790     1\n",
            "1791     1\n",
            "1792     1\n",
            "1794     1\n",
            "1777     1\n",
            "1795     1\n",
            "1796     1\n",
            "1797     1\n",
            "1798     1\n",
            "1799     1\n",
            "1800     1\n",
            "1801     1\n",
            "1802     1\n",
            "1803     1\n",
            "1804     1\n",
            "1805     1\n",
            "1806     1\n",
            "1778     1\n",
            "1776     1\n",
            "1747     1\n",
            "1760     1\n",
            "1748     1\n",
            "1749     1\n",
            "1750     1\n",
            "1751     1\n",
            "1752     1\n",
            "1753     1\n",
            "1754     1\n",
            "1755     1\n",
            "1756     1\n",
            "1757     1\n",
            "1758     1\n",
            "1759     1\n",
            "1761     1\n",
            "1775     1\n",
            "1762     1\n",
            "1763     1\n",
            "1765     1\n",
            "1766     1\n",
            "1767     1\n",
            "1768     1\n",
            "1769     1\n",
            "1770     1\n",
            "1771     1\n",
            "1772     1\n",
            "1773     1\n",
            "1774     1\n",
            "1680     1\n",
            "1678     1\n",
            "1027     1\n",
            "1677     1\n",
            "1579     1\n",
            "1580     1\n",
            "1581     1\n",
            "1582     1\n",
            "1583     1\n",
            "1584     1\n",
            "1586     1\n",
            "1588     1\n",
            "1590     1\n",
            "1591     1\n",
            "1592     1\n",
            "1593     1\n",
            "1595     1\n",
            "1596     1\n",
            "1597     1\n",
            "1598     1\n",
            "1599     1\n",
            "1600     1\n",
            "1601     1\n",
            "1602     1\n",
            "1603     1\n",
            "1604     1\n",
            "1606     1\n",
            "1608     1\n",
            "1609     1\n",
            "1611     1\n",
            "1612     1\n",
            "1578     1\n",
            "1577     1\n",
            "1576     1\n",
            "1561     1\n",
            "1549     1\n",
            "1550     1\n",
            "1551     1\n",
            "1552     1\n",
            "1553     1\n",
            "1554     1\n",
            "1555     1\n",
            "1556     1\n",
            "1557     1\n",
            "1558     1\n",
            "1559     1\n",
            "1560     1\n",
            "1562     1\n",
            "1575     1\n",
            "1563     1\n",
            "1564     1\n",
            "1565     1\n",
            "1566     1\n",
            "1567     1\n",
            "1568     1\n",
            "1569     1\n",
            "1570     1\n",
            "1571     1\n",
            "1572     1\n",
            "1573     1\n",
            "1574     1\n",
            "1613     1\n",
            "1614     1\n",
            "1615     1\n",
            "1661     1\n",
            "1649     1\n",
            "1650     1\n",
            "1651     1\n",
            "1652     1\n",
            "1653     1\n",
            "1654     1\n",
            "1655     1\n",
            "1656     1\n",
            "1657     1\n",
            "1658     1\n",
            "1659     1\n",
            "1660     1\n",
            "1662     1\n",
            "1647     1\n",
            "1663     1\n",
            "1664     1\n",
            "1665     1\n",
            "1666     1\n",
            "1669     1\n",
            "1670     1\n",
            "1671     1\n",
            "1672     1\n",
            "1673     1\n",
            "1674     1\n",
            "1675     1\n",
            "1676     1\n",
            "1648     1\n",
            "1646     1\n",
            "1616     1\n",
            "1630     1\n",
            "1617     1\n",
            "1618     1\n",
            "1619     1\n",
            "1620     1\n",
            "1621     1\n",
            "1622     1\n",
            "1623     1\n",
            "1625     1\n",
            "1626     1\n",
            "1627     1\n",
            "1628     1\n",
            "1629     1\n",
            "1631     1\n",
            "1645     1\n",
            "1632     1\n",
            "1633     1\n",
            "1634     1\n",
            "1635     1\n",
            "1636     1\n",
            "1638     1\n",
            "1639     1\n",
            "1640     1\n",
            "1641     1\n",
            "1642     1\n",
            "1643     1\n",
            "1644     1\n",
            "1807     1\n",
            "1808     1\n",
            "1810     1\n",
            "1811     1\n",
            "1970     1\n",
            "1971     1\n",
            "1972     1\n",
            "1973     1\n",
            "1974     1\n",
            "1975     1\n",
            "1976     1\n",
            "1977     1\n",
            "1978     1\n",
            "1979     1\n",
            "1980     1\n",
            "1981     1\n",
            "1982     1\n",
            "1983     1\n",
            "1984     1\n",
            "1986     1\n",
            "1987     1\n",
            "1988     1\n",
            "1989     1\n",
            "1990     1\n",
            "1991     1\n",
            "1993     1\n",
            "1994     1\n",
            "1995     1\n",
            "1996     1\n",
            "1997     1\n",
            "1        1\n",
            "1969     1\n",
            "1968     1\n",
            "1967     1\n",
            "1951     1\n",
            "1938     1\n",
            "1939     1\n",
            "1940     1\n",
            "1941     1\n",
            "1942     1\n",
            "1943     1\n",
            "1944     1\n",
            "1945     1\n",
            "1947     1\n",
            "1948     1\n",
            "1949     1\n",
            "1950     1\n",
            "1952     1\n",
            "1966     1\n",
            "1953     1\n",
            "1954     1\n",
            "1956     1\n",
            "1957     1\n",
            "1958     1\n",
            "1959     1\n",
            "1960     1\n",
            "1961     1\n",
            "1962     1\n",
            "1963     1\n",
            "1964     1\n",
            "1965     1\n",
            "1999     1\n",
            "2000     1\n",
            "2001     1\n",
            "2048     1\n",
            "2036     1\n",
            "2037     1\n",
            "2038     1\n",
            "2039     1\n",
            "2040     1\n",
            "2041     1\n",
            "2042     1\n",
            "2043     1\n",
            "2044     1\n",
            "2045     1\n",
            "2046     1\n",
            "2047     1\n",
            "2049     1\n",
            "2034     1\n",
            "2050     1\n",
            "2051     1\n",
            "2052     1\n",
            "2053     1\n",
            "2054     1\n",
            "2055     1\n",
            "2057     1\n",
            "2058     1\n",
            "2059     1\n",
            "2060     1\n",
            "2061     1\n",
            "2062     1\n",
            "2035     1\n",
            "2033     1\n",
            "2002     1\n",
            "2016     1\n",
            "2003     1\n",
            "2004     1\n",
            "2005     1\n",
            "2006     1\n",
            "2008     1\n",
            "2009     1\n",
            "2010     1\n",
            "2011     1\n",
            "2012     1\n",
            "2013     1\n",
            "2014     1\n",
            "2015     1\n",
            "2018     1\n",
            "2032     1\n",
            "2020     1\n",
            "2021     1\n",
            "2022     1\n",
            "2023     1\n",
            "2024     1\n",
            "2025     1\n",
            "2026     1\n",
            "2027     1\n",
            "2028     1\n",
            "2029     1\n",
            "2030     1\n",
            "2031     1\n",
            "1937     1\n",
            "1936     1\n",
            "1935     1\n",
            "1857     1\n",
            "1843     1\n",
            "1844     1\n",
            "1845     1\n",
            "1846     1\n",
            "1848     1\n",
            "1849     1\n",
            "1850     1\n",
            "1851     1\n",
            "1852     1\n",
            "1853     1\n",
            "1854     1\n",
            "1856     1\n",
            "1858     1\n",
            "1841     1\n",
            "1859     1\n",
            "1860     1\n",
            "1861     1\n",
            "1862     1\n",
            "1863     1\n",
            "1864     1\n",
            "1865     1\n",
            "1866     1\n",
            "1867     1\n",
            "1868     1\n",
            "1869     1\n",
            "1870     1\n",
            "1842     1\n",
            "1840     1\n",
            "1872     1\n",
            "1824     1\n",
            "1812     1\n",
            "1813     1\n",
            "1814     1\n",
            "1815     1\n",
            "1816     1\n",
            "1817     1\n",
            "1818     1\n",
            "1819     1\n",
            "1820     1\n",
            "1821     1\n",
            "1822     1\n",
            "1823     1\n",
            "1825     1\n",
            "1839     1\n",
            "1826     1\n",
            "1827     1\n",
            "1829     1\n",
            "1830     1\n",
            "1831     1\n",
            "1832     1\n",
            "1833     1\n",
            "1834     1\n",
            "1835     1\n",
            "1836     1\n",
            "1837     1\n",
            "1838     1\n",
            "1871     1\n",
            "1873     1\n",
            "1934     1\n",
            "1919     1\n",
            "1907     1\n",
            "1908     1\n",
            "1909     1\n",
            "1910     1\n",
            "1911     1\n",
            "1912     1\n",
            "1913     1\n",
            "1914     1\n",
            "1915     1\n",
            "1916     1\n",
            "1917     1\n",
            "1918     1\n",
            "1920     1\n",
            "1905     1\n",
            "1921     1\n",
            "1922     1\n",
            "1923     1\n",
            "1924     1\n",
            "1925     1\n",
            "1926     1\n",
            "1927     1\n",
            "1928     1\n",
            "1929     1\n",
            "1931     1\n",
            "1932     1\n",
            "1933     1\n",
            "1906     1\n",
            "1903     1\n",
            "1874     1\n",
            "1887     1\n",
            "1875     1\n",
            "1876     1\n",
            "1877     1\n",
            "1878     1\n",
            "1879     1\n",
            "1880     1\n",
            "1881     1\n",
            "1882     1\n",
            "1883     1\n",
            "1884     1\n",
            "1885     1\n",
            "1886     1\n",
            "1888     1\n",
            "1901     1\n",
            "1889     1\n",
            "1890     1\n",
            "1891     1\n",
            "1892     1\n",
            "1893     1\n",
            "1894     1\n",
            "1895     1\n",
            "1896     1\n",
            "1897     1\n",
            "1898     1\n",
            "1899     1\n",
            "1900     1\n",
            "1547     1\n",
            "1546     1\n",
            "1544     1\n",
            "1283     1\n",
            "1189     1\n",
            "1191     1\n",
            "1192     1\n",
            "1193     1\n",
            "1194     1\n",
            "1195     1\n",
            "1196     1\n",
            "1197     1\n",
            "1198     1\n",
            "1199     1\n",
            "1200     1\n",
            "1201     1\n",
            "1202     1\n",
            "1203     1\n",
            "1204     1\n",
            "1205     1\n",
            "1206     1\n",
            "1208     1\n",
            "1209     1\n",
            "1210     1\n",
            "1211     1\n",
            "1212     1\n",
            "1213     1\n",
            "1214     1\n",
            "1215     1\n",
            "1216     1\n",
            "1217     1\n",
            "1188     1\n",
            "1187     1\n",
            "1186     1\n",
            "1170     1\n",
            "1157     1\n",
            "1158     1\n",
            "1159     1\n",
            "1160     1\n",
            "1161     1\n",
            "1162     1\n",
            "1163     1\n",
            "1164     1\n",
            "1165     1\n",
            "1166     1\n",
            "1167     1\n",
            "1168     1\n",
            "1171     1\n",
            "1185     1\n",
            "1172     1\n",
            "1173     1\n",
            "1174     1\n",
            "1175     1\n",
            "1176     1\n",
            "1177     1\n",
            "1178     1\n",
            "1179     1\n",
            "1180     1\n",
            "1181     1\n",
            "1182     1\n",
            "1183     1\n",
            "1218     1\n",
            "1219     1\n",
            "1220     1\n",
            "1268     1\n",
            "1255     1\n",
            "1257     1\n",
            "1258     1\n",
            "1259     1\n",
            "1260     1\n",
            "1261     1\n",
            "1262     1\n",
            "1263     1\n",
            "1264     1\n",
            "1265     1\n",
            "1266     1\n",
            "1267     1\n",
            "1269     1\n",
            "1253     1\n",
            "1270     1\n",
            "1271     1\n",
            "1272     1\n",
            "1273     1\n",
            "1274     1\n",
            "1275     1\n",
            "1276     1\n",
            "1277     1\n",
            "1278     1\n",
            "1279     1\n",
            "1280     1\n",
            "1281     1\n",
            "1254     1\n",
            "1251     1\n",
            "1222     1\n",
            "1236     1\n",
            "1224     1\n",
            "1225     1\n",
            "1226     1\n",
            "1227     1\n",
            "1228     1\n",
            "1229     1\n",
            "1230     1\n",
            "1231     1\n",
            "1232     1\n",
            "1233     1\n",
            "1234     1\n",
            "1235     1\n",
            "1237     1\n",
            "1250     1\n",
            "1238     1\n",
            "1239     1\n",
            "1240     1\n",
            "1241     1\n",
            "1242     1\n",
            "1243     1\n",
            "1244     1\n",
            "1245     1\n",
            "1246     1\n",
            "1247     1\n",
            "1248     1\n",
            "1249     1\n",
            "1156     1\n",
            "1155     1\n",
            "1154     1\n",
            "1075     1\n",
            "1061     1\n",
            "1062     1\n",
            "1063     1\n",
            "1064     1\n",
            "1065     1\n",
            "1066     1\n",
            "1067     1\n",
            "1068     1\n",
            "1070     1\n",
            "1071     1\n",
            "1072     1\n",
            "1073     1\n",
            "1076     1\n",
            "1059     1\n",
            "1077     1\n",
            "1078     1\n",
            "1079     1\n",
            "1080     1\n",
            "1081     1\n",
            "1082     1\n",
            "1083     1\n",
            "1084     1\n",
            "1085     1\n",
            "1086     1\n",
            "1087     1\n",
            "1088     1\n",
            "1060     1\n",
            "1058     1\n",
            "1090     1\n",
            "1041     1\n",
            "1028     1\n",
            "1029     1\n",
            "1030     1\n",
            "1031     1\n",
            "1032     1\n",
            "1033     1\n",
            "1034     1\n",
            "1036     1\n",
            "1037     1\n",
            "1038     1\n",
            "1039     1\n",
            "1040     1\n",
            "1042     1\n",
            "1056     1\n",
            "1043     1\n",
            "1044     1\n",
            "1045     1\n",
            "1046     1\n",
            "1047     1\n",
            "1049     1\n",
            "1050     1\n",
            "1051     1\n",
            "1052     1\n",
            "1053     1\n",
            "1054     1\n",
            "1055     1\n",
            "1089     1\n",
            "1091     1\n",
            "1153     1\n",
            "1137     1\n",
            "1124     1\n",
            "1125     1\n",
            "1126     1\n",
            "1127     1\n",
            "1128     1\n",
            "1129     1\n",
            "1130     1\n",
            "1131     1\n",
            "1132     1\n",
            "1133     1\n",
            "1135     1\n",
            "1136     1\n",
            "1138     1\n",
            "1122     1\n",
            "1139     1\n",
            "1140     1\n",
            "1141     1\n",
            "1143     1\n",
            "1144     1\n",
            "1145     1\n",
            "1146     1\n",
            "1147     1\n",
            "1148     1\n",
            "1150     1\n",
            "1151     1\n",
            "1152     1\n",
            "1123     1\n",
            "1121     1\n",
            "1092     1\n",
            "1105     1\n",
            "1093     1\n",
            "1094     1\n",
            "1095     1\n",
            "1096     1\n",
            "1097     1\n",
            "1098     1\n",
            "1099     1\n",
            "1100     1\n",
            "1101     1\n",
            "1102     1\n",
            "1103     1\n",
            "1104     1\n",
            "1106     1\n",
            "1120     1\n",
            "1107     1\n",
            "1108     1\n",
            "1109     1\n",
            "1111     1\n",
            "1112     1\n",
            "1113     1\n",
            "1114     1\n",
            "1115     1\n",
            "1116     1\n",
            "1117     1\n",
            "1118     1\n",
            "1119     1\n",
            "1282     1\n",
            "1284     1\n",
            "1543     1\n",
            "1285     1\n",
            "1450     1\n",
            "1451     1\n",
            "1452     1\n",
            "1453     1\n",
            "1454     1\n",
            "1455     1\n",
            "1456     1\n",
            "1457     1\n",
            "1458     1\n",
            "1459     1\n",
            "1460     1\n",
            "1461     1\n",
            "1462     1\n",
            "1463     1\n",
            "1464     1\n",
            "1465     1\n",
            "1466     1\n",
            "1467     1\n",
            "1468     1\n",
            "1469     1\n",
            "1470     1\n",
            "1471     1\n",
            "1472     1\n",
            "1473     1\n",
            "1475     1\n",
            "1476     1\n",
            "1477     1\n",
            "1449     1\n",
            "1448     1\n",
            "1447     1\n",
            "1430     1\n",
            "1416     1\n",
            "1417     1\n",
            "1418     1\n",
            "1420     1\n",
            "1421     1\n",
            "1422     1\n",
            "1423     1\n",
            "1424     1\n",
            "1425     1\n",
            "1426     1\n",
            "1427     1\n",
            "1429     1\n",
            "1432     1\n",
            "1446     1\n",
            "1433     1\n",
            "1434     1\n",
            "1435     1\n",
            "1436     1\n",
            "1437     1\n",
            "1438     1\n",
            "1439     1\n",
            "1440     1\n",
            "1441     1\n",
            "1442     1\n",
            "1443     1\n",
            "1444     1\n",
            "1478     1\n",
            "1479     1\n",
            "1480     1\n",
            "1528     1\n",
            "1515     1\n",
            "1516     1\n",
            "1517     1\n",
            "1518     1\n",
            "1519     1\n",
            "1520     1\n",
            "1521     1\n",
            "1523     1\n",
            "1524     1\n",
            "1525     1\n",
            "1526     1\n",
            "1527     1\n",
            "1529     1\n",
            "1513     1\n",
            "1530     1\n",
            "1531     1\n",
            "1532     1\n",
            "1533     1\n",
            "1534     1\n",
            "1535     1\n",
            "1536     1\n",
            "1537     1\n",
            "1538     1\n",
            "1539     1\n",
            "1540     1\n",
            "1541     1\n",
            "1514     1\n",
            "1512     1\n",
            "1481     1\n",
            "1495     1\n",
            "1482     1\n",
            "1483     1\n",
            "1484     1\n",
            "1485     1\n",
            "1486     1\n",
            "1487     1\n",
            "1489     1\n",
            "1490     1\n",
            "1491     1\n",
            "1492     1\n",
            "1493     1\n",
            "1494     1\n",
            "1496     1\n",
            "1511     1\n",
            "1497     1\n",
            "1498     1\n",
            "1499     1\n",
            "1500     1\n",
            "1501     1\n",
            "1502     1\n",
            "1503     1\n",
            "1505     1\n",
            "1507     1\n",
            "1508     1\n",
            "1509     1\n",
            "1510     1\n",
            "1415     1\n",
            "1414     1\n",
            "1413     1\n",
            "1333     1\n",
            "1320     1\n",
            "1322     1\n",
            "1323     1\n",
            "1324     1\n",
            "1325     1\n",
            "1326     1\n",
            "1327     1\n",
            "1328     1\n",
            "1329     1\n",
            "1330     1\n",
            "1331     1\n",
            "1332     1\n",
            "1334     1\n",
            "1318     1\n",
            "1335     1\n",
            "1336     1\n",
            "1337     1\n",
            "1338     1\n",
            "1339     1\n",
            "1340     1\n",
            "1341     1\n",
            "1342     1\n",
            "1343     1\n",
            "1344     1\n",
            "1345     1\n",
            "1347     1\n",
            "1319     1\n",
            "1317     1\n",
            "1349     1\n",
            "1299     1\n",
            "1286     1\n",
            "1287     1\n",
            "1288     1\n",
            "1289     1\n",
            "1291     1\n",
            "1292     1\n",
            "1293     1\n",
            "1294     1\n",
            "1295     1\n",
            "1296     1\n",
            "1297     1\n",
            "1298     1\n",
            "1300     1\n",
            "1316     1\n",
            "1301     1\n",
            "1302     1\n",
            "1304     1\n",
            "1305     1\n",
            "1306     1\n",
            "1307     1\n",
            "1308     1\n",
            "1309     1\n",
            "1310     1\n",
            "1312     1\n",
            "1314     1\n",
            "1315     1\n",
            "1348     1\n",
            "1350     1\n",
            "1412     1\n",
            "1396     1\n",
            "1384     1\n",
            "1385     1\n",
            "1386     1\n",
            "1387     1\n",
            "1388     1\n",
            "1389     1\n",
            "1390     1\n",
            "1391     1\n",
            "1392     1\n",
            "1393     1\n",
            "1394     1\n",
            "1395     1\n",
            "1398     1\n",
            "1382     1\n",
            "1399     1\n",
            "1400     1\n",
            "1401     1\n",
            "1402     1\n",
            "1403     1\n",
            "1404     1\n",
            "1406     1\n",
            "1407     1\n",
            "1408     1\n",
            "1409     1\n",
            "1410     1\n",
            "1411     1\n",
            "1383     1\n",
            "1381     1\n",
            "1351     1\n",
            "1364     1\n",
            "1352     1\n",
            "1353     1\n",
            "1354     1\n",
            "1355     1\n",
            "1356     1\n",
            "1357     1\n",
            "1358     1\n",
            "1359     1\n",
            "1360     1\n",
            "1361     1\n",
            "1362     1\n",
            "1363     1\n",
            "1365     1\n",
            "1379     1\n",
            "1366     1\n",
            "1367     1\n",
            "1368     1\n",
            "1369     1\n",
            "1370     1\n",
            "1371     1\n",
            "1373     1\n",
            "1374     1\n",
            "1375     1\n",
            "1376     1\n",
            "1377     1\n",
            "1378     1\n",
            "3996     1\n",
            "Name: count, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "\n",
        "over_train_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/over_train_indices.npy'\n",
        "val_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/val_indices.npy'\n",
        "test_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/test_indices.npy'\n",
        "\n",
        "\n",
        "over_train_indices = np.load(over_train_indices_path)\n",
        "val_indices = np.load(val_indices_path)\n",
        "test_indices = np.load(test_indices_path)\n",
        "\n",
        "df_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/final_df.csv'\n",
        "\n",
        "# Read the CSV files\n",
        "#df = pd.read_csv(df_file)\n",
        "\n",
        "\n",
        "X_train = df.take(over_train_indices)\n",
        "\n",
        "\n",
        "print(X_train.shape)\n",
        "\n",
        "print(X_train['RECORD_ID'].value_counts())\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Pre-processing and EDA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    RECORD_ID                                    Brand                   Make           Model                                                                                                                                                                                                                                      Year                               Fitment                                               Type OEM Part Number                                                   Product Name  \\\n",
            "0           0                                ['dayco']                  DODGE     W100 SERIES                                                                                                                                                                                                                          ['1967', '1967']                ['Direct Replacement']                          ['Radiator Coolant Hose']              []                                                             []   \n",
            "1           1                               ['toyota']                    NaN         COROLLA                                                                                                                                                                                                                                  ['2007']                                    []                                                 []              []                                                             []   \n",
            "2           2                               ['nissan']                 NISSAN          ALTIMA                                                                                                                                                                                                                                  ['2018']                                    []                                                 []              []                                                             []   \n",
            "3           3  ['standard', 'standard motor products']                   FORD         MUSTANG                                                                                                                                                                                                          ['1986', '1983', '1983', '1986']                                    []                                                 []   ['84469JXJR']                              ['Starter Motor Relay Connector']   \n",
            "4           4                                 ['febi']                    NaN          CLK320                                                                                                          ['1999', '2005', '1998', '2000', '2001', '2002', '2003', '2004', '1998', '1999', '2000', '2001', '2002', '2003', '2004', '2005']                                    []                                                 []   ['81245RKMK']  ['Crankcase Breather Hose', 'Engine Crankcase Breather Hose']   \n",
            "5           5                                 ['febi']                    NaN            280S                                                                                                                                                                          ['1968', '1969', '1970', '1971', '1968', '1969', '1970', '1971']                                    []                                                 []     ['37726WX']               ['Camshaft Gear', 'Engine Timing Camshaft Gear']   \n",
            "6           6                    ['usa standard gear']                 TOYOTA        STANDARD                                                                                                                                                                                                                                  ['1991']                                    []                                                 []              []                                                             []   \n",
            "7           7    ['national seal/bearing', 'national']                    NaN             NaN                                                                                                                                                                                                                                      ['']                                    []  ['Automatic Transmission Extension Housing Seal']  ['B7TZ7A011A']                                                             []   \n",
            "8           8                             ['ac delco']                    NaN            S112                                                                                                                                                                                                                  ['1957', '1957', '1957']  ['International S112 Ignition Coil']                                                 []      ['H716YN']                                              ['Ignition Coil']   \n",
            "9           9                  ['mann', 'mann-filter']                    NaN             NaN                                                                                                                                                                                                                                      ['']                                    []                              ['Engine Oil Filter']              []                                                             []   \n",
            "10         10                              ['new-apf']                  MAZDA           B4000                                                                                                                                                                                                  ['2000', '2006', '2007', '2008', '2009']                                    []                                                 []              []                                                             []   \n",
            "11         11                                 ['ford']                   FORD          TAURUS                                                                                                                                                                                                                                  ['2014']                                    []                                                 []              []                                                             []   \n",
            "12         12                                  ['ngk']                    NaN             NaN                                                                                                                                                                                                                                      ['']                                    []                                     ['Spark Plug']              []                                                             []   \n",
            "13         13                                ['dayco']                MERCURY           COMET  ['1961', '1960', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1971', '1972', '1973', '1974', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1971', '1972', '1973', '1974', '1975', '1976', '1977']                                    []                                                 []     ['24777WG']                     ['Radiator Hose', 'Radiator Coolant Hose']   \n",
            "14         14                ['ac delco professional']              ACURA CSX             CSX                                                                                          ['2006', '2007', '2008', '2009', '2010', '2011', '2006', '2007', '2008', '2009', '2010', '2011', '2006', '2007', '2008', '2009', '2010', '2011']               ['Professional -- New']                      ['Direct Ignition Coil Boot']      ['37YNPZ']                                                             []   \n",
            "15         15                                 ['remy']                    NaN             NaN                                                                                                                                                                                                                                      ['']                                    []                                     ['Alternator']  ['028903028D']                                                             []   \n",
            "16         16                         ['partscollect']                 JAGUAR              XF                                                                                                                                                                                                  ['2009', '2010', '2011', '2012', '2013']                ['Direct Replacement']                               ['Engine Mount Kit']              []                                                             []   \n",
            "17         17                               ['airtex']                   FORD  COUNTRY SQUIRE                                                                                                                                                                  ['1965', '1966', '1967', '1965', '1966', '1967', '1965', '1966', '1967']    ['Ford Country Squire Water Pump']                                                 []      ['W576GB']                                                 ['Water Pump']   \n",
            "18         18                                  ['wix']                AUTOCAR              DK                                                                                                                                                                                                                          ['1984', '1985']                ['Direct Replacement']                                                 []              []                                                             []   \n",
            "19         19  ['standard', 'standard motor products']  CHEVROLET TRAILBLAZER     TRAILBLAZER                                                                                                                                                                                                                  ['2009', '2009', '2009']               ['Spark Plug Wire Set']                            ['Spark Plug Wire Set']      ['78YWWX']                                                             []   \n",
            "\n",
            "                      Manufacturer Part Number Vehicle Identification Number                   SKU ManufacturePartNumber  \\\n",
            "0   ['sx-6425 1967 belts cooling hoses pipes']                            []                    []                    []   \n",
            "1                     ['corolla~336959456lkq']         ['JTDBR32E770118632']                    []                    []   \n",
            "2                                           []         ['1N4AL3AP8JC163431']  ['211207', '187356']                    []   \n",
            "3                                ['84469jxjr']                            []     ['400:84469JXJR']                    []   \n",
            "4                                ['81245rkmk']                            []     ['400:81245RKMK']                    []   \n",
            "5                                           []                            []       ['400:37726WX']                    []   \n",
            "6          ['zmbk162bws', 'usa standard gear']                            []                    []        ['zmbk162bws']   \n",
            "7                                   ['415988']                            []        ['NAT:415988']            ['415988']   \n",
            "8                                           []                            []        ['400:H716YN']                    []   \n",
            "9                                ['h 932/5 x']                            []      ['M6:H 932/5 X']         ['h 932/5 x']   \n",
            "10                                          []                            []           ['1274612']                    []   \n",
            "11             ['taurus (sedan)~373920163lkq']         ['1FAHP2E88EG140987']                    []                    []   \n",
            "12                                   ['96779']                            []         ['NGK:96779']             ['96779']   \n",
            "13                                 ['24777wg']                            []       ['400:24777WG']                    []   \n",
            "14                                          []                            []        ['400:37YNPZ']                    []   \n",
            "15                                   ['94105']                            []         ['317:94105']             ['94105']   \n",
            "16                               ['c2p 17845']                            []                    []                    []   \n",
            "17                                          []                            []        ['400:W576GB']                    []   \n",
            "18                                ['sx-qj52d']                            []                    []                    []   \n",
            "19                                          []                            []        ['400:78YWWX']                    []   \n",
            "\n",
            "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  Part Title  \\\n",
            "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            ['UNITRANS ZMBK162BWS Bearings - Manual Transmission Bearing', 'Manual Transmission Bearing and Seal Overhaul Kit-Base, W59 USA Standard Gear']   \n",
            "7                                                                                                              ['CARQUEST 415988', 'Automatic Transmission Output Shaft Seal Timken 415988', 'Auto Plus/Timken 415988 - Automatic Transmission Output Shaft Seal', 'Automatic Transmission Output Shaft Seal Rear Coast to Coast Automotive Products', 'Advance 415988 Automatic Transmission Extension Housing Seal', 'Auto Plus/AutoExtra 415988 - Automatic Transmission Extension Housing Seal', 'Automatic Transmission Extension Housing Seal-5 Speed Trans Master Pro 415988', 'Automatic Transmission Extension Housing Seal-Auto Trans Extension Housing Seal', 'Auto Extra 415988', 'CARQUEST/National Oil Seals - Automatic Transmission Extension Housing Seal', \"O'Reilly 415988 Automatic Transmission Extension Housing Seal\", 'Coast To Coast 415988 - Automatic Transmission Output Shaft Seal', 'Automatic Transmission Extension Housing Seal-Output Shaft Seal Moog 415988', \"Automatic Transmission Output Shaft Seal O'Reilly 415988\", 'Auto Plus/National Oil Seals - Automatic Transmission Extension Housing Seal']   \n",
            "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         []   \n",
            "9                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     ['Engine Oil Filter-Eng Code: AAA, FI MANN H 932/5 X']   \n",
            "10                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "11                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "12                                                                                                                                                                                                                                                                                                                                                                                   ['Auto Plus/NGK 96779 Secondary Ignition - Spark Plug', 'Hood Lift Support-No Mighty Lift 96779', 'Transmission Filter Kit CARQUEST 96779', 'Disc Brake Rotor-Specialty - Street Performance Raybestos 96779', 'Steelman 96779 Sensors - Tire Pressure Monitoring System Sensor Component Kit', 'CARQUEST Trans Filters 96779 Service Kits - Transmission Filter Kit', 'Transmission Filter Kit Advance 96779', 'Auto Plus/Raybestos 96779 Drums and Rotors - Disc Brake Rotor, Rear', 'Advance 96779 Transmission Filter Kit', 'Advance 96779 Spark Plug', 'Auto Extra 96779', 'CARQUEST/NGK 96779 Secondary Ignition - Spark Plug', 'Spark Plug-Laser Iridium NGK 96779', 'Spark Plug-GAS NGK Canada 96779', 'AVM Ind 96779 Reman Door Lift Support']   \n",
            "13                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "14                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "15  ['Alternator-Eng Code: AEG Remy 94105', 'Side Window Deflector-Ventvisor Deflector 4 Pc. Auto Ventshade 94105', 'Catalytic Converter Magnaflow 94105', 'Alternator-New Advance 94105', 'CARQUEST/Magnaflow 94105 Catalytic Converter - Catalytic Converter', 'Catalytic Converter Magnaflow CO 94105', 'Catalytic Converter-Universal Standard Grade Federal(Exc. CA) Magnaflow 49 State', 'Engine Coolant Pipe Auto-Kool 94105', 'Car Sound 94105', 'CARQUEST 94105', 'PB/Magnaflow 94105 Catalytic Converter - Catalytic Converter', 'Alternator-New Remy 94105', 'Exhaust Pipe AP Exhaust 94105 fits 2006 Kia Optima 2.7L-V6', 'HVAC Heater Core 4 Seasons 94105', 'Catalytic Converter-Universal Standard Grade Federal(Exc. CA) Magnaflow ME 94105', 'Catalytic Converter Magnaflow CA 94105', 'Alternator-New CARQUEST 94105', 'Advance 94105', 'Advance 94105 Catalytic Converter', 'Catalytic Converter-RWD Magnaflow NY 94105', 'Alternator-New Champion New Alternators/Starters 94105', 'Wheel Bearing Autopart Intl 94105 fits 1998 Audi A6', 'UNITRANS 94105 Hoses and Pipes - Engine Coolant Bypass Pipe', 'Pronto 94105']   \n",
            "16                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "17                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "18                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "19                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        []   \n",
            "\n",
            "                                                                                                                                                                                                                                                                                                                                                                     Part Brands  \\\n",
            "0                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "1                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "2                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "3                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "4                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "5                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "6                                                                                                                                                                                                                                                                                                                                              ['Unitrans', 'USA Standard Gear']   \n",
            "7                                                                           ['Timken', 'Master Pro', 'MOOG', 'Carquest', 'Coast To Coast Automotive Products', \"O'Reilly Auto Parts Precision Joints\", 'Auto Plus', 'ADVANCE/Drive Train', 'Coast To Coast', 'National Oil Seals', 'Auto Extra', \"O'Reilly Auto Parts\", 'National Seal/Bearing', 'Auto Plus/National Oil Seals']   \n",
            "8                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "9                                                                                                                                                                                                                                                                                                                                                                ['MANN-FILTER']   \n",
            "10                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "11                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "12                                                                                                                       ['NGK Canada', 'Carquest', 'Mighty Lift', 'Auto Plus', 'ADVANCE/Drive Train', 'Raybestos Unbranded', 'Steelman', 'Auto Extra', 'Raybestos', 'Auto Plus/NGK', 'CARQUEST/NGK', 'ADVANCE/Ignition & Engine Filters', 'Avm Ind', 'ADVANCE/CARQUEST', 'NGK']   \n",
            "13                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "14                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "15  ['Advance Auto Parts', 'Car Sound', 'Auto Ventshade (AVS)', 'Unitrans', 'Champion', 'Magnaflow California Converter', 'Autopart International', 'MagnaFlow', 'AP Exhaust', 'Remy2020', 'CARQUEST/Magnaflow', 'Magnaflow Colorado Converter', 'PEP BOYS', 'Pronto', 'Magnaflow Maine Converter', 'Carquest', 'Remy', 'Auto-Kool', 'ADVANCE/Exhaust & Clutch', 'Four Seasons']   \n",
            "16                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "17                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "18                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "19                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "\n",
            "                                                                                                                                                                                                                                                                                                                                                                         SubType  \\\n",
            "0                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "1                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "2                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "3                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "4                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "5                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "6                                                                                                                                                                                                                                                                                           ['Manual Transmission Bearing and Seal Overhaul Kit', 'Manual Transmission Bearing']   \n",
            "7   ['Automatic Transmission Output Shaft Seal|Multi-Purpose Seal', 'Automatic Transmission Extension Housing Seal', 'Automatic Transmission Extension Housing Seal|Automatic Transmission Output Shaft Seal', 'Automatic Transmission Output Shaft Seal', 'Automatic Transmission Extension Housing Seal|Automatic Transmission Output Shaft Seal|Automatic Transmission Seal']   \n",
            "8                                                                                                                                                                                                                                                                                                                                                                             []   \n",
            "9                                                                                                                                                                                                                                                                                                                                ['Engine Oil Filter|Engine Oil Filter Element']   \n",
            "10                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "11                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "12                                                                                                                                            ['Spark Plug', 'Transmission Filter Kit', 'Tire Pressure Monitoring System Sensor Component Kit', 'Disc Brake Rotor', 'Door Lift Support', 'Hood Lift Support', 'Brake Drum|Disc Brake Kit|Disc Brake Rotor|Disc Brake Rotor Set']   \n",
            "13                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "14                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "15                                                                                                                                                                                 ['HVAC Heater CORE', 'Alternator', 'Exhaust Pipe', 'Wheel Bearing', 'Engine Coolant Bypass Pipe', 'Engine Coolant Pipe', 'Side Window Deflector|Vent Window Assembly', 'Catalytic Converter']   \n",
            "16                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "17                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "18                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "19                                                                                                                                                                                                                                                                                                                                                                            []   \n",
            "\n",
            "                                                                                                                                                                                                                                                                Part Types  CATEGORY                                                                       ITEM_TITLE  \n",
            "0                                                                                                                                                                                                                                                                       []       1.0     dayco upper radiator coolant hose for 1967 dodge w100 series 63l v8 belts ky  \n",
            "1                                                                                                                                                                                                                                                                       []       4.0          2007 corolla front driver left cv axle shaft oem 91k miles lkq336959456  \n",
            "2                                                                                                                                                                                                                                                                       []       0.0                                      alternator 4 cylinder fits 18 altima 187356  \n",
            "3                                                                                                                                                                                                                                                                       []       0.0           for 1983 ford mustang starter motor relay connector smp 84469jxjr 1986  \n",
            "4                                                                                                                                                                                                                                                                       []       2.0   for 1998-2005 mercedes clk320 crankcase breather hose febi 81245rkmk 1999 2000  \n",
            "5                                                                                                                                                                                                                                                                       []       2.0                 for 1968-1971 mercedes 280s camshaft gear febi 37726wx 1969 1970  \n",
            "6                                                                                                                                                                                                                           ['Bearings', 'Manual Transmission Components']       4.0     usa standard manual transmission bearing kit 1991 toyota 4-cyl with synchros  \n",
            "7                                                                                                                                                                                                        ['Gaskets and Sealing Systems', '41|Gaskets and Sealing Systems']       4.0  automatic transmission extension housing seal-auto trans extension housing seal  \n",
            "8                                                                                                                                                                                                                                                                       []       3.0                       ignition coil for 1957 international s112 39l 6 cyl h716yn  \n",
            "9                                                                                                                                                                                                                                                              ['Filters']       2.0                                                  engine oil filter mann h 9325 x  \n",
            "10                                                                                                                                                                                                                                                                      []       4.0    cv axle shaft for 2006-2009 mazda b4000 40l v6 gas 4wd front left side 2299in  \n",
            "11                                                                                                                                                                                                                                                                      []       1.0       2014 ford taurus sedan coolant overflow bottle oem 123k miles lkq373920163  \n",
            "12                                                                                                                                                 ['Drums and Rotors|Service Kits', 'Drums and Rotors', 'Doors', 'Sensors', 'Service Kits', 'Secondary Ignition', 'Hood']       3.0                                                         spark plug-gas ngk 96779  \n",
            "13                                                                                                                                                                                                                                                                      []       1.0     for 1960-1968 1971-1977 mercury comet radiator hose lower dayco 24777wg 1961  \n",
            "14                                                                                                                                                                                                                                                                      []       3.0     ac delco direct ignition coil boot fits acura csx 2006-2011 20l 4 cyl 37ynpz  \n",
            "15  ['Hoses and Pipes', 'Wheel Bearings, Seals, and Related Components', 'Motors, CORE, Case and Related Components', 'Exhaust and Tail Pipes', 'Alternator / Generator and Related Components', 'Catalytic Converter', 'Glass, Windows and Related Components|Side Body']       0.0                                               alternator-eng code aeg remy 94105  \n",
            "16                                                                                                                                                                                                                                                                      []       2.0         engine motor mount left  right c2p 17845 c2p17845 for 09-13 jaguar xf xj  \n",
            "17                                                                                                                                                                                                                                                                      []       1.0       water pump for 1965-1967 ford country squire 1966 w576gb engine water pump  \n",
            "18                                                                                                                                                                                                                                                                      []       2.0    wix engine oil filter for 1984-1985 autocar dk 140l l6 - filtration system wy  \n",
            "19                                                                                                                                                                                                                                                                      []       3.0                    spark plug wire set fits chevy trailblazer 2009 60l v8 78ywwx  \n",
            "(5000, 19)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from transformers import RobertaTokenizer, TFRobertaForSequenceClassification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer\n",
        "import tensorflow as tf\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle\n",
        "from transformers import AutoTokenizer, TFAutoModelForSequenceClassification\n",
        "from sklearn.metrics import classification_report, hamming_loss\n",
        "from collections import Counter\n",
        "import seaborn as sns\n",
        "\n",
        "# Display the entire sorted DataFrame\n",
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', 500)\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "\n",
        "\n",
        "# Download required NLTK data\n",
        "#nltk.download('punkt')\n",
        "#nltk.download('stopwords')\n",
        "#nltk.download('wordnet')\n",
        "\n",
        "df_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/final_df.csv'\n",
        "desc_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/desc.csv'\n",
        "fitment_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/ftmnt_train.csv'\n",
        "\n",
        "# Read the CSV files\n",
        "df = pd.read_csv(df_file)\n",
        "\n",
        "print(df.head(20))\n",
        "\n",
        "print(df.shape)\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "df.replace(\"[]\", np.nan, inplace=True)\n",
        "df.replace(\"['']\", np.nan, inplace=True)\n",
        "df.replace('[\"]', np.nan, inplace=True)\n",
        "df = df.drop(columns=['ManufacturePartNumber'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "ExecuteTime": {
          "end_time": "2024-07-28T02:08:42.085726Z",
          "start_time": "2024-07-28T02:08:37.709227Z"
        },
        "id": "xJWWOYMm2Mlr"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to\n",
            "[nltk_data]     /Users/cyrusaghaee/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Index(['RECORD_ID', 'Brand', 'Make', 'Model', 'Year', 'Fitment', 'Type', 'OEM Part Number', 'Product Name', 'Manufacturer Part Number', 'Vehicle Identification Number', 'SKU', 'ManufacturePartNumber', 'Part Title', 'Part Brands', 'SubType', 'Part Types', 'CATEGORY', 'ITEM_TITLE'], dtype='object')\n",
            "df:  (5000, 18)\n",
            "desc:  (5000, 2)\n",
            "fitment:  (151106, 4)\n",
            "merged_train_df:  (203645, 22)\n",
            "merged_val_df:  (17766, 22)\n",
            "merged_test_df:  (14732, 22)\n"
          ]
        },
        {
          "ename": "",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
            "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
            "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
            "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
          ]
        },
        {
          "ename": "",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31mFailed to interrupt the Kernel. \n",
            "\u001b[1;31mThe kernel died. Error: ... View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from transformers import RobertaTokenizer, TFRobertaForSequenceClassification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer\n",
        "import tensorflow as tf\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle\n",
        "from transformers import AutoTokenizer, TFAutoModelForSequenceClassification\n",
        "from sklearn.metrics import classification_report, hamming_loss\n",
        "from collections import Counter\n",
        "import seaborn as sns\n",
        "\n",
        "# Display the entire sorted DataFrame\n",
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', 500)\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "\n",
        "\n",
        "# Download required NLTK data\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "df_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/final_df.csv'\n",
        "desc_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/desc.csv'\n",
        "fitment_file = '/Users/cyrusaghaee/DS 207/Data/ebay_data/ftmnt_train.csv'\n",
        "\n",
        "over_train_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/over_train_indices.npy'\n",
        "val_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/val_indices.npy'\n",
        "test_indices_path = '/Users/cyrusaghaee/DS 207/Final Project/test_indices.npy'\n",
        "\n",
        "# Read the CSV files\n",
        "df = pd.read_csv(df_file)\n",
        "\n",
        "print(df.columns)\n",
        "\n",
        "df.replace(\"[]\", np.nan, inplace=True)\n",
        "df.replace(\"['']\", np.nan, inplace=True)\n",
        "df.replace('[\"]', np.nan, inplace=True)\n",
        "df = df.drop(columns=['ManufacturePartNumber'])\n",
        "\n",
        "\n",
        "over_train_indices = np.load(over_train_indices_path)\n",
        "val_indices = np.load(val_indices_path)\n",
        "test_indices = np.load(test_indices_path)\n",
        "\n",
        "\n",
        "\n",
        "train_df = df.take(over_train_indices)\n",
        "val_df = df.take(val_indices)\n",
        "test_df = df.take(test_indices)\n",
        "\n",
        "\n",
        "\n",
        "desc = pd.read_csv(desc_file)\n",
        "fitment = pd.read_csv(fitment_file)\n",
        "desc = desc.drop(columns=['Decoded_DESC'])\n",
        "\n",
        "\n",
        "merged_train_df = fitment.merge(train_df, on='RECORD_ID', how='right').merge(desc, on='RECORD_ID', how='left')\n",
        "merged_val_df = fitment.merge(val_df, on='RECORD_ID', how='right').merge(desc, on='RECORD_ID', how='left')\n",
        "merged_test_df = fitment.merge(test_df, on='RECORD_ID', how='right').merge(desc, on='RECORD_ID', how='left')\n",
        "\n",
        "print(\"df: \",df.shape)\n",
        "print(\"desc: \",desc.shape)\n",
        "print(\"fitment: \",fitment.shape)\n",
        "\n",
        "print(\"merged_train_df: \",merged_train_df.shape)\n",
        "print(\"merged_val_df: \",merged_val_df.shape)\n",
        "print(\"merged_test_df: \",merged_test_df.shape)\n",
        "\n",
        "X_train = merged_train_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_val = merged_val_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "X_test = merged_test_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "\n",
        "# Function to create the concatenated string for each row\n",
        "def create_x_string(row):\n",
        "    # List comprehension to create 'column: value' strings for non-null values\n",
        "    return ', '.join([f\"{col}: {val}\" for col, val in row.items() if pd.notnull(val)])\n",
        "\n",
        "# Apply the function to each row in X and create a new column 'x_string'\n",
        "merged_train_df['processed_x_string'] = X_train.apply(create_x_string, axis=1).str.lower()\n",
        "merged_val_df['processed_x_string'] = X_val.apply(create_x_string, axis=1).str.lower()\n",
        "merged_test_df['processed_x_string'] = X_test.apply(create_x_string, axis=1).str.lower()\n",
        "\n",
        "\n",
        "merged_train_df['FTMNT_MAKE'] = merged_train_df['FTMNT_MAKE'].str.lower()\n",
        "merged_val_df['FTMNT_MAKE'] = merged_val_df['FTMNT_MAKE'].str.lower()\n",
        "merged_test_df['FTMNT_MAKE'] = merged_test_df['FTMNT_MAKE'].str.lower()\n",
        "\n",
        "# Group the data by description and aggregate the compatible makes into lists\n",
        "grouped_train = merged_train_df.groupby('processed_x_string')['FTMNT_MAKE'].apply(list).reset_index()\n",
        "grouped_train['FTMNT_MAKE'] = grouped_train['FTMNT_MAKE'].apply(lambda x: list(set(x)))\n",
        "grouped_train = grouped_train[['FTMNT_MAKE','processed_x_string']]\n",
        "\n",
        "grouped_val = merged_val_df.groupby('processed_x_string')['FTMNT_MAKE'].apply(list).reset_index()\n",
        "grouped_val['FTMNT_MAKE'] = grouped_val['FTMNT_MAKE'].apply(lambda x: list(set(x)))\n",
        "grouped_val = grouped_val[['FTMNT_MAKE','processed_x_string']]\n",
        "\n",
        "\n",
        "grouped_test = merged_test_df.groupby('processed_x_string')['FTMNT_MAKE'].apply(list).reset_index()\n",
        "grouped_test['FTMNT_MAKE'] = grouped_test['FTMNT_MAKE'].apply(lambda x: list(set(x)))\n",
        "grouped_test = grouped_test[['FTMNT_MAKE','processed_x_string']]\n",
        "\n",
        "\n",
        "\n",
        "grouped_train.to_csv('merged_train_df.csv', index=False)\n",
        "grouped_val.to_csv('merged_val_df.csv', index=False)\n",
        "grouped_test.to_csv('merged_test_df.csv', index=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "X = merged_df.drop(columns=['FTMNT_YEAR', 'FTMNT_MAKE', 'FTMNT_MODEL', 'RECORD_ID'])\n",
        "\n",
        "# Function to create the concatenated string for each row\n",
        "def create_x_string(row):\n",
        "    # List comprehension to create 'column: value' strings for non-null values\n",
        "    return ', '.join([f\"{col}: {val}\" for col, val in row.items() if pd.notnull(val)])\n",
        "\n",
        "# Apply the function to each row in X and create a new column 'x_string'\n",
        "merged_df['x_string'] = X.apply(create_x_string, axis=1)\n",
        "\n",
        "# Apply preprocessing with prioritization\n",
        "merged_df['processed_x_string'] = merged_df['x_string'].str.lower()\n",
        "merged_df['FTMNT_MAKE'] = merged_df['FTMNT_MAKE'].str.lower()\n",
        "\n",
        "# Group the data by description and aggregate the compatible makes into lists\n",
        "grouped = merged_df.groupby('processed_x_string')['FTMNT_MAKE'].apply(list).reset_index()\n",
        "grouped['FTMNT_MAKE'] = grouped['FTMNT_MAKE'].apply(lambda x: list(set(x)))\n",
        "\n",
        "grouped.head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    brand: [\"bolt n' go\"], make: hyundai, model: v...\n",
              "1    brand: [\"caili's company\"], make: mazda, model...\n",
              "2    brand: [\"ton's performance\"], make: gmc, model...\n",
              "3    brand: ['%20used'], make: toyota, model: 4 run...\n",
              "4    brand: ['1autopowerstore'], make: ford, model:...\n",
              "5    brand: ['2006 mini mini cooper stock number: 0...\n",
              "6    brand: ['2mplastic'], year: ['2015', '2016', '...\n",
              "7    brand: ['4 seasons', 'fan motor', 'four season...\n",
              "8    brand: ['4 seasons', 'four seasons'], make: ch...\n",
              "9    brand: ['4 seasons', 'four seasons'], make: fo...\n",
              "Name: processed_x_string, dtype: object"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "grouped['processed_x_string'].head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "grouped.to_csv('merged_df.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [],
      "source": [
        "def preprocess_text(text):\n",
        "    # Lowercase and remove special characters except hyphens\n",
        "    return re.sub(r'[^a-z0-9\\s-]', '', text.lower())\n",
        "\n",
        "\n",
        "# Preprocess the text\n",
        "merged_df['processed_x_string'] = merged_df['x_string'].apply(preprocess_text)\n",
        "\n",
        "grouped.to_csv('preprocessed_merged_df.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "gpuType": "V28",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}