house_price_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict House Price "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Goal is to create a machine pipeline that can accurately predict the sales price for each house."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data is from https://www.kaggle.com/c/house-prices-advanced-regression-techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7.11\n"
     ]
    }
   ],
   "source": [
    "from platform import python_version\n",
    "print(python_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "\n",
       "[4 rows x 81 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This dataset contains 1460 observations with 79 features. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 81)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will do following operations:\n",
    "- Check for missing values and discard columns that contain too much missing information\n",
    "- Impute missing values in numerical column by using iterative imputer\n",
    "- Convert categorical columns with onehot encoder\n",
    "- Create pipeline to apply imputing, standard scaling and machine learning models\n",
    "- Test with PCA and without PCA\n",
    "- Choose best parameters for machine learning models\n",
    "- Finally apply on the test dataset provided by kaggle for submission"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Inspection and Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print percentage of missing values in each column/feature\n",
    "def print_null_percentage(df):\n",
    "    m=df.isnull().sum()\n",
    "    null_columns=[]\n",
    "    print('Feature Name | Num Missing Value | Total Num Value | Percentage')\n",
    "    print('================================================================')\n",
    "    for i,v in enumerate(m):\n",
    "        if v!=0:\n",
    "            print(\"%s %d  %d, P: %0.2f\"%(m.index[i],v,df.shape[0],100*v/df.shape[0]))\n",
    "            null_columns.append(m.index[i])\n",
    "    return null_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Name | Num Missing Value | Total Num Value | Percentage\n",
      "================================================================\n",
      "LotFrontage 259  1460, P: 17.74\n",
      "Alley 1369  1460, P: 93.77\n",
      "MasVnrType 8  1460, P: 0.55\n",
      "MasVnrArea 8  1460, P: 0.55\n",
      "BsmtQual 37  1460, P: 2.53\n",
      "BsmtCond 37  1460, P: 2.53\n",
      "BsmtExposure 38  1460, P: 2.60\n",
      "BsmtFinType1 37  1460, P: 2.53\n",
      "BsmtFinType2 38  1460, P: 2.60\n",
      "Electrical 1  1460, P: 0.07\n",
      "FireplaceQu 690  1460, P: 47.26\n",
      "GarageType 81  1460, P: 5.55\n",
      "GarageYrBlt 81  1460, P: 5.55\n",
      "GarageFinish 81  1460, P: 5.55\n",
      "GarageQual 81  1460, P: 5.55\n",
      "GarageCond 81  1460, P: 5.55\n",
      "PoolQC 1453  1460, P: 99.52\n",
      "Fence 1179  1460, P: 80.75\n",
      "MiscFeature 1406  1460, P: 96.30\n"
     ]
    }
   ],
   "source": [
    "null_columns=print_null_percentage(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop columns with too many missing values\n",
    "drop_cols = ['Alley','MiscFeature','PoolQC','Fence','MiscFeature']\n",
    "df.drop(columns=drop_cols,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Relevant Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Use multivariate imputer to impute values for 2 columns\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer, IterativeImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder\n",
    "from sklearn.decomposition import PCA, KernelPCA, SparsePCA\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Log Root Mean Square Error function to quantify the success of predictor \n",
    "def logrmse(y_true, y_pred):\n",
    "    return np.sqrt(np.mean((np.log(y_true)-np.log(y_pred))**2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a Bayesian method for imputing missing values in numerical columns. \n",
    "\n",
    "Missing values in categorical column will be signed as None and regarded as a new category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find categorical and numerical features for preprocessing steps. \n",
    "categorical = [c for c in df.columns if df[c].dtype=='object']\n",
    "numerical = list(set(df.columns).difference(set(categorical+['Id','SalePrice'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 39 categorical and 36 numerical features\n"
     ]
    }
   ],
   "source": [
    "print('There are {} categorical and {} numerical features'.format(len(categorical), len(numerical)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing steps\n",
    "# Impute missing values\n",
    "iter_imp = IterativeImputer(estimator=BayesianRidge(),\n",
    "                         missing_values=np.nan, \n",
    "                         sample_posterior=True, \n",
    "                         max_iter=100, \n",
    "                         min_value=0, \n",
    "                         random_state=1)\n",
    "# impute missing numerical values by using Bayesian Ridge\n",
    "numerical_transformer = Pipeline(steps = [('imputer',iter_imp),\n",
    "                                          ('scaler',StandardScaler())])\n",
    "# fill missing values as 'None' so that they will be a new category\n",
    "categorical_transformer = Pipeline(steps = [('imputer',SimpleImputer(strategy='constant',fill_value='None')),\n",
    "                                            ('ohe', OneHotEncoder(sparse=False,handle_unknown='ignore'))])\n",
    "ordinal_transformer = Pipeline(steps = [('imputer',SimpleImputer(strategy='constant')),\n",
    "                                        ('ohe', OrdinalEncoder())])\n",
    "preprocessor = ColumnTransformer(transformers = [('num',numerical_transformer,numerical),\n",
    "                                                 ('cat',categorical_transformer,categorical)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split dataset into training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(df.iloc[:,1:-1],\n",
    "                                                 df.iloc[:,-1],\n",
    "                                                 test_size=0.33,\n",
    "                                                 random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Linear Regression with L1 Regularization (Lasso)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare Lasso with and without PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1a. Lasso with PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_lr_wpca = Pipeline(steps=[('preprocessor', preprocessor),\n",
    "                      ('reduce',PCA(n_components=10)),\n",
    "                      ('classifier', Lasso(alpha=1.0))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the best alpha that gives the maximum validation score\n",
    "alpha = np.logspace(0, 3, num=20, endpoint=True, base=10.0)\n",
    "param_grid=[{\n",
    "    'classifier__alpha': alpha\n",
    "}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_lr_wpca = GridSearchCV(pipe_lr_wpca,n_jobs=4, param_grid=param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_lr_wpca.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_lr_report = pd.DataFrame(gs_lr_wpca.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Mean test score with PCA')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABCQklEQVR4nO3deXyV5Zn4/8+VlQSyEEggC5AAAVlUCAHFvYgVVFyg02q11dbqMK1df2NHp52u3850aqfTzY7jbmvVUqHKUHCpVVRUIGFfBZIAgUACSUggZL9+fzxP4HA4SU6Sc3Jykuv9ep0X59zPdt8HyJV7F1XFGGOMCYSIUGfAGGNM/2FBxRhjTMBYUDHGGBMwFlSMMcYEjAUVY4wxAWNBxRhjTMBYUDHGtEtERovISRGJ7OAcFZHxvZkv03dZUDG9RkRKRKRRRIZ7pW9yfzBl93J+rhGR0gDd6x0R+VIg7tWXqOoBVR2iqi3Q83KKyA9EpMkNVNUi8oGIzPY4ni4iT4lImYjUisguEfmhiAz2OEdEpEhEdvSsdCYYLKiY3lYM3NH2QUQuBOJCl53+QUSiQp2HLviTqg4BUoH3gWVuoEgBPsT59zBbVROA64BkYJzH9VcBacBYEZnZqzk3nbKgYnrbH4DPe3y+G/i95wkiEisiPxeRAyJyVEQeE5E499hQEVkhIhUiUuW+z/K49h0R+bGIrHF/033Du2bknjcYWAVkuL81nxSRDBGJEJGHRGSfiBwXkSXuDztEZJCIPO+mV4vIehEZISI/Aa4Efuve57c+nufzWvdYiog8IyKH3TK94nHdfSKyV0QqRWS5iGR4HFMR+YqI7AH2uGk3uTW/tlrARb7+Etzf/n/jvo8WkVMi8jP3c5yI1Lvfdbb7nKhOyjlXRPa4+X9URMTXcz2pahPwHDASGAZ8C6gF7lLVEvecg6r6dVXd4nHp3cCrwEr3velLVNVe9uqVF1ACzAV2A5OASOAgMAZQINs975fAciAFSAD+D/gP99gwYBEQ7x77M/CKxzPeAfYBE3B+430H+Gk7+bkGKPVK+wbwEZAFxAL/C7zoHvtHNy/xbt5nAIkez/1SB2Xv6Nq/An8ChgLRwNVu+hzgGJDn5uU3wLse91TgTfd7inPPKwcucZ9xt/udx/rIzxxgq/v+Mvc7W+txbLP7Ptt9TlR75XSPr8CpUYwGKoB57XwPPwCed9/HAo8AB93PHwE/7OTfUDxQA9zg/js4BsSE+t+2vc6+rKZiQqGttnIdsAs41HbA/Q33PuCbqlqpqrXAvwO3A6jqcVVdqqp17rGfAFd73f8ZVf1YVU8DS4BpXcjbPwLfUdVSVW3A+SH4Kbd5qQknqI1X1RZVLVTVGj/v6/NaEUkH5gOLVbVKVZtUdbV7zZ3A06q6wc3Lw8Bsr76n/3C/p9M439v/qupa9xnPAQ3ApT7y8yGQKyLDcJqTngIyRWQIzve52sc1Hfmpqlar6gHgbTr+zj8tItU4v1DMAG5104cBZZ08ZyFOmd7ACWRRwI1dzKsJonBqhzX9xx+Ad4EcvJq+cNrZ44FCjxYUwfnNGxGJB/4bmIfzmz1AgohEqtuZDBzxuF8dMKQLeRsD/EVEWj3SWoARbr5HAS+JSDLwPE4AavLjvj6vddMqVbXKxzUZwIa2D6p6UkSOA5k4NRBwfjB75v1uEfmqR1qMe59zqOppESnACSBX4QTnacDlbtpv/CiTp65850tU9S4f6ceB9E6ec7d7fTPQLCLL3LS/dCWzJnispmJ6narux+mwvwFY5nX4GHAamKKqye4rSZ2OXYD/D5gIXKKqiTg/EMEJPF3Oio+0g8B8j2cnq+ogVT3k1iJ+qKqTcZqMbuJs/1CHy313cO1BIMUNNN4O4wQK4Ew/0DA8anZezz0I/MQr7/Gq+mI72VqN09Q1HVjvfr4emIUT9H0WpaNy9tDfgNtExOfPJbfvbA5wl4gcEZEjwKeAG3z1m5nQsKBiQuVeYI6qnvJMVNVW4Angv0UkDUBEMkXkeveUBJygU+12oH+/B3k4CgwTkSSPtMeAn4jIGPfZqSJyi/v+EyJyoThzNmpwmrRaPO41tr0HtXetqpbhDBj4ndsxHi0ibYHyBeALIjJNRGJxmgHXqtuJ7cMTwGIRuUQcg0XkRhFJaOf81TiBbYeqNuL2lwDFqlrRzjUdlrOHfgEkAs95fP+ZIvILd8DB54CPcX6pmOa+JgCleIwoNKFlQcWEhKruU9WCdg7/C7AX+EhEanB+g53oHvslTqf0MZyO3dd6kIddwItAkTtaKgP4Fc4ggTdEpNZ9xiXuJSOBl3GCwk6cH8rPu8d+hdP3UiUiv/bxuI6u/RxOkNmF09H+DTd/bwH/BizF6WsYh9u31E55CnD6VX4LVOF8h/d08BV8gPNdttVKdgD1tF9L8aec3aaqlTi1uCZgrfv9vwWcwCnL3cDvVPWI5wvnFwEbBdZHiKpt0mWMMSYwrKZijDEmYCyoGGOMCRgLKsYYYwLGgooxxpiAGdCTH4cPH67Z2dmhzoYxxoSVwsLCY6qa6uvYgA4q2dnZFBS0N6rVGGOMLyKyv71jQW3+EpF5IrLbXWX1IR/HH3RXVN0kIttEpMWd0NZ2PFJENorICo+0fxCR7SLSKiL5HunZInLa436PBbNsxhhjzhe0moo7c/hRnEUDS4H1IrJcVc9srKOqj+CsUoqILMBdRNDjNl/HmSiW6JG2DWdRuf/18dh9qjotkOUwxhjjv2DWVGYBe1W1yF0C4iXglg7OvwNndjNwZp2fG4EnPU9S1Z2qujsI+TXGGNNDwQwqmZy7gmqpm3Yed+XZeTjLUbT5JfBtoNXXNe3IcZvLVovIlV3LrjHGmJ4KZlDxtWpse2vCLADWtDV9ichNQLmqFnbheWXAaFWdjrOD3Asikuh9kojcLyIFIlJQUdHemnnGGGO6I5hBpRRnr4g2WThLeftyOx5NXzh7OtwsIiU4zWZzROR5Xxe2UdUGVT3uvi/k7O5/3uc9rqr5qpqfmupzRJwxxphuCmZQWY+zs1yOiMTgBI7l3ie5y45fjbPnNACq+rCqZqlqtnvd39vZ1MfzPqnu4ABEZCyQCxQFqjDGGGM6F7Sg4u7M9gDwOs4IriWqul1EFovIYo9TbwPe8N5Xoz0icpuIlAKzgb+KyOvuoauALSKyGWeJ8cVeI8lMP9fSqry47gA19f5sxGiMCYYBvfR9fn6+2uTH/uP/Nh/mqy9u5JtzJ/D1ubmhzo4x/ZaIFKpqvq9jtvaX6RdUlSfec1o7l20sZSD/smRMKFlQMf3CuuJKtpSeYFZOCvuP11G4vyrUWTJmQLKgYvqFJ94rImVwDP9zZx5x0ZEs3VAa6iwZMyBZUDFhb1/FSf62s5y7Lh3DsCGxzJ86khVbyqhvagl11owZcCyomLD35HvFxERF8PnZYwBYNCOL2vpm3txxNMQ5M2bgsaBiwtqxkw0s3VDKorwshg+JBeDSscNITxrEMmsCM6bXWVAxYe0PH+6nsbmVe6/IOZMWGSHcNj2Td/cco7y2PoS5M2bgsaBiwlZ9Uwt/+Gg/cyelMT5tyDnHFuZl0dKqLN/U3spAxphgsKBiwtbSDaVUnmrkS1eOPe/Y+LQhXDwqmZcLrQnMmN5kQcWEpdZW5an3irkoK4lLclJ8nrMoL5NdR2rZcbiml3NnzMBlQcWEpbd2lVN07BRfunIsIr52WYAFF2UQHSnWYW9ML7KgYsLSE+8WkZkcxw1TR7Z7ztDBMcy5II1XNh2muaUre70ZY7rLgooJO5sOVrOupJIvXJ5NVGTH/4QX5WVx7GQD7+051ku5M2Zgs6Biws4T7xWRMCiK22eN7vTcayamMTQ+mpetCcyYXmFBxYSVg5V1rNpaxmcvGc2Q2KhOz4+JiuDmizN4c8dRTpy2fVaMCTYLKiasPL2mmAgR7rks2+9rFs3IorG5lb9uKQtexowxQJCDiojME5HdIrJXRB7ycfxBEdnkvraJSIuIpHgcjxSRjSKywiPtH0Rku4i0iki+1/0edp+1W0SuD2bZTO87UdfEn9Yf5OaLM0hPivP7ugszkxifNsRGgRnTC4IWVNz94h8F5gOTgTtEZLLnOar6iKpOU9VpwMPAaq8tgL+OsxWxp23AQuBdr+dNxtnPfgowD/hd2571pn94Yd0B6hpbfE527IiIsCgvi4L9VZQc82vXamNMNwWzpjIL2KuqRaraCLwE3NLB+XcAL7Z9EJEs4EbgSc+TVHWnqu72cf0twEuq2qCqxcBeNw+mH2hsbuXZD4q5YvxwJmckdvn6W6dnIALLNh4KQu6MMW2CGVQygYMen0vdtPOISDxO7WKpR/IvgW8D/k4w8Ot5InK/iBSISEFFRYWftzah9n+bD3O0poEvXZnT+ck+pCfFccX44SzbUEprq201bEywBDOo+Jrm3N7/5gXAmramLxG5CShX1cJAP09VH1fVfFXNT01N7cLtTai07T8/cUQCV0/o/t/ZwrxMSqtOs76ksvOTjTHdEsygUgqM8vicBbS3ZOzteDR9AZcDN4tICU6z2RwReT6AzzNh5L09x9h1pJYvXZnT7pIs/rh+ykgGx0SybIM1gRkTLMEMKuuBXBHJEZEYnMCx3PskEUkCrgZebUtT1YdVNUtVs93r/q6qd3XyvOXA7SISKyI5QC6wLjBFMaH0xHtFpCbEcvO0jB7dJz4mivkXpvPXrWWcbrStho0JhqAFFVVtBh4AXscZwbVEVbeLyGIRWexx6m3AG6rq17AcEblNREqB2cBfReR193nbgSXADuA14Cuqaj85wtzOshre23OMey7LJjaq54P5FuVlcbKhmTd2HAlA7owx3kR14HZa5ufna0FBQaizYTrw/y3ZzMqtZXz48ByS42N6fL/WVuXKn73NuLQh/P6LNjjQmO4QkUJVzfd1zGbUmz7raE09yzcf4jMzRwUkoABEuFsNv7+ngqM1ttWwMYFmQcX0Wc9+UEJLq/LFy7s3jLg9C/MyaVV4xeasGBNwFlRMn3SqoZk/frSfeVNHMnpYfEDvPTZ1CNNHJ7N0QykDufnXmGCwoGL6pCUFB6mpb+7ykiz+WpSXxcdHT7Ldtho2JqAsqJg+p7mllafeLyZ/zFDyRg8NyjNuuiidmMgIltoik8YElAUV0+e8tv0IpVWnue+q4NRSAJLjY5g7OY3lmw7TZFsNGxMwFlRMn6KqPPFuEdnD4pk7aURQn7VwehbHTzWyeretAWdMoFhQMX3K+pIqNpee4N4rxxIZ0f0lWfxx9cRUhg2OYdlGawIzJlAsqJg+5Yn3ihgaH82n8rKC/qzoyAhunpbB33aUU13XGPTnGTMQWFAxfUZRxUn+tvMon7t0DHExvbO/2qK8LBpbWllhWw0bExAWVEyf8dT7xURHRvC52dm99swpGYlMHJFgo8CMCRALKqZPOH6ygZcLS1k4PZPUhNhee66IsGhGJhsPVFNUcbLXnmtMf2VBxfQJz390gIbm1m7v7NgTt0zLJEKwfVaMCQALKibk6pta+P2HJcy5II3xaQm9/vwRiYO4IjeVv2w8ZFsNG9NDFlRMyC3bcIjjpxq5L0hLsvhjUV4mh6pP81Hx8ZDlwZj+IKhBRUTmichuEdkrIg/5OP6giGxyX9tEpEVEUjyOR4rIRhFZ4ZGWIiJvisge98+hbnq2iJz2uN9jwSybCYzWVuXJ94uYmpnIpWNTOr8gSD45eSRDYqOsCcyYHgpaUBGRSOBRYD4wGbhDRCZ7nqOqj6jqNFWdBjwMrFbVSo9Tvo6za6Snh4C3VDUXeMv93GZf2/1UdTGmz/v7rnKKKk5x35Vje7T/fE/FxURy44XprNpaRl1jc8jyYUy4C2ZNZRawV1WLVLUReAm4pYPz7wBebPsgIlnAjcCTXufdAjznvn8OuDVQGTa974n3ishIGsQNF6aHOisszMvkVGMLr2+3rYaN6a5gBpVM4KDH51I37TwiEg/MA5Z6JP8S+DbgvdrfCFUtA3D/TPM4luM2l60WkSvbedb9IlIgIgUVFbbmUyhtKa1mbXElX7wih+jI0HfvzcxOYVRKnDWBGdMDwfyf7Ksto72hNQuANW1NXyJyE1CuqoVdeF4ZMFpVpwPfAl4QkcTzMqD6uKrmq2p+ampqF25vAu2J94pJiI3iMzNHhTorQNtWw1m8v/cYZSdOhzo7xoSlYAaVUsDzp0UWcLidc2/Ho+kLuBy4WURKcJrN5ojI8+6xoyKSDuD+WQ6gqg2qetx9XwjsAyYEpigm0A5W1rFyaxl3XDKahEHRoc7OGYvyMlGFVza290/VGNORYAaV9UCuiOSISAxO4FjufZKIJAFXA6+2panqw6qaparZ7nV/V9W73MPLgbvd93e3XSciqe7gAERkLJALFAWjYKbnnllTggD3XJYd6qycY8ywweSPGWpbDRvTTUELKqraDDwAvI4zgmuJqm4XkcUi4jky6zbgDVU95eetfwpcJyJ7gOvczwBXAVtEZDPwMrDYaySZ6SNOnG7iT+sPcNNF6WQkx4U6O+dZNCOLveUn2XroRKizYkzYiQrmzVV1JbDSK+0xr8/PAs92cI93gHc8Ph8HrvVx3lLO7eg3fdSL6w5wqrElaPvP99QNF6bz/eXbWVpYykVZyaHOjjFhJfRDbsyA0tjcyrNrSrhs3DCmZiaFOjs+JcVF88nJI1i++TCNzbbVsDFdYUHF9KoVWw5zpKY+qPvPB8KivCyq6pp4e3d5qLNiTFixoGJ6jaryxHvF5KYN4ZoJfXs495W5wxk+JJZlts+KMV1iQcX0mjV7j7OzrCbkS7L4IyoyglunZfD3XeVUnbKtho3xlwUV02ueeK+I4UNiuWV6Rqiz4peFeVk0tSj/t8XmrBjjLwsqplfsPlLL6o8ruOeyMcRG9c7+8z01OSORSemJLLVlW4zxmwUV0yueeK+IuOhI7rxkTKiz0iWL8jLZfLCaveW21bAx/rCgYoKuvKaeVzcd4h/ysxg6OCbU2emSm6dlEBkh1mFvjJ8sqJige/aDEppblXuv6P3953sqLWEQV+UO5y8bD9FiWw0b0ykLKiaoTjU088e1B7h+8kjGDBsc6ux0y6IZWZSdqOejIttq2JjOWFAxQfXngoOcON3U5yc7dmTupBEkDIpiaaE1gRnTmU6DioiMEJGnRGSV+3myiNwb/KyZcNfSqjy1ppi80cnMGDM01NnptkHRkdx0UQarth3hVINtNWxMR/ypqTyLs9Jw2+SCj4FvBCk/ph95ffsRDlae5v4wrqW0WZSXyemmFlZts62GjemIP0FluKouwd3W113SviWouTJhT1V5/N0ixgyL57rJI0OdnR6bMWYoY4bF2ygwYzrhT1A5JSLDcLcCFpFLAdtownSocH8Vmw5Wc+8VOURG9O0lWfwhIiycnsWHRcc5VG1bDRvTHn+CyrdwdlscJyJrgN8DX/Xn5iIyT0R2i8heEXnIx/EHRWST+9omIi0ikuJxPFJENorICo+0FBF5U0T2uH8O9Tj2sPus3SJyvT95NMHx+LtFJMdH86kZWaHOSsAsPLPVsM2wN6Y9HQYVd3veq93XZcA/AlNUdUtnN3avfRSYD0wG7hCRyZ7nqOojqjpNVacBDwOrvXZr/DrOrpGeHgLeUtVc4C33M+69bwemAPOA37VtL2x6V/GxU7y58yh3XTKG+Jig7gPXq0alxDMrJ8W2GjamAx0GFVVtAW5R1WZV3a6q21S1yc97zwL2qmqRqjYCLwG3dHD+HcCLbR9EJAu4EXjS67xbgOfc988Bt3qkv6SqDapaDOx182B62VPvFxEdEcHnLwuvJVn8sSgvk6KKU2w6WB3qrBjTJ/nT/LVGRH4rIleKSF7by4/rMoGDHp9L3bTziEg8Tu3CczvgXwLfxh0g4GGEqpYBuH+mdeV5InK/iBSISEFFRYUfxTBdUXmqkZcLS7l1egZpCYNCnZ2Au+HCdGKjIlhmi0wa45M/QeUynCalHwH/5b5+7sd1vnpn22szWACsaWv6EpGbgHJVLfTjOV16nqo+rqr5qpqfmtq3N4oKN8dPNvCFZ9bR1KLc10f3n++phEHRXD9lJMs3H6ah2QZBGuOt0wZvVf1EN+9dCozy+JwFtLcxxe14NH0BlwM3i8gNwCAgUUSeV9W7gKMikq6qZSKSDrTt99qV55kAO1R9ms89tZZDVaf537tmkDsiIdRZCppFM7JYvvkwb+8qZ97U9FBnx5g+xZ8Z9Uki8ou2JiMR+S8RSfLj3uuBXBHJEZEYnMCx3Nf9cQYCvNqWpqoPq2qWqma71/3dDSi497jbfX+3x3XLgdtFJFZEcoBcYJ0f+TQ9tOdoLYt+9wEVtQ384d5LmDt5RKizFFRXjB9OWkIsLxdaE5gx3vxp/noaqAU+7b5qgGc6u8idJPkAzmz8ncASVd0uIotFZLHHqbcBb6jqKT/z/FPgOhHZA1znfkZVtwNLgB3Aa8BX3IEGJogK91fxqcc+pEWVJf84m1k5KZ1fFOYiI4Tbpmfyzu5yjp9sCHV2jOlTpLOhkSKyyR3y22FaOMrPz9eCgoJQZyNsvb27nC8/v4ERibH84d5LGJUSH+os9ZrdR2q5/pfv8v0Fk/nC5eG3pL8xPSEihaqa7+uYPzWV0yJyhcfNLgdsSvEA98rGQ9z3XAFjUwfz58WXDaiAAjBxZAJTMxNtFJgxXvwJKv8EPCoiJSJSAvwWWNzxJaY/e/r9Yr7xp03MzE7hpfsvJTUhNtRZComF07PYeugEHx+tDXVWjOkzOg0qqrpJVS8GLgIuUtXpqro5+FkzfY2q8sjru/jRih3MmzKSZ74wk4RB0aHOVsjcPC2DqAhhqS0yacwZ/oz++ncRSVbVGlWtEZGhIvL/eiNzpu9obmnl4WVbefTtfdwxazSP3pnHoOiBvQrO8CGxXDMxlVdsq2ETRlpblZ1lNew4XBOU+/uzMNN8Vf3Xtg+qWuXOH/luUHJk+pz6pha+/tJGXt9+lK/OGc+3rpuASPivPBwIC/Oy+NvOctbsPcZVE2wyrel7Gppb2Fp6gnUllawvrqRgfxW19c3MmzKSxz43I+DP8yeoRIpIrKo2AIhIHDAwG9EHoJr6Ju57roC1xZU20smHayelkTgoimUbSi2omD6htr6Jwv1VrC+pZH2JswVFY7Oz2tW41MHcdFE6M7NTmJkdnOH//gSV54G3ROQZnGVPvsjZBR1NP1ZR28DdT6/j46O1/PIz07h1us+l2wa02KhIFlycwdINpdTWNw3oPiYTGuW19RSUVLGuuJL1JZXsLKuhVZ35VFMzEvncpWPcIDKUYUOCXx/wZ5mWn4nIFmAuzvpaP1bV14OeMxNSB47X8bmn11Je08CTd+dzzcS0zi8aoBbNyOKPaw+watsRPp0/qvMLjOkmVWX/8bozTVnrSyopOV4HwKDoCKaPGsoDc3KZlZ3C9NHJDI7t/a0nOn2iiAzGmfH+mohMBCaKSHQXlsA3YWZnWQ2ff3odTS2t/PG+S8gbPbTziwaw6aOSyRk+mKWFpRZUTEC1uJ3qTlOW05xVUeus4pAcH03+mBQ+e8loZmanMDUziehIf2aJBJc/Yexd4Ep3h8W/AQXAZ4A7g5kxExrriiu597n1DImN4oUvze7XC0MGioiwKC+Tn7/xMQcr6wbcRFATOPVNLWw+WE3Bfqc5a8P+KmobmgHITI7j8nHDmJnj9IeMTx1CRB/cqtufoCKqWici9wK/cZvDNgY7Y6b3vbnjKA+8sIHMoXH84d5LyEyOC3WWwsat052g8peNh/jatbmhzo4JEydON7Fhf9WZ5qwtpSdobHE61SeMGMKCaRnMyk5hZk5K2Px/9CuoiMhsnJrJvV24zoSRJQUHeXjZVqZmJPLMF2aRMjgm1FkKK1lD45k9dhjLNpTy1Tnjbci18eloTf2ZDvV1xZXsPlqLKkRFCFMzk7jn8mxmZqeQP2YoQ8P0/6A/weHrOPvH/8VdZXgs8HZws2V602Or9/HTVbu4Mnc4j901IySde/3BwrxMHnx5CxsOVDFjTP9frdl0TFUpOnaK9cWVTk2kpJKDlc6yifExkeSNHsr8qenMzBnK9FFDiYvpH5OJ/Rn99S5Ov0rb5yLga8HMlOkdra3KT1/bxePvFnHTRen84tPTiIkKfUdfuJp/YTrfe3U7SzccsqAyADW3tLKjrIZ1xZUUlFRRsL+SYycbAUgZHMPM7KHcPdupiUzOSOwTnerBYL+SDlCqyr8s3cKfC0v5/Owx/GDBlD7Z6RdOhsRGMW/qSFZsPsz3bpo84Jex6e9ON7aw6WD1mZFZG/ZXcarR2cJpVEocV+WmnulUH5c6eMA0iQY1qIjIPOBXQCTwpKr+1Ov4g5wdRRYFTAJSgTqc2lGsm/6yqn7fveZi4DFgCFAC3OmuSZaNsxnYbvd+H6mqrabcjlXbjvDnwlK+fM04Hrx+4oD5Bx9sC/My+cvGQ7y1s5wbL7KthvuT6rpGCkqcmerrSirZdugETS2KCEwckcDCvCw3iAwlPSk8OtWDwZ95Kper6prO0nxcFwk8irM7YymwXkSWq+qOtnNU9RHgEff8BcA3VbVSnJ9wc1T1pIhEA++LyCpV/Qh4EvhnVV0tIl8EHgT+zb3lvv6weViw1Te18O8rd3LByARbxyvALhs3nJGJg1i2odSCSpg7XH36TIf6+pJKPj56EoDoSOGirGTuvWIss3KGMmN0CknxtpJCG39qKr8B8vxI8zYL2Ov2wSAiLwG34Gz368sdwIsA6mxHedJNj3ZfbcvATuRsH8+bONsV/xvGb0+vKaa06jR//NIlRPXTdt1QiYwQbp2eyRPvFVFR2zBg95oJN6rK3vKTrG+riRRXcqja6VQfEhtF3pihLLgog5k5KUwblWxNmx1oN6i4w4gvA1JF5FsehxJxmrM6kwkc9PhcClzSzrPigXk4e9q3pUUChcB44FFVXese2gbcDLwK/APgOYU5x51DUwN8V1Xf8/Gs+4H7AUaPHu1HMfqX8tp6Hv37XuZOGsHl44eHOjv90qK8TB5bvY/lmw9z7xW2AGdf1NTSyvbDNWdGZhWUVFJV5ywSMnxIDDOzU7j3ihxm5aRwwcgE++WrCzqqqcTg9FtEAZ7TqmuAT/lxb19tKu1tOrEAWKOqlWdOVG0BpolIMvAXEZmqqttwFrT8tYh8D1gONLqXlAGjVfW4iMwAXhGRKap6zqYBqvo48Dg4e9T7UY5+5b9e/5jGlla+c+OkUGel38odkcBFWUksLSy1oNJH1DU2s/FA9ZmmrI0Hqjnd5HSqjxkWz7WTRpyZZJg9LN6ahHug3aCiqquB1SLyrKruBxCRCGCI9w/qdpRybi0iCzjczrm34zZ9+chHtYi8g1OT2aaqu4BPuvmZANzontcANLjvC0VkHzABZ1kZA2w7dIIlhQe59/IccoYPDnV2+rVFeVl8f/l2dpbVMCk9MdTZGXAqTzU6o7KKK1m/v4rth07Q3Op0qk8amchnZo46s3JvWuKgUGe3X/GnT+U/RGQx0ILTHJUkIr9wO9k7sh7IFZEc4BBO4Pis90kikgRcDdzlkZYKNLkBJQ5nheT/dI+lqWq5G+C+izMSrO2aSlVtcSdo5gJFfpRvQFBVfrxiB8lx0XzVlhEJugUXZ/D//rqDZRtK+c6Nk0OdnX5NVSmtOk3B/krWFTt9InvLnS7ZmKgIpmUlc/9VY5mZk8KMMUNJtO0JgsqfoDLZHbJ7J7AS+Bec4NJhUFHVZhF5AKcjPRJ42p2Rv9g9/ph76m04qyCf8rg8HXjO7VeJAJao6gr32B0i8hX3/TLgGff9VcCPRKQZJwAu9mxOG+he336EtcWV/PjWqSTF2X+qYEsZHMMnJqbxyqbD/Mu8C6xNPoBaW5U95SfPWf697EQ9AAmxUczIHspt0zOZlZPChZlJ1qney8QZaNXBCSLbgWnAC8Bv3aG8m1X14l7IX1Dl5+drQUH/bx1raG5h7i9WExcdycqvXWk/4HrJa9uOsPj5Qp75wkw+YfvRdFtjcytbD50405xVsL+KE6edTvW0hFhm5qQ4/SHZKUwcmUCkTeINOhEpVNV8X8f8qan8L84kw83AuyIyBqez3oSJZ9aUcLDyNH+4d5YFlF4054I0kuOjWbbhkAWVLjjZ0MyGM9vhVrLpYDX1Tc7KvWOHD2belJFnJhmOTrFO9b7Gn7W/fg382iNpv4h8InhZMoFUUdvAb/++l2svSOPKXNtDvTfFREVw88UZ/Gn9QWrqm6wtvx3VdY18VHT8TH/IjrIaWlqVCIEpGUncMWs0s7JTyM9OsXk/YcCfGfUjgH8HMlR1vohMBmYDTwU7c6bnfvHmx9Q3tfCvNoQ4JBbmZfH7D/ezcksZt88aePOi2lN5qpHXtx9h5dYyPtx3nOZWJTYqgmmjkvnyNeOYmZ1C3pihDLEVs8OOP39jz+J0hn/H/fwx8CcsqPR5Ow7X8Kf1B7jnshzGpQ4JdXYGpIuzkhiXOphlGw4N+KBy7GQDr28/wqqtR/iw6DgtrcrolHi+dOVY5k5K48KsJGKjrFM93PkTVIar6hIReRjOjOpqCXK+TA+1DSFOjIvm6zaEOGREhIV5WTzy+m4OHK9j9LCBtdVweW09r28/ysotZawtPk6rQs7wwSy+eizzp6YzJSPR+kT6GX+CyikRGYY7G15ELgVOBDVXpsfe3HGUD4uO86NbpthidyF22/RMfv7GbpZtLOUbcyeEOjtBV15Tz6ptTtPWupJKVGFs6mC+8onx3HBhOheMTLBA0o/5E1S+hbMcyjgRWYOzNP0/BDVXpkcamlv4ycqd5KYN4bMDvMmlL8hIjuOyccNYtuEQX782t1/+QD1yop5V28pYubWMgv1VqEJu2hC+NieXGy5MZ8KIIf2y3OZ8/gSV7Tgz3ifirOe1G2dCoumjfv/BfvYfr+O5L9oQ4r5iUV4W31qymYL9VczM7h+7Qh6uPn2mRlK4vwpw9hX5xrUTuOHCkeSOSOjkDqY/8ieofKiqeTjBBQAR2UDnS9+bEDh+soFfv7WHT0xM5eoJNoS4r7h+ykjiY7axtLA0rINKaVUdq7YeYeW2MjYeqAZgUnoi//zJCcybms74NBsQMtB1tPT9SJzl6+NEZDpnVx1OBAZWb2MY+cWbH1PX1GLrTfUxg2OjmD81nb9uKeMHN08Jq6VDDlbWsXKr07S1udTpTp2SkciD10/khgvTbXFSc46OairXA/fgrC78X5wNKjXAvwY3W6Y7dh2p4cV1B/j87Gz7jbEPWpSXydINpbyx4yg3X5wR6ux0aP/xU/x1axmrth5h6yEnkFyUlcS/zLuAGy4cyZhhFkiMbx0tff8czqKOi1R1aS/myXRD2xDihEHRfGOuDSHuiy4dO4yMJGer4b4YVIqPnWLl1jL+uqWMHWXOSkwXj0rmX2+4gPlT0xmVYg0UpnP+LNNiASUMvLWznDV7j/ODBZNJjo8JdXaMDxERwm15mfzPO/sor60nLSH0+3jsLT/Jqq1l/HVrGbuO1AKQNzqZ7944iXlTR5I11AKJ6RpbA6EfaGxu5ScrdzIudTB3Xjom1NkxHViYl8Wjb+/j1Y2Hue+qsSHJw56jtWeatnYfdQJJ/pihfO+mycybOpKM5LiQ5Mv0DxZU+oHff1hC8bFTPHPPTKJtCHGfNi51CNNGJbN0Q2mvBRVVZffRWlZudYb/7i0/iQjMzE7hBwsmM29qOiOTQl9rMv2DX0FFRC4Dsj3PV9Xf+3HdPOBXOJt0PamqP/U6/iBwp0deJuFMrqwD3gVi3fSXVfX77jUX4+z2OARnSf4727Y3dpeSuRdnk66vqerr/pQvnFWeauRXb+3hqgmpXDPRhhCHg0V5mfzbq9vZfvgEUzKSgvIMVWVnWa0zamtbGUUVp4gQmJWTwt2zp3D9lJG2ja4JCn9WKf4DMA7YhPPDGpwlWzoMKu6ujY8C1+HsV79eRJar6o62c9wtiR9xz18AfFNVK8WZejtHVU+KSDTwvoisUtWPgCeBf3Y3C/si8CDwb+7qybcDU4AM4G8iMkFV+/U6Zf/95sfUNbbw3Rsn2YzlMLHg4gx+tGIHyzYcCmhQUVW2H65h5dYyVm07QvExJ5DMHjeML16ew/VTRtrS8Sbo/Kmp5ONsKdzxFpHnmwXsVdUiABF5CbgF2NHO+XcALwK4zzrppke7r7bnT8SpxQC8ibNd8b+5935JVRuAYhHZ6+bhwy7mO2wcrannhXUH+Oys0Uyw2cthIzk+hmsvGMGrmw7x0PwLetRkqapsPXSClVuPsGpbGfuP1xEZIVw2bhj3XTmW66eMYNgQCySm9/gTVLYBI4GyLt47Ezjo8bkUuMTXiSISD8wDHvBIiwQKgfHAo6q61iM/NwOv4qxBNsrjeR95PS/Tx7PuB+4HGD06vNfF+shdPvwzM0d1frLpUxbNyOK17Ud4b08Fcy4Y0aVrVZXNpSfOTEgsrTpNVIRw2fjhfPmacVw3eSQpg20EoAkNv5a+B3aIyDqgoS1RVW/u5DpfbTHt1XYWAGtUtdLj/i3ANBFJBv4iIlNVdRvwReDXIvI9nIUuG7vyPFV9HHgcnD3qOylDn7auuJIhsVFMSk8MdVZMF109IZWUwTEsLTzkV1BpbVU2Hqxmldu0daj6NNGRwhXjh/O1a3P55OQRNpTc9An+BJUfdPPepZytRYAzM/9wO+fejtv05U1Vq0XkHZyazDZV3QV8EkBEJgA3duN5/cK64kpmjBlKZIT1pYSbtq2GX1h3gBN1TT63J2htVTYcqDrTtFV2op6YyAiuzB3Ot66bwNxJI2xbA9Pn+DP5cXU3770eyBWRHOAQTuD4rPdJIpKEswryXR5pqUCTG1DigLnAf7rH0lS1XEQigO/ijAQDp9bygoj8AqejPhdY182893mVpxrZU36SW6ef18JnwsSivCye/aCEFVsPc+clzvyillalcH+V29lextGaBmKiIrgqN5Vvz5vItZNG2F73pk/zZ/TXpcBvcIb7xuAMDz6lqh22ubg7RD6A05EeCTytqttFZLF7vC0Y3Aa8oaqnPC5Px1kiJhJnmf0lqrrCPXaHiHzFfb8MZ6tj3HsvwRkI0Ax8pT+P/Fpf4rQUXpITviveDnRTMxOZMGIILxeWMnb4EFZtc5q2KmobiI2K4JqJqdxwYTpzLkgjwQKJCRPS2aAuESnAqWX8GWck2OeBXFUN+0Ul8/PztaCgoEf3eOLdIoYNiWFhXlaAcuWfH6/YwR8+2s/WH3zS9vUOY4+t3sdPV+0CYFB0BJ+YmHYmkAyOtbnJpm8SkUJVzfd1zK9/taq6V0Qi3d/8nxGRDwKawzD2cmEpmUPjej2orC+pZPqoZAsoYe6OWaM5VtvA9NFD+cQFqcTHWCAx4c2ff8F1IhIDbBKRn+EMLbZ1r4GtpSdoam2lqaW1V597sqGZbYdO8MAnxvfqc03gJcVF892bbO8b03/4M+vqc+55DwCncEZYLQpmpsLB6o8rWPDb9ymqOEVDc+8GlcL9VbQqzLT+FGNMH+PP6K/97gisdFX9YS/kKSwcOH52XEFv11TWF1cSGSHkjR7aq881xpjOdFpTcdfk2gS85n6eJiLLg5yvPs9zna3GXq6prCuuZGpmknXkGmP6HH+av36As4ZWNYCqbsJZsXhAiwhRUKlvamHTwWpmZVstxRjT9/gTVJpV9UTQcxJmPBcE7s3mr80Hq2lsaWVWzrBee6YxxvjLn6CyTUQ+C0SKSK6I/AYY8EOKPRdG6aymsv3wCarrGjs8x19tkx5nWk3FGNMH+RNUvoqzR0kDzvpcNcA3gpinsOBZU2nsoKZS19jMwt99wP+s3heQ564trmTiiARbPNAY0yf5M/qrDviO+zIufzvq15dU0dDcyqGq0z1+ZnNLKxv2V/X6REtjjPGXP2t/5QP/yvnbCV8UvGz1fed01HdQU1mz9xgA5bUN7Z7jrx1lNZxqbGGWzU8xxvRR/oxJ/SPOlr1bgd4dO9uHefapNLW0v37a+3vcoFJT3+Nnrit2+lMsqBhj+ip/gkqFqg74eSneIjx6o1palZZWPW9fk+MnG9hRVkNMVATltQ2oao/2kV9bXMmYYfGMSBzU7XsYY0ww+dNR/30ReVJE7hCRhW2voOesjxOvjSZ99at8sO84ANdNGkFdYwsnG5q7/bzWVqWgpJJZ2VZLMcb0Xf7UVL4AXABEc7b5S3H2MhmwvCscjS2txHHuisFr9h4jYVAUcy5I469byyivbej2vhh7K05SVddkTV/GmD7Nn6Bysape2J2bi8g84Fc4m3Q9qao/9Tr+IHCnR14mAalAHfAuEOumv6yq33evmYaz2+MgnM24vqyq60QkG9gJ7Hbv95GqLu5Ovv0s2zmfvWsqqsp7e45x2bhhpCc5zVXlNQ2MSx3Sreettf4UY0wY8Kf56yMR6fLa3O6ujY8C84HJODs2nnMfVX1EVaep6jTgYWC1qlbizImZo6oXA9OAee4OlAA/A37oXvM993ObfW33C2ZAAfDuGfGeVX+gso5D1ae5Yvxw0tw+kPLa7nfWry+uZERiLKNT4rt9D2OMCTZ/aipXAHeLSDHOD3sB1I8hxbOAvapaBCAiLwG34Gz368sdOJMrUWc7ypNuerT7ahtipUDbVsZJwGE/yhBwEZ3UVN53hxJfPn44wxNiAaem0h2qyrriSmblDOtRR78xxgSbP0FlXjfvnQkc9PhcClzi60QRiXef84BHWiRQCIwHHlXVte6hbwCvi8jPcWpal3ncKkdENuLM+v+uqr7Xzbx3ylefiqc1e4+RkTSInOHOfmZx0ZEc7eaw4oOVpzlSU2+LSBpj+jy/9lPp5r19/Urd3oSOBcAat+mr7bktwDQRSQb+IiJTVXUb8E/AN1V1qYh8GngKmIuzI+VoVT0uIjOAV0RkiqrWnJMpkfuB+wFGjx7dzaJBhHdQ8aiptLQqH+w7znWTRpypWaQlxnZ7AuS6krb+FFtE0hjTt/nTp9JdpTi7RLbJov2mqttxm768qWo18A5na0x3c3bk2Z9xmtlQ1QZVPe6+LwT2ARN83O9xVc1X1fzU1NQuFOdc53XUe9RUdhyuobquiStyh59JS0uI7XZNZV3xcZLjo8lN614nvzHG9JZgBpX1QK6I5Lh73N8OnDeJUkSSgKuBVz3SUt0aCu6uk3OBXe7hw+75AHOAPR7XRLrvxwK5QFHgi+WbZ01lbbEzP2X2uLM1i7TEQVR0t6ZSXEn+mBQivKtHxhjTxwRt60BVbRaRB4DXcYYUP62q20VksXv8MffU24A3VPWUx+XpwHNukIgAlqjqCvfYfcCvRCQKqMdtygKuAn4kIs1AC7DYszktkA5Xn+Y/X9t1Tprn6K/y2gZioyJISzg78z0tIZbV3Qgq5TX1lByv485LxnQ/w8YY00uCuh+tqq4EVnqlPeb1+VngWa+0LcD0du75PjDDR/pSYGmPMuynitoGiipOnZPmWVM5UddEcvy5kxxHJA7iZEMzpxqau7QN8Nn+FJufYozp+4LZ/NVveQ8nhnNrKtWnG0mOO3e/k7S2YcVdrK2sK64kPiaSKRmJnZ9sjDEhZkGlG3xNFWnwqKlU1zWR5FVTaWsK62pn/briSmaMGUpUpP1VGWP6PvtJ1Q2+gso5zV+nm0iO827+6npNpbqukd1Ha20RSWNM2LCg0g3eKxTDuXuqVNc1kRTnu6bSlX1VCkqqULX+FGNM+LCg0g0RPr61xuaWM+9PnD6/oz4xLopYd18Vf60rqSQmMoKLRyV3N6vGGNOrLKh0g6+O+rbJj/VNLZxuaiE5/tyOehFxZtV3oaayrriSi0clMSg6svOTjTGmD7Cg0g2+piC2NX/VnG4COK/5C5wmsKN+Lip5qqGZbYdOWNOXMSasWFDpBl8rBbeN/qp2g4p38xc4nfX+Ln+/8UA1za3KTOukN8aEEQsq3eBrtZS2eSrVdR3XVPztU1lXUkmEwIwxtjKxMSZ8WFDpBl81lbYhxSfaaipekx/BWam4tr6Z040t5x3ztq74OFMykrq9/bAxxoSCBZVu8FVTaQsq1XWNgO/mrzPDijtpAmtobmHjgWpr+jLGhB0LKt3ge57KuTUV7xn1cHapls4667cdOkFDc6t10htjwo4FlW7wbv0S8aypNBEZIST4WDRyhJ971a8tdhaRnGk7PRpjwowFlW7w3tckLjqShraO+tONJA6K8tnvkubnXvXv7KrggpEJDBsSG6AcG2NM77Cg0g3e4SIuOpKmMx31zedNfGyTHB9NTGQERzuoqZTX1rN+fyXzpo4MVHaNMabXWFDpBu8Z9XExkWdm1FfXNfocTgzOqLHUhFgqOqipvL79KKpww4XpgcuwMcb0kqAGFRGZJyK7RWSviDzk4/iDIrLJfW0TkRYRSRGRQSKyTkQ2i8h2EfmhxzXTROQj95oCEZnlcexh91m7ReT6YJXLe/RXfEzkOR31vkZ+tUlLjO2wprJqaxnjUgfbfvTGmLAUtKDibgX8KDAfmAzcISKTPc9R1UdUdZqqTgMeBla7WwA3AHNU9WJgGjBPRC51L/sZ8EP3mu+5n3HvfTswBZgH/K5tz/rAF+7cj3HRked01Hsve+9pRMKgdvtUjp9s4KOi48yfmu6zT8YYY/q6YNZUZgF7VbVIVRuBl4BbOjj/DuBFAHWcdNOj3Vfb2vIKtG2DmAQcdt/fArykqg2qWgzsdfMQcD6bvzwmP7bX/AVOTaW9WfVv7DhKq8L8C60/xRgTnoIZVDKBgx6fS92084hIPE7tYqlHWqSIbALKgTdVda176BvAIyJyEPg5Tg3H7+eJyP1us1lBRUVFN4rlu6O+sUVpaVVq6ptIaqejHpwRYCdON1HfdP6s+lXbjjBmWDyT023rYGNMeApmUPHVfqM+0gAWAGvcpi/nRNUWt4krC5glIlPdQ/8EfFNVRwHfBJ7qyvNU9XFVzVfV/NTUVP9K4sV3TaWF2vomVOmw+SvNnatS4VVbqa5r5IO9x5g3daQ1fRljwlYwg0opMMrjcxZnm6q83Y7b9OVNVauBd3BqMgB3A8vc93/mbBNXV57XI97zVGKjImlobj2zmGSHHfVnZtWf21n/5o6jNLcqN0y1UV/GmPAVzKCyHsgVkRwRicEJHMu9TxKRJOBq4FWPtFQRSXbfxwFzgV3u4cPu+QBzgD3u++XA7SISKyI5QC6wLtCFAoj0CiojEgdxtKae46ec2kdHQeXsrPpzayqvbTtCZnIcF2UlBTi3xhjTe85fSyRAVLVZRB4AXgcigadVdbuILHaPP+aeehvwhqqe8rg8HXjOHb0VASxR1RXusfuAX4lIFFAP3O/eb7uILAF2AM3AV1S18+WAuyHSq3lqXOpgmlqUbYdqAN/L3rc5O6v+bE2lpr6J9/Yc4/Ozx1jTlzEmrAUtqACo6kpgpVfaY16fnwWe9UrbAkxv557vAzPaOfYT4CfdzrCfvH/uj0115pQU7q8CIMnHsvdthsbHEBUhHPWoqfx9ZzmNLa026ssYE/ZsRn03eDd/jUsdDMCGA05Q6aj5KyJCSEuIPWeuysqtZYxIjGX6KFtA0hgT3iyodIN381dyfAwpg2MorToNdNz8BZCaOOjMSsWnGppZ/XEF86emnzcAwBhjwo0FlW7w9cO/rbYyJDaK6MiOv9YRHjWVt3eX09DcagtIGmP6BQsqATJ2uNOv0lktBdpm1Ts1lVVbjzB8SIzt8miM6RcsqATIWLem4ldQSRhEVV0TJ0438fbucq6fMvK8fhpjjAlHFlR6aO9P5gNnR4B11EnfZkSiM6z45cJS6hpbmG8THo0x/YQFlR5qq2G09an4E1TSEpwJkL//sISh8dFcMtaavowx/YMFlR5qm6w4KiWeqAjxu08FYP/xOj45eWSnHfvGGBMugjr5cSCJjozgJ7dNZUpG58ustNVUAObZhEdjTD9iQSWAPjNztF/nDRscQ2SEEB8TyeXjhgc5V8YY03ssqIRARIQwdvhg8rNTiImypi9jTP9hQSVEXv6nyxgUbQHFGNO/WFAJEX869I0xJtzYr8rGGGMCxoKKMcaYgAlqUBGReSKyW0T2ishDPo4/KCKb3Nc2EWkRkRQRGSQi60Rks4hsF5EfelzzJ49rSkRkk5ueLSKnPY495v08Y4wxwRW0PhV318ZHgetw9o9fLyLLVXVH2zmq+gjwiHv+AuCbqlopzozCOap6UkSigfdFZJWqfqSqn/F4xn8BJzweu09VpwWrTMYYYzoWzJrKLGCvqhapaiPwEnBLB+ffAbwIoI6Tbnq0+1LPk93A8+m2a4wxxoReMINKJnDQ43Opm3YeEYkH5gFLPdIi3aatcuBNVV3rddmVwFFV3eORliMiG0VktYhc2c6z7heRAhEpqKio6HKhjDHGtC+YQcXXWu7qIw1gAbBGVSvPnKja4jZlZQGzRGSq1zVnajauMmC0qk4HvgW8ICKJ52VA9XFVzVfV/NTUVP9LY4wxplPBDCqlwCiPz1nA4XbOvZ12mrFUtRp4B6cmA4CIRAELgT95nNegqsfd94XAPmBCt3NvjDGmy4IZVNYDuSKSIyIxOIFjufdJIpIEXA286pGWKiLJ7vs4YC6wy+OyucAuVS31uibSfT8WyAWKAl0oY4wx7Qva6C9VbRaRB4DXgUjgaVXdLiKL3eNtQ35vA95Q1VMel6cDz7lBIgJYoqorPI77qtlcBfxIRJqBFmCxZ3OaMcaY4AvqMi2quhJY6ZX2mNfnZ4FnvdK2ANM7uO89PtKW4tHRb4wxpvfZjHpjjDEBY0HFGGNMwFhQMcYYEzC29H03Lf2n2ew5erLzE40xZgCxoNJNM8akMGNMSqizYYwxfYo1fxljjAkYCyrGGGMCxoKKMcaYgLGgYowxJmAsqBhjjAkYCyrGGGMCxoKKMcaYgLGgYowxJmBEtb3NGPs/EakA9nfz8uHAsQBmJxxYmQcGK/PA0JMyj1FVn1vnDuig0hMiUqCq+aHOR2+yMg8MVuaBIVhltuYvY4wxAWNBxRhjTMBYUOm+x0OdgRCwMg8MVuaBIShltj4VY4wxAWM1FWOMMQFjQcUYY0zAWFDpBhGZJyK7RWSviDwU6vwEgoiMEpG3RWSniGwXka+76Ski8qaI7HH/HOpxzcPud7BbRK4PXe57RkQiRWSjiKxwP/frMotIsoi8LCK73L/v2QOgzN90/11vE5EXRWRQfyuziDwtIuUiss0jrctlFJEZIrLVPfZrEZEuZURV7dWFFxAJ7APGAjHAZmByqPMVgHKlA3nu+wTgY2Ay8DPgITf9IeA/3feT3bLHAjnudxIZ6nJ0s+zfAl4AVrif+3WZgeeAL7nvY4Dk/lxmIBMoBuLcz0uAe/pbmYGrgDxgm0dal8sIrANmAwKsAuZ3JR9WU+m6WcBeVS1S1UbgJeCWEOepx1S1TFU3uO9rgZ04/xlvwfkhhPvnre77W4CXVLVBVYuBvTjfTVgRkSzgRuBJj+R+W2YRScT54fMUgKo2qmo1/bjMriggTkSigHjgMP2szKr6LlDpldylMopIOpCoqh+qE2F+73GNXyyodF0mcNDjc6mb1m+ISDYwHVgLjFDVMnACD5DmntZfvodfAt8GWj3S+nOZxwIVwDNuk9+TIjKYflxmVT0E/Bw4AJQBJ1T1DfpxmT10tYyZ7nvvdL9ZUOk6X+2L/WZctogMAZYC31DVmo5O9ZEWVt+DiNwElKtqob+X+EgLqzLj/MaeB/yPqk4HTuE0i7Qn7Mvs9iPcgtPMkwEMFpG7OrrER1pYldkP7ZWxx2W3oNJ1pcAoj89ZOFXpsCci0TgB5Y+qusxNPupWiXH/LHfT+8P3cDlws4iU4DRjzhGR5+nfZS4FSlV1rfv5ZZwg05/LPBcoVtUKVW0ClgGX0b/L3KarZSx133un+82CStetB3JFJEdEYoDbgeUhzlOPuSM8ngJ2quovPA4tB+52398NvOqRfruIxIpIDpCL08EXNlT1YVXNUtVsnL/Hv6vqXfTvMh8BDorIRDfpWmAH/bjMOM1el4pIvPvv/FqcPsP+XOY2XSqj20RWKyKXut/V5z2u8U+oRyyE4wu4AWd01D7gO6HOT4DKdAVONXcLsMl93QAMA94C9rh/pnhc8x33O9hNF0eI9LUXcA1nR3/16zID04AC9+/6FWDoACjzD4FdwDbgDzijnvpVmYEXcfqMmnBqHPd2p4xAvvs97QN+i7vyir8vW6bFGGNMwFjzlzHGmICxoGKMMSZgLKgYY4wJGAsqxhhjAsaCijHGmICxoGJMiIhIiYgM7+k5xvQlFlSMMcYEjAUVY3qBiLwiIoXunh73ex3Ldvc2eU5Etrh7ncR7nPJVEdng7nFxgXvNLBH5wF0U8gOPGfLGhJQFFWN6xxdVdQbObOWvicgwr+MTgcdV9SKgBviyx7FjqpoH/A/wz27aLuAqdRaF/B7w70HNvTF+sqBiTO/4mohsBj7CWcgv1+v4QVVd475/HmfZnDZti3sWAtnu+yTgz+4uf/8NTAlGpo3pKgsqxgSZiFyDs1LubFW9GNgIDPI6zXu9JM/PDe6fLThL1wP8GHhbVacCC3zcz5iQsKBiTPAlAVWqWuf2iVzq45zRIjLbfX8H8L4f9zzkvr8nILk0JgAsqBgTfK8BUSKyBaeG8ZGPc3YCd7vnpOD0n3TkZ8B/iMgaIDKQmTWmJ2yVYmNCzN2+eYXblGVMWLOaijHGmICxmooxxpiAsZqKMcaYgLGgYowxJmAsqBhjjAkYCyrGGGMCxoKKMcaYgPn/AY9bFgPbacZrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(gs_lr_report['param_classifier__alpha'],gs_lr_report['mean_test_score'])\n",
    "plt.xlabel('alpha')\n",
    "plt.ylabel('mean test score')\n",
    "plt.title('Mean test score with PCA')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1b. Lasso without PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_lr = Pipeline(steps=[('preprocessor', preprocessor),\n",
    "                      ('classifier', Lasso(alpha=1.0))])\n",
    "\n",
    "# Find the best alpha that gives the maximum test score\n",
    "\n",
    "alpha = np.logspace(0, 3, num=20, endpoint=True, base=10.0)\n",
    "param_grid=[{\n",
    "    'classifier__alpha': alpha\n",
    "}]\n",
    "\n",
    "gs_lr_wop = GridSearchCV(pipe_lr,n_jobs=4, param_grid=param_grid)\n",
    "\n",
    "gs_lr_wop.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Mean test score without PCA')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0OElEQVR4nO3dd3xV9fnA8c+TPcgggwBhhBGmypQlw1lxUn9aZ+uoVq3WUVt/tbV1/NraVltbB3WUulddVVScrWUJKCAoM2GFJZBAQhKyk+f3xzkJl/QGbkJuTnLzvF+v88q9Z9z7fG+S89zvON8jqooxxhjTWJjXARhjjGmfLEEYY4zxyxKEMcYYvyxBGGOM8csShDHGGL8sQRhjjPHLEoQxIUBE+ohIqYiEH2YfFZGBbRmX6dgsQZj/IiJbRKRKRNIarV/hnmSy2jieE0Vkeyu91n9E5JrWeK32RFW3qmoXVa2Fti2niNwjIi8cYZ8tIlLuJrHdIvK0iHTx2X66iMwTkRIRyReRuSJybqPXONH9+/vfYJXFHMoShGnKZuCS+iciciwQ6104oUFEIryOwUPnqGoXYDRwPPBLABG5AHgNeA7oBWQAdwHnNDr+CmCf+9O0BVW1xZZDFmALzj/vFz7r/gjcCSiQ5a6LdtdvBXYDjwOx7rauwLtAPlDoPu7l83r/AX4NLARKgI+AND+xxAPlQB1Q6i49cb7c3AFsBPYCrwIp7jExwAvu+iLgC5yTzm+BWqDCfZ1H/byf32PdbSnA08BOt0xv+Rz3A2ADzglsNtDTZ5sCNwK5wGZ33dnACvc9PgOOa+J3cS/wiPs4EjgA3O8+j3XL0hXIct8noqlyutuvd+MoBGYC4m4Lc3/necAenJN1krvtRGC7n7+RU4HpQBVQ7b7XysP8TZ3q8/wB929CcP5+bj/C32Sc+3dysft+Y73+P+kMi+cB2NL+Fp9//vXAUCAc2Ab05dAE8Rf3ZJgCJADvAL9zt6UC57v/2Ak43xDf8nmP/+Cc3Ae5J7r/AL9vIh5/J6hbgcU43zijgSeAl91t17mxxLmxjwESfd73msOU/XDHvgf8wz0hRwLT3PUnAwU434yjgUeAeT6vqcDH7ucU6+63BxjvvscV7mce7Seek4Gv3ceT3M9sic+2le7jLPd9Ipoqp7v9XSAZ6IOTvKe7276Pk+D6A12AN4HnD/P5b8E94QP3AC8E8jflPu4NrMb5gjDEjavfEY7/HvCN+3m9Azzs9f9JZ1isickczvPA5cBpwDpgR/0GERGcb80/VtV9qloC3IfzDQ9V3auqb6hqmbvtt8C0Rq//tKrmqGo5Tg1gZDNiuw64U1W3q2olzknqArcJpxonQQ1U1VpVXaaqxQG+rt9jRaQHcAZwvaoWqmq1qs51j7kMeEpVl7ux/ByY2Kiv5nfu51SO87k9oapL3Pd4FqgEJviJZxGQLSKpwFTg70Cm234/DZjr55jD+b2qFqnqVuBTDn7mlwEPquomVS11y3BxKzeJvSUiRcACnLjvw/mswTn5H84VwD/U6WN5CbhERCJbMTbjhyUIczjPA5cCV+I0OfhKx/mWvUxEitx//A/c9YhInIg8ISJ5IlIMzAOSG42y2eXzuAznm2ug+gL/9HnvtTjNKhlu3B8Cr4jIThG5vxknk6aO7Q3sU9VCP8f0xGmaAcA9we4FMn322dYo9p/Ux+7G39t9nUO4CWUpTjKYinNi/Qw4gZYliKY+80PK4D6OwPk8W8u3VTVZVfuq6g1u2fa623o0dZCI9AZOAl50V72N0xR4VivGZvywBGGapKp5OJ3VZ+I0OfgqwOkbGO7+0yerapI6nZAAPwEGA+NVNRHn5AZOm3OzQ/Gzbhtwhs97J6tqjKrucL/d36uqw3CaZc7GqQk19VoH36jpY7cBKSKS7OewnTgnfQBEJB7nm/EOn31833cb8NtGscep6stNhDUXpzlpFE6fyFzgdGAcTuL1W5TDlfNIZcBpgqrB6Vs6gPNlAAA3yacfxXv5Wo/zeZx/mH2+h3OuekdEdgGbcBLE5Yc5xrQCSxDmSK4GTlbVA74rVbUO+BvwZxHpBiAimSJyurtLAk4CKRKRFODuo4hhN5AqIkk+6x4Hfisifd33TheRGe7jk0TkWPdEVozTbFTr81r9m3qjpo5V1W+A94G/ikhXEYkUkfqk9xJwlYiMFJFonKaTJaq6pYm3+RtwvYiMF0e8iJwlIglN7D8X52S4RlWrcPsXcDq885s45rDl9ONl4Mci0s9tvroPp0mnBsgBYtwYI3E6s6MbvVeWiDT7fKKqCtwG/EpErhKRRBEJE5HJIvKku9vlOJ31I32W84Gz3KY3EySWIMxhqepGVV3axOaf4XRsLnabkT7BqTWA04Edi1PTWIzT/NTSGNbhnMA2uU0yPYGHcDrIPxKREvc9xruHdAdexznBr8U5wdaP038Ip6+iUEQe9vN2hzv2ezgJYx1OJ/Otbnz/An4FvIHTlj4Aty+mifIsxemHeBRnNNEGnGa8pnyG81nW1xbW4IxQaqr2EEg5G3sKp3ltHk6tsQK4yY13P3ADMAunVnQA8L0u5TX3514RWR7Aex1CVV8HLsLpKN+Jk3B+A7wtIhNwOuBnquoun2U2zud2SRMva1pB/RA3Y4wx5hBWgzDGGOOXJQhjjDF+WYIwxhjjlyUIY4wxfoXUxGFpaWmalZXldRjGGNNhLFu2rEBV0/1tC6kEkZWVxdKlTY3INMYY05iI5DW1zZqYjDHG+GUJwhhjjF+WIIwxxvhlCcIYY4xfliCMMcb4ZQnCGGOMX5YgjDHG+BVS10F0ZOVVtbyzcicVNbVkd0sgO6MLqfFROHf2NMaYtmcJwmN7iit4blEeLyzJo6is+pBtXeMiyc5IILtbF7K7dWFQRgIDM7qQ3iXaEocxJugsQXhk7TfFzJq/mdkrd1BTp3xrWAbXTOlP765x5O4pIXd3acPPd1bupLiipuHYpNhIJ2nUJ48MJ3l0S7DEYYxpPZYg2pCqMjcnn1nzN7NgQwGxkeFcOq4PV53Qj6y0+Ib9uifFMCU7/ZDj8ksqyd1TSs7uEnL3lLJhdynvr/qGl31qHZMGpPL498aQGBPZpuUyxoSmkLqj3NixY7U9zsVUUV3L2yt2MGv+ZnL3lJKRGM0Vk7K4dFwfkuOiWvy6qkpBaRW5e0r4cmsRf/kkh4HdEnj2+8fTLSGmFUtgjAlVIrJMVcf622Y1iCD7cmshP3huKQWlVQztkciDF47g7ON6EhVx9APIRIT0hGjSE6KZNCCNYzOTuP6FZVzw2CKev3ocfVPjj/wixhjTBBvmGkRlVTXc+o8VREeE8+I145lz82T+Z3SvVkkO/kwdlM6L14ynpKKa8x9bxKod+4PyPsaYzsESRBD9/v11bN1Xxp8uHMEJA9PapAN5VJ+uvHb9RKLChUueXMyijXuD/p7GmNBkCSJIFuQW8NyiPL5/Qj8m9E9t0/ce2C2B1384iYykGK54+nM+WLWrTd/fGBMaLEEEQXFFNbe/vpL+6fHcfvpgT2LomRzLa9dNZHjPRG54cRmvfL7VkziMMR2XJYgguHf2GvaUVPLghSOJiQz3LI6u8VG8eM14pmSnc8ebXzPz0w2E0qg1Y0xwWYJoZR+t3sUby7dzw4kDGNk72etwiIuKYNYVY/n2yJ488OF6fv3uWurqLEkYY47Mhrm2on0HqvjFP79mWI9Ebjo52+twGkSGh/HghSPpGh/FUws3s+9AJQ98ZwSR4fb9wBjTNEsQrURV+eVbX7O/vJoXrhkftKGsLRUWJtx19jDSukTzwIfrKSqv5q+XjSYuyv4EjDH+ta+zWAc2e+VO5ny9ix+fNogh3RO9DscvEeHGkwby+/85lnk5+Vw2awlFZVVeh2WMaacsQbSC3cUV3PX2akb1Sea6qQO8DueILh7Xh79eNobVO4v5zuOL+GZ/udchGWPaIUsQreDPH+dQWVPLn74zgvCwjjGb6vRjuvPsVePYtb+C8//6GRv2lHodkjGmnbEE0QoWbdrLtEHp9E/v4nUozTJxQCqvXDeBqlrlO49/xoptRV6HZIxpRyxBHKX8kkry9pYxpm9Xr0NpkeE9k3jjhxNJiInk0r8tZl5OvtchGWPaCUsQR2n51kKADpsgAPqmxvP69RPpmxrP1c9+weyVO70OyRjTDliCOErL8wqJCg9jeM8kr0M5Kt0SY3jl2gmM6tOVW175kmc/2+J1SMYYj1mCOErL8go5JjPR0yk1WktSbCTPfX8cpw7N4O7Zq3nw4xybmsOYTswSxFGorKnlqx37GZuV4nUorSYmMpzHLhvNhWN78fC/cvnlW6uotak5jOmUgpogRGS6iKwXkQ0icoef7beLyAp3WSUitSKS4m67xV23WkRuDWacLbV6ZzFVNXWM7tNx+x/8iQgP4w/nH8cPTxzAi0u2MvPTDV6HZIzxQNAShIiEAzOBM4BhwCUiMsx3H1V9QFVHqupI4OfAXFXdJyLHAD8AxgEjgLNFpP1MbuRanud0UI/um+xtIEEgIvxs+hDOPLY7Mz/dwPbCMq9DMsa0sWDWIMYBG1R1k6pWAa8AMw6z/yXAy+7jocBiVS1T1RpgLnBeEGNtkWV5hfRJiaNbQozXoQTNnWcNQwR++95ar0MxxrSxYCaITGCbz/Pt7rr/IiJxwHTgDXfVKmCqiKS6284Eejdx7LUislRElubnt90YflVlaV5hhx7eGojM5Fh+dNJA3l+1i/m5do2EMZ1JMBOEvzknmurtPAdYqKr7AFR1LfAH4GPgA2AlUOPvQFV9UlXHqurY9PT0o486QNsLy8kvqWR0iCcIgGum9Kdvahz3zF5NVU2d1+EYY9pIMBPEdg791t8LaOoKrIs52LwEgKr+XVVHq+pUYB+QG5QoW2iZ2/8wJsQ6qP2JiQzn7nOGsTH/AM98ttnrcIwxbSSYCeILIFtE+olIFE4SmN14JxFJAqYBbzda38392Qf4HxolEK8tyyskPiqcwd0TvA6lTZw8JIOTh3TjoU9y2V1c4XU4xpg2ELQE4XYu/wj4EFgLvKqqq0XkehG53mfX84CPVPVAo5d4Q0TWAO8AN6pqYbBibYlleYWM6tO1w8ze2hruOnsY1bXK799f53Uoxpg2ENTbianqHGBOo3WPN3r+DPCMn2OnBDO2o1FaWcO6XcX8qB3dVrQtZKXFc+3U/jz66QYuGdeHcf1C5wJBY8x/syupW2DltiLqtGNP0NdSN5w0gJ5JMdz19ipqaq3D2phQZgmiBZblFSICI3snex1Km4uLiuCXZw9j3a4SXvp8q9fhGGOCyBJECyzLK2RQtwSSYiO9DsUTZxzTnUkDUvnjh+vZW1rpdTjGmCCxBNFMdXXK8q2FjMnqfM1L9USEe88dTllVLQ98uN7rcIwxQWIJoply95RSUlHTKa5/OJzsjASunJTFP5ZuY6XdqtSYkGQJopkaLpDrhB3Ujd1yajap8dHcNXs1dTYluDEhxxJEMy3LKyQ1Poq+qXFeh+K5hJhIfnHmEFZuK+L1Zdu9DscY08osQTRT7p4ShvVMRKTzXCB3OOeNymRs36784YN17C+r9jocY0wrsgTRDKrK5oIDZKXGex1KuyEi3HPucPaVVfHnT3K8DscY04osQTRDYVk1JRU1ZKVZgvB1TGYSl43vw3OLtrD2m2KvwzHGtBJLEM2wZa8zXVSW9T/8l59+azBJsZHcPXs1qtZhbUwosATRDHluguhrTUz/JTkuittPH8Lnm/cxe2VTs7obYzoSSxDNsLmgjDCB3imxXofSLl10fG+OzUzivjlrKa30e38nY0wHYgmiGfL2HqBncizREeFeh9IuhYcJ984Yzu7iSh75d7u6v5MxpgUsQTTDlr1lNoLpCEb36coFY3rx1ILNbMwv9TocY8xRsATRDFsKDpCVZh3UR/Kz6UOIiQjnHuuwNqZDswQRoKKyKvaXV1sNIgDpCdH8+LRBzM8t4KM1u70OxxjTQpYgArRlbxlgI5gCdfnEvgzOSOD/3llDRXWt1+EYY1rAEkSAthQ4Q1z7WRNTQCLCw7jn3OHsKCrnsf9s9DocY0wLWIII0Ja9BxCBXl0tQQRq4oBUzj6uB4/P3ci2fWVeh2OMaSZLEAHK21tGz6RYYiJtiGtz3HnWUMJE+PW7a7wOxRjTTJYgArTZRjC1SI+kWG46ZSAfrdnNf9bv8TocY0wzWIIIUN7eA9ZB3UJXT+5Hv7R47n1nDZU11mFtTEdhCSIA+8uqKSyrtkn6Wig6Ipy7zhnG5oIDPLVgi9fhGGMCZAkiAAdncbUaREudNLgbpw7N4JF/57Jrf4XX4RhjAmAJIgANCcLuA3FU7jp7GDV1yn1z1nodijEmAJYgApDnXiTXJ8WamI5Gn9Q4rp82gNkrd7J4016vwzHGHIEliABs2XuAnkkxNsS1Ffxw2gAyk2O5++3V1NTWeR2OMeYwLEEEYEuBjWBqLbFR4fzq7GGs313C84vzvA7HGHMYliACkLe3zK6BaEWnD89gSnYaD36cQ0FppdfhGGOacMQEISIZIvJ3EXnffT5MRK4OfmjtQ3FFNXsPVNkIplYkItx9znDKq2q5/4N1XodjjGlCIDWIZ4APgZ7u8xzg1iDF0+5sbZjF1WoQrWlgty5cPbkfry7dzpdbC70OxxjjRyAJIk1VXwXqAFS1Bug0l8PWj9nvkWT3oW5tN52STbeEaO56ezW1dXZjIWPam0ASxAERSQUUQEQmAPuDGlU7Ut9GnpYQ7XEkoadLdAR3njWUr3fs59Wl27wOxxjTSCAJ4jZgNjBARBYCzwE3BTWqdqQ+QaTGR3kcSWg6d0RPxmWlcP8H6ygqq/I6HGOMj8MmCBEJB6a5yyTgOmC4qn7VBrG1CwWlVSTERNg1EEEiItxz7nD2l1fz4Mc5XodjjPFx2AShqrXADFWtUdXVqrpKVavbKLZ2Ib+0kvQu1rwUTMN6JvK9CX15YXEeq3d2mtZLY9q9QJqYForIoyIyRURG1y9Bj6ydKCipJM0SRNDddtpgkuOiuPvt1ahah7Ux7UEgCWISMBz4P+BP7vLHQF5cRKaLyHoR2SAid/jZfruIrHCXVSJSKyIp7rYfi8hqd/3LIhITeLFaT35pJenWQR10SXGR/Gz6YJbmFfLWih1eh2OMIYAEoaon+VlOPtJxbv/FTOAMYBhwiYgMa/TaD6jqSFUdCfwcmKuq+0QkE7gZGKuqxwDhwMXNLl0rcGoQ1kHdFr4zpjcjeiVx35x1lFR0qpZMY9qlQK6kThKRB0Vkqbv8SUSSAnjtccAGVd2kqlXAK8CMw+x/CfCyz/MIIFZEIoA4YGcA79mqKmtqKa6osSamNhIWJtw74xjySyp55N8bvA7HmE4vkCamp4AS4EJ3KQaeDuC4TMB3cPt2d91/EZE4YDrwBoCq7sBpxtoKfAPsV9WPmjj22vrklZ+fH0BYgdtb6gy7tGsg2s7I3slcNLY3Ty3YzIY9JV6HY0ynFkiCGKCqd7s1gU2qei/QP4DjxM+6pnofzwEWquo+ABHpilPb6IczxUe8iHzX34Gq+qSqjlXVsenp6QGEFbiGi+SsBtGm/nf6YOKiwrln9hrrsDbGQ4EkiHIRmVz/REROAMoDOG470NvneS+abia6mEObl04FNqtqvjus9k2czvI2dTBBWB9EW0rtEs1PvjWYBRsKePDjHDbml1qiMMYDEQHs80PgWZ9+h0LgygCO+wLIFpF+wA6cJHBp453c150G+NYQtgIT3KancuAUYGkA79mq8kusBuGVy8b34aM1u3jk3xt45N8byEyOZeqgdKYNSmPSwDQSYyK9DtGYkHfEBKGqK4ARIpLoPi8O5IVVtUZEfoQzE2w48JSqrhaR693tj7u7ngd8pKoHfI5dIiKvA8uBGuBL4MmAS9VKCtw+CBvm2vYiwsN48ZoJbN1bxtzcfObl5PPOyp28/PlWwsOEUb2TmToonamD0jk2M4nwMH8tmsaYoyFHqrqLyH3A/apa5D7vCvxEVX8Z/PCaZ+zYsbp0aetVNO6ZvZo3lm3n63tPb7XXNC1XXVvH8rxC5uXmMy+ngFU796MKyXGRnDAwjWnZTsLonuTJJTPGdEgiskxVx/rbFkgT0xmq+ov6J6paKCJnAu0uQbS2gtJKG8HUjkSGhzG+fyrj+6dy++mwt7SSBRsKmJdTwLzcfN776hsABmV0YaqbLMb1S7F5tIxpoUASRLiIRKtqJYCIxAKd4qxZUGoXybVnqV2imTEykxkjM1FV1u0qYV5OPvNy83luUR6zFmwmOsJJKlOz05g2KJ2B3bogYs1RxgQikATxAvAvEXkaZ5jq94FngxpVO1FQWkV2ty5eh2ECICIM7ZHI0B6JXDdtAGVVNSzZtI+5bsL4zXtr+c17a+mRFMOU7DSmDkpn8sA0kuPsC4AxTQmkk/p+EfkKZ+ipAL9W1Q+DHlk7UFBaycT+qV6HYVogLiqCk4Z046Qh3QDYXljG/NwC5uXk8/6qXby6dDthAsf1Sm4YHTWiVzIR4YGM/DamczhighCReJxRRh+IyGBgsIhEhvq031U1dRSVVdsQ1xDRq2scl4zrwyXj+lBTW8fK7UXMzXESxqP/zuXhf+WSGBPBCQPTGkZHZSbbbWZN5xZIE9M8YIo7eukTnOsRLgIuC2ZgXtt7oP5Wo9YEEWoiwsMY0zeFMX1TuO20QRSVVbmd3c7oqPdX7QJgQHq8kyyy05nQP5XYKOvsNp1LIAlCVLVMRK4GHnGbnL4MdmBeKyhx52GyGkTIS46L4uzjenL2cT1RVXL3lLqd3QW8tGQrTy/cQlR4GMf369owOmpI9wTr7DYhL6AEISITcWoMVzfjuA6tfpoNu0iucxERBmUkMCgjgWum9KeiupbPN+9rGB31u/fX8bv319EtIZop2elMHZTGlOx0Uuye5SYEBXKivwXnXg3/dK+E7g98GtywvJdfnyCsBtGpxUSGN/RJAHyzv5z5OQXMzc3nk7W7eWP5dkTg2MwkZ3RUdjrHZ6UQZld2mxAQyCimeTj9EPXPN+HczCek2Uyuxp8eSbFceHxvLjy+N7V1ylfbixpGRz0+dxMzP93IhP4pPHTxKDIS7Ypu07GFfFNRS+WXVBIfFW4dk6ZJ4WHCqD5dGdWnKzefks3+8mreWbmT3763ljMfms+fLxrZUPMwpiOyQd9NKCitsmk2TLMkxUby3Ql9mf2jE0jtEsUVT3/OAx+uo6a2zuvQjGmRQG45ekIg60JNUVmVdTyaFsnOSODtGydz4ZjezPx0I5f+bQnf7A/kFirGtC+B1CAeCXBdSCksqyI51u45YFomNiqcP1xwHH+5aCSrdu7nzIfm8+n6PV6HZUyzNNkH4Q5tnQSki8htPpsSce7vENKKyqrJ7pbgdRimg/v2qEyO7ZXEjS8u56qnv+C6af356bcGE2lTepgO4HB/pVFAF5wkkuCzFAMXBD80b+0vqyY5zmoQ5ugNSO/CWzeewKXj+/DE3E1c/ORidhRZk5Np/5qsQajqXGCuiDyjqnkAIhIGdAn0rnIdVXVtHSWVNSTHWh+EaR0xkeHcd96xTOifyi/e/JqzHp7PHy8YwanDMrwOzZgmBVLP/Z2IJLqT9q0B1ovI7UGOy1P7y515CK0GYVrbuSN68s5Nk8lMjuWa55bym3fXUFVjo5xM+xRIghjm1hi+DcwB+gDfC2ZQXisqc+ZhsgRhgqFfWjxv/HASl0/sy6wFm7nwiUVs21fmdVjG/JdAEkSkiETiJIi33Wm+D38j6w6uqKy+BmFNTCY4YiLD+b8Zx/DXy0azcU8pZz08nw9X7/I6LGMOEUiCeALYAsQD80SkL05HdciqTxBdrQZhguzMY3vw7s2T6Zsaz3XPL+Oe2auprKn1OixjgAAShKo+rKqZqnqmOvKAk9ogNs8U1jcxWSe1aQN9U+N5/YcTuXJSFs98toULHlvE1r3W5GS8F8iV1Bki8ncRed99Pgy4IuiReai+kzrJahCmjURHhHPPucN5/LtjyNt7gLMens+cr7/xOizTyQXSxPQM8CHQ032eA9wapHjahaKyasLDhMQYm8vQtK3px3TnvZun0L9bF254cTm/emsVFdXW5GS8EUiCSFPVV4E6AFWtAUL6L7awrIqk2Ei7Y5jxRO+UOF67biI/mNKP5xfncf5jn7G54IDXYZlOKJAEcUBEUnFHLonIBGB/UKPyWFF5tc3DZDwVFRHGnWcNY9blY9lRVM45jyxg9sqdXodlOplAEsRtwGxggIgsBJ4jxG8YVFRWZddAmHbh1GEZvHfzFAZ3T+Dml7/kF//82pqcTJsJJEGsBqbhTNx3HTAcWBfMoLxWVFZt10CYdiMzOZZXrp3AddP689KSrXx75kI25pd6HZbpBAJJEItUtUZVV6vqKvdCuUXBDsxLRWXWxGTal8jwMH5+xlCevvJ4dhdXcM4jC3jryx1eh2VCXJMJQkS6i8gYIFZERonIaHc5EYhrqwC94DQxWQ3CtD8nDenGnFumMLxnIrf+YwU/e/0ryqusyckEx+HGcZ4OXAn0Av4E1A/pKQZ+EdywvFNVU8eBqlrrgzDtVo+kWF7+wQT+/EkOf/3PRlZsK2LmZaMYaPcvMa2syRqEqj6rqicBV6rqyap6krvMUNU32zDGNlVUbhP1mfYvIjyM208fwrNXjaOgtJJzHlnI68u2ex2WCTGBTLXxRlsE0l7st4n6TAcydVA6c26ZwojeSfz0tZX85NWVlFXVeB2WCRF238NGiurvBWGd1KaDyEiM4cVrJnDzKdm8+eV2zn10Iet3lXgdlgkBliAaKTxgTUym4wkPE247bRAvXD2eorJqZsxcwD++2IpqSM/Mb4IsoAQhIpNE5FIRubx+CXZgXjlYg7AmJtPxnDAwjTm3TGZM36787I2v+fE/VnCg0pqcTMsccTY6EXkeGACs4OAcTIpzRXXIKa6fydWamEwH1S0hhue+P56Zn27gL5/k8NX2/cy8bDRDeyR6HZrpYAKpQYwFTlDVG1T1JncJaKoNEZkuIutFZIOI3OFn++0issJdVolIrYikiMhgn/UrRKRYRG5tZtlapLjC+bbVxWZyNR1YeJhw8ynZvHjNBEora5gxcyEvLbEmJ9M8gSSIVUD35r6wiIQDM4EzgGHAJe69JBqo6gOqOlJVRwI/B+aq6j5VXe+zfgxQBvyzuTG0RHF5NQnREYSH2UyupuObOCCVObdMYXy/FH7xz6+5+ZUVlFRUex2W6SACmu4bWCMiH4rI7PolgOPGARtUdZOqVgGvADMOs/8lwMt+1p8CbHTvZBd0xRXVJFrzkgkhaV2iefaqcdx++mDe+2on5zyygFU7QnpCZtNKAmlHuaeFr50JbPN5vh0Y729HEYkDpgM/8rP5YvwnjvpjrwWuBejTp08LQz2ouLyGBGteMiEmLEy48aSBHJ+Vws0vf8n//PUzfnX2UL47oa/d98Q0KZAL5eb6WwJ4bX9/dU01gJ4DLFTVfYe8gEgUcC7w2mHie1JVx6rq2PT09ADCOjyrQZhQNq5fCnNumcKkgan86u3V3PjScoqtyck0IZB7Uk8QkS9EpFREqtyO5OIAXns70NvneS+gqTueNFVLOANYrqq7A3i/VlFSUUNijCUIE7pS4qN46orjueOMIXy4ejdnP7yAr7YXeR2WaYcC6YN4FKd/IBeIBa5x1x3JF0C2iPRzawIX49x46BAikoRzv4m3/bxGU/0SQVNcXk1irDUxmdAWFiZcP20Ar143gZraOs5/7DOeXrjZRjmZQwR0oZyqbgDCVbVWVZ8GTgzgmBqcPoUPgbXAq6q6WkSuF5HrfXY9D/hIVQ+56a7bL3Ea0KYTAxZXVFsNwnQaY/qm8N7NU5g2KJ1731nD9S8sa5iPzJhAviqXuTWAFSJyP/ANEB/Ii6vqHGBOo3WPN3r+DPCMn2PLgNRA3qe11NUppZU11gdhOpWu8VH87fKx/H3BZn7//jrOemQ+j146mpG9k70OzXgskBrE99z9fgQcwOlXOD+YQXmlpLIGVUi0UUymkxERrpnSn9eun4gqXPDYZ8yav8manDq5QEYx5eGMSOqhqveq6m1uk1PIqZ9mw2oQprMa1acrc26ewslDuvGb99byg+eWUlRW5XVYxiOBjGI6B2cepg/c5yMDvFCuw6kf7md9EKYzS4qL5InvjeHuc4YxNyefMx+az7K8Qq/DMh4IpInpHpyroosAVHUFkBWsgLxUXO7Mw2SjmExnJyJcdUI/3vjhJCLCw7joiUU8MXcjdXXW5NSZBJIgalS1U1yXbzUIYw51XK9k3r15Mt8ansHv3l/H1c9+wb4D1uTUWQQ0WZ+IXAqEi0i2iDwCfBbkuDzR0AdhCcKYBokxkcy8dDS/njGchRv2cuZD8/l8874jH2g6vEASxE3AcKAS56K1YuDWIMbkmfqpvq2JyZhDiQjfm5jFmzdMIiYyjEv+tpiZn26wJqcQF8gopjJVvVNVj3fnPLpTVSvaIri2Vl+D6BJtCcIYf47JTOKdmyZz5rE9eODD9Vzx9OcUlFZ6HZYJkkBGMY0VkTdFZLmIfFW/tEVwba2koob4qHAiwu1W3cY0JSEmkocvHsl95x3Lks37OPOh+SzauNfrsEwQBHImfBHnSufzcWZdrV9CTllVDfFWezDmiESES8f34e0bT6BLdASXzVrMQ5/kUmtNTiElkASRr6qzVXWzqubVL0GPzAMHqmqJiwr3OgxjOoyhPRJ556bJzBiZyZ8/yeHyp5awpyQkW6A7pUC+Lt8tIrOAf+F0VAOgqm06iV5bKK+qITbKahDGNEd8dAQPXjiCif1TuWv2Kk7501ymDkpnmrtkJMZ4HaJpoUDOhlcBQ4BIoM5dp7TxLKttocxqEMa0iIhw4fG9GdknmSfnbWJeTj7vffUNAIMzEpg2OJ2p2emMzepKTKT9j3UUgSSIEap6bNAjaQfKqmrtdqPGHIVBGQn88TsjUFXW7SphXk4+83LzeWbhFp6ct4mYyDAm9k9l6qB0pg5Kp39avN3ytB0L5Gy4WESGqeqaoEfjsfKqWjISo70Ow5gOT0QY2iORoT0SuW7aAMqqali8aS/zcgqYm5PPp+84p5NeXWOdZJGdzgkDU0mwi1TblUASxGTgChHZjNMHIYCq6nFBjcwDZdU1xFkfhDGtLi4qgpOHZHDykAwAtu0rY25OPnNz8nn7yx28tGQrEWHC6D5dmToojamD0jmmZxJhYVa78FIgZ8PpQY+inSivqiXW+iCMCbreKXF8d0JfvjuhL1U1dSzfWtjQHPXHj3L440c5pMZHMTk7janZ6UwZlEa3BOvsbmtHTBChOqTVn7KqWuKsA82YNhUVEcaE/qlM6J/K/04fQn5JJQs25DMvp4D5ufm8vWInAMN6JDaMjhrTtytREXZBa7BZe4qrrk4pr7ZRTMZ4LT0hmvNG9eK8Ub2oq1PWfFPM3Jx85uXkM2v+Jh6fu5H4qHAmDkhlmtvZ3Tc1oLsgm2ayBOGqqKlFFeLsSmpj2o2wMOGYzCSOyUzixpMGUlJRzaKNe5mX6/RffLJ2DwB9U+OcZJGdzsQBqTYjQiuxT9FVVlULYDUIY9qxhJhIvjW8O98a3h1VZcveMua5nd2vLd3Oc4vyiAwXxvZNcYfSpjGsR6INpW0hSxCucjdBxFofhDEdgojQLy2efmnxXDEpi8qaWpZtKWRubj5z1+fzhw/W8YcPnCarKdlpTBuUzuSBaaR2saHsgbIE4TpYg7CPxJiOKDoinEkD05g0MI2fnzGUPcUVzMt1r7tYt4c3l+9ABI7NTGJqdjrTBqczsncykTZ7c5PsbOgqq3JuFmRNTMaEhm6JMVwwphcXjOlFbZ2yasf+hs7ux+Zu5NFPN5AQHcGkgakNF+v1TonzOux2xRKEq9z6IIwJWeFhwojeyYzonczNp2Szv7yazzYUMC/XGU774erdAPRPj2+oXUzol9rpr4uyBOGyJiZjOo+k2EjOOLYHZxzbA1VlY34pc3MKmJeTz8ufb+WZz7YQFRHGuKyUhqG0gzK6dLrObjsbusqq3U7qTv6NwZjORkQY2C2Bgd0SuHpyPyqqa/l8876G0VG/nbOW385ZS/fEmIZpQCYPTCM5Lsrr0IPOEoSr3PogjDFATGR4w2yzvwR2FpUz373u4oNVu3h16XbCBEb0TmZqtrPfyN7JhIfgvFGWIFw2zNUY40/P5FguOr4PFx3fh5raOlZuP9jZ/fC/c3noX7kkxkQwJTu9oYbRIynW67BbhSUIV3m1cy8ka2IyxjQlIjyMMX27MqZvV247bRBFZVUs2FDA3PXORIPvfe3cJGlQRpeGzu7js1I67E2SLEG4Ktw+iGibAMwYE6DkuCjOPq4nZx/XE1UlZ3cpc3P2MC+ngOcW5TFrwWZiIsMY3+/gvFED0jvOTZIsQbgqqmuJiQzrML84Y0z7IiIM7p7A4O4JXDvVuUnSkk37nOao3Hz+713nJkmZybHurLTORX2J7fgmSZYgXOXVtdb/YIxpNXFREZw0pBsnDekGODdJmudOA/LOyp28/PlWwsOEUb2TG2oXx2a2r5skWYJwOTUISxDGmODonRLHZeP7ctn4vlTX1vHl1qKGobR/+jiHP32cQ9e4SLezO52p2Wl0S/T2JkmWIFzl1XVWgzDGtInI8DDG9UthXL8Ufnr6YPaWVvp0dhcwe6Vzk6ShPRKZOiiNadnpjMnqSnRE256jLEG4yqusBmGM8UZql2hmjMxkxshM6uqUtbsO3iTpqQWbeWLuJuKiwpnYP7XhrnpZacG/SZIlCFd9J7UxxngpLEwY3jOJ4T2TuOHEgZRW1jg3SXI7u/+1zrlJUp+UOOe6i+x0Jg1Mo0sQbpIU1AQhItOBh4BwYJaq/r7R9tuBy3xiGQqkq+o+EUkGZgHHAAp8X1UXBSvWiupauwbCGNPudImO4LRhGZw2LAOALQUH3EkG83lz+Q5eWLyVxJgIlv/qNCJaeeryoCUIEQkHZgKnAduBL0Rktqquqd9HVR8AHnD3Pwf4saruczc/BHygqheISBQQ1Hl4y6trSYptv8PNjDEGICstnqy0eC6fmEVVTR1L8/axvbC81ZMDBLcGMQ7YoKqbAETkFWAGsKaJ/S8BXnb3TQSmAlcCqGoVUBXEWCmvriXGahDGmA4kKiKMSQPSgvb6wWx0zwS2+Tzf7q77LyISB0wH3nBX9QfygadF5EsRmSUifntkRORaEVkqIkvz8/NbHGyljWIyxphDBDNB+LvaQ5vY9xxgoU/zUgQwGnhMVUcBB4A7/B2oqk+q6lhVHZuent7iYMutk9oYYw4RzDPidqC3z/NewM4m9r0Yt3nJ59jtqrrEff46TsIImvKqWmLaeIyxMca0Z8FMEF8A2SLSz+1kvhiY3XgnEUkCpgFv169T1V3ANhEZ7K46hab7Lo6aqlJRY6OYjDHGV9A6qVW1RkR+BHyIM8z1KVVdLSLXu9sfd3c9D/hIVQ80eombgBfd5LIJuCpYsVbV1qGKXShnjDE+gnodhKrOAeY0Wvd4o+fPAM/4OXYFMDZ40R1U4d4Lwqb6NsaYg+yMCFTW3wvCahDGGNPAEgRQWePUIGKsBmGMMQ3sjMjBu8lZH4QxxhxkCYKDfRCWIIwx5iBLEEBFTX0Nwj4OY4ypZ2dEDjYxtfXNOIwxpj2zBIEzDxNYDcIYY3zZGRHfJiarQRhjTD1LEBysQdiFcsYYc5CdEfG5DsJqEMYY08ASBFDpNjFFBeGOTMYY01HZGRGocmsQ0dZJbYwxDeyMyMEmJqtBGGPMQXZGxGliigiToNz02xhjOio7I+I0MUXZCCZjjDmEnRVxmphsiKsxxhzKzoo410HYNBvGGHMoSxA4txy1JiZjjDmUnRVxOqmtickYYw5lZ0XcJia7BsIYYw5hZ0XqO6mtD8IYY3xZgsCamIwxxh87K+LUIKyT2hhjDmVnRZw+CJtmwxhjDmVnRZwmJpvq2xhjDmUJAruS2hhj/LGzIs5cTDbM1RhjDmVnRWyYqzHG+GMJAjh1aDeG90z0OgxjjGlXIrwOoD34y8WjvA7BGGPaHatBGGOM8csShDHGGL8sQRhjjPHLEoQxxhi/LEEYY4zxyxKEMcYYvyxBGGOM8csShDHGGL9EVb2OodWISD6Q14JD04CCVg6nvbMydw5W5s7haMrcV1XT/W0IqQTRUiKyVFXHeh1HW7Iydw5W5s4hWGW2JiZjjDF+WYIwxhjjlyUIx5NeB+ABK3PnYGXuHIJSZuuDMMYY45fVIIwxxvhlCcIYY4xfnT5BiMh0EVkvIhtE5A6v42ktItJbRD4VkbUislpEbnHXp4jIxyKS6/7s6nPMz93PYb2InO5d9C0nIuEi8qWIvOs+D/XyJovI6yKyzv1dT+wEZf6x+ze9SkReFpGYUCyziDwlIntEZJXPumaXU0TGiMjX7raHRUQCDkJVO+0ChAMbgf5AFLASGOZ1XK1Uth7AaPdxApADDAPuB+5w198B/MF9PMwtfzTQz/1cwr0uRwvKfRvwEvCu+zzUy/sscI37OApIDuUyA5nAZiDWff4qcGUolhmYCowGVvmsa3Y5gc+BiYAA7wNnBBpDZ69BjAM2qOomVa0CXgFmeBxTq1DVb1R1ufu4BFiL8881A+ekgvvz2+7jGcArqlqpqpuBDTifT4chIr2As4BZPqtDubyJOCeRvwOoapWqFhHCZXZFALEiEgHEATsJwTKr6jxgX6PVzSqniPQAElV1kTrZ4jmfY46osyeITGCbz/Pt7rqQIiJZwChgCZChqt+Ak0SAbu5uofBZ/AX4X6DOZ10ol7c/kA887TarzRKReEK4zKq6A/gjsBX4Btivqh8RwmVupLnlzHQfN14fkM6eIPy1xYXUuF8R6QK8AdyqqsWH29XPug7zWYjI2cAeVV0W6CF+1nWY8roicJogHlPVUcABnGaHpnT4Mrtt7jNwmlF6AvEi8t3DHeJnXYcqc4CaKudRlb+zJ4jtQG+f571wqqshQUQicZLDi6r6prt6t1vtxP25x13f0T+LE4BzRWQLTlPhySLyAqFbXnDKsF1Vl7jPX8dJGKFc5lOBzaqar6rVwJvAJEK7zL6aW87t7uPG6wPS2RPEF0C2iPQTkSjgYmC2xzG1Cnekwt+Btar6oM+m2cAV7uMrgLd91l8sItEi0g/Ixunc6hBU9eeq2ktVs3B+j/9W1e8SouUFUNVdwDYRGeyuOgVYQwiXGadpaYKIxLl/46fg9K+Fcpl9NaucbjNUiYhMcD+vy32OOTKve+q9XoAzcUb4bATu9DqeVizXZJyq5FfACnc5E0gF/gXkuj9TfI650/0c1tOMkQ7tbQFO5OAoppAuLzASWOr+nt8CunaCMt8LrANWAc/jjNwJuTIDL+P0s1Tj1ASubkk5gbHuZ7UReBR3Bo1AFptqwxhjjF+dvYnJGGNMEyxBGGOM8csShDHGGL8sQRhjjPHLEoQxxhi/LEEY00pEZIuIpB3tPsa0F5YgjDHG+GUJwpgWEJG3RGSZe1+Caxtty3Lvz/CsiHzl3q8hzmeXm0RkuTtH/xD3mHEi8pk76d5nPldHG+MZSxDGtMz3VXUMzlWqN4tIaqPtg4EnVfU4oBi4wWdbgaqOBh4DfuquWwdMVWfSvbuA+4IavTEBsARhTMvcLCIrgcU4k6RlN9q+TVUXuo9fwJn6pF79xInLgCz3cRLwmnv3sD8Dw4MRtDHNYQnCmGYSkRNxZhWdqKojgC+BmEa7NZ7Dxvd5pfuzFmfKboBfA5+q6jHAOX5ez5g2ZwnCmOZLAgpVtcztQ5jgZ58+IjLRfXwJsCCA19zhPr6yVaI05ihZgjCm+T4AIkTkK5xv/ov97LMWuMLdJwWnv+Fw7gd+JyILce6VboznbDZXY1qZe4vXd93mImM6LKtBGGOM8ctqEMYYY/yyGoQxxhi/LEEYY4zxyxKEMcYYvyxBGGOM8csShDHGGL/+HznTx6A2XCgqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "gs_lr_wop_report = pd.DataFrame(gs_lr_wop.cv_results_)\n",
    "plt.plot(gs_lr_wop_report['param_classifier__alpha'],gs_lr_wop_report['mean_test_score'])\n",
    "plt.xlabel('alpha')\n",
    "plt.ylabel('mean test score')\n",
    "plt.title('Mean test score without PCA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso with pca, Training score: 0.763\n",
      "Lasso with pca, Test score: 0.818\n",
      "Lasso without pca, Training score: 0.923\n",
      "Lasso without pca, Test score: 0.888\n",
      "---------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/esin/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: RuntimeWarning: invalid value encountered in log\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso with pca, Training RMSE: nan\n",
      "Lasso with pca, Test RMSE: 0.199\n",
      "Lasso without pca, Training RMSE: 0.108\n",
      "Lasso without pca, Test RMSE: 0.106\n"
     ]
    }
   ],
   "source": [
    "print(\"Lasso with pca, Training score: %.3f\" % gs_lr_wpca.score(X_train, y_train))\n",
    "print(\"Lasso with pca, Test score: %.3f\" % gs_lr_wpca.score(X_test, y_test))\n",
    "print(\"Lasso without pca, Training score: %.3f\" % gs_lr_wop.score(X_train, y_train))\n",
    "print(\"Lasso without pca, Test score: %.3f\" % gs_lr_wop.score(X_test, y_test))\n",
    "print('---------------------------')\n",
    "print(\"Lasso with pca, Training RMSE: %.3f\" % logrmse(gs_lr_wpca.predict(X_train), y_train.values))\n",
    "print(\"Lasso with pca, Test RMSE: %.3f\" % logrmse(gs_lr_wpca.predict(X_test), y_test))\n",
    "print(\"Lasso without pca, Training RMSE: %.3f\" % logrmse(gs_lr_wop.predict(X_train), y_train))\n",
    "print(\"Lasso without pca, Test RMSE: %.3f\" % logrmse(gs_lr_wop.predict(X_train), y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso performs quite well without PCA."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test with Random Forest Regressor with different PCA types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_rf = Pipeline(steps=[('preprocessor', preprocessor),\n",
    "                         ('reduce_dim',PCA()),\n",
    "                         ('classifier', RandomForestRegressor(n_estimators=500,\n",
    "                                                           random_state=42))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare different PCA algorithms and number of components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_components = [1, 5, 10, 50]\n",
    "n_estimators_rf = [100,500,1000]\n",
    "param_grid = [\n",
    "    {\n",
    "        'reduce_dim':[PCA(),KernelPCA()],\n",
    "        'reduce_dim__n_components' : n_components,\n",
    "        'classifier__n_estimators': n_estimators_rf\n",
    "    }\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_rf = GridSearchCV(pipe_rf,n_jobs=4,param_grid=param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gs_rf.fit(X_train,y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_rf_report=pd.DataFrame(gs_rf.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_classifier__n_estimators</th>\n",
       "      <th>param_reduce_dim</th>\n",
       "      <th>param_reduce_dim__n_components</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51.652182</td>\n",
       "      <td>1.662478</td>\n",
       "      <td>1.412773</td>\n",
       "      <td>0.097149</td>\n",
       "      <td>100</td>\n",
       "      <td>PCA(n_components=10)</td>\n",
       "      <td>1</td>\n",
       "      <td>{'classifier__n_estimators': 100, 'reduce_dim'...</td>\n",
       "      <td>0.703160</td>\n",
       "      <td>0.757407</td>\n",
       "      <td>0.715395</td>\n",
       "      <td>0.835991</td>\n",
       "      <td>0.824361</td>\n",
       "      <td>0.767263</td>\n",
       "      <td>0.054553</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>54.298085</td>\n",
       "      <td>1.279450</td>\n",
       "      <td>1.446241</td>\n",
       "      <td>0.095829</td>\n",
       "      <td>100</td>\n",
       "      <td>PCA(n_components=10)</td>\n",
       "      <td>5</td>\n",
       "      <td>{'classifier__n_estimators': 100, 'reduce_dim'...</td>\n",
       "      <td>0.827896</td>\n",
       "      <td>0.832613</td>\n",
       "      <td>0.787731</td>\n",
       "      <td>0.900994</td>\n",
       "      <td>0.899477</td>\n",
       "      <td>0.849742</td>\n",
       "      <td>0.044082</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>54.539925</td>\n",
       "      <td>1.098607</td>\n",
       "      <td>1.434471</td>\n",
       "      <td>0.094646</td>\n",
       "      <td>100</td>\n",
       "      <td>PCA(n_components=10)</td>\n",
       "      <td>10</td>\n",
       "      <td>{'classifier__n_estimators': 100, 'reduce_dim'...</td>\n",
       "      <td>0.824391</td>\n",
       "      <td>0.846983</td>\n",
       "      <td>0.779830</td>\n",
       "      <td>0.900517</td>\n",
       "      <td>0.907530</td>\n",
       "      <td>0.851850</td>\n",
       "      <td>0.047819</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56.425828</td>\n",
       "      <td>1.542438</td>\n",
       "      <td>1.458856</td>\n",
       "      <td>0.151594</td>\n",
       "      <td>100</td>\n",
       "      <td>PCA(n_components=10)</td>\n",
       "      <td>50</td>\n",
       "      <td>{'classifier__n_estimators': 100, 'reduce_dim'...</td>\n",
       "      <td>0.791333</td>\n",
       "      <td>0.838095</td>\n",
       "      <td>0.757142</td>\n",
       "      <td>0.904165</td>\n",
       "      <td>0.910181</td>\n",
       "      <td>0.840183</td>\n",
       "      <td>0.060465</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53.742189</td>\n",
       "      <td>0.934195</td>\n",
       "      <td>1.496450</td>\n",
       "      <td>0.118155</td>\n",
       "      <td>100</td>\n",
       "      <td>KernelPCA()</td>\n",
       "      <td>1</td>\n",
       "      <td>{'classifier__n_estimators': 100, 'reduce_dim'...</td>\n",
       "      <td>0.703139</td>\n",
       "      <td>0.757407</td>\n",
       "      <td>0.715640</td>\n",
       "      <td>0.835991</td>\n",
       "      <td>0.824361</td>\n",
       "      <td>0.767307</td>\n",
       "      <td>0.054512</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0      51.652182      1.662478         1.412773        0.097149   \n",
       "1      54.298085      1.279450         1.446241        0.095829   \n",
       "2      54.539925      1.098607         1.434471        0.094646   \n",
       "3      56.425828      1.542438         1.458856        0.151594   \n",
       "4      53.742189      0.934195         1.496450        0.118155   \n",
       "\n",
       "  param_classifier__n_estimators      param_reduce_dim  \\\n",
       "0                            100  PCA(n_components=10)   \n",
       "1                            100  PCA(n_components=10)   \n",
       "2                            100  PCA(n_components=10)   \n",
       "3                            100  PCA(n_components=10)   \n",
       "4                            100           KernelPCA()   \n",
       "\n",
       "  param_reduce_dim__n_components  \\\n",
       "0                              1   \n",
       "1                              5   \n",
       "2                             10   \n",
       "3                             50   \n",
       "4                              1   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "0  {'classifier__n_estimators': 100, 'reduce_dim'...           0.703160   \n",
       "1  {'classifier__n_estimators': 100, 'reduce_dim'...           0.827896   \n",
       "2  {'classifier__n_estimators': 100, 'reduce_dim'...           0.824391   \n",
       "3  {'classifier__n_estimators': 100, 'reduce_dim'...           0.791333   \n",
       "4  {'classifier__n_estimators': 100, 'reduce_dim'...           0.703139   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0           0.757407           0.715395           0.835991           0.824361   \n",
       "1           0.832613           0.787731           0.900994           0.899477   \n",
       "2           0.846983           0.779830           0.900517           0.907530   \n",
       "3           0.838095           0.757142           0.904165           0.910181   \n",
       "4           0.757407           0.715640           0.835991           0.824361   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "0         0.767263        0.054553               20  \n",
       "1         0.849742        0.044082                5  \n",
       "2         0.851850        0.047819                1  \n",
       "3         0.840183        0.060465               13  \n",
       "4         0.767307        0.054512               19  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_rf_report.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAGDCAYAAAACpSdYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyB0lEQVR4nO3de5xVBbn/8c/jAN5SK0RNxLCkFCxvSFZ2VI6dyI6RXQg1TU8nw592/dn1dyrtcirNkye1yMwwKxUvFRVIaqZlkaJZiqjhFaIUMUURReD5/bHX4HbYw+w1M2v2DHzer9d+se7r2Ys933lm7bXXjsxEkiRJUnM2aXUBkiRJ0kBiAy1JkiSVYAMtSZIklWADLUmSJJVgAy1JkiSVYAMtSZIklWADPcBFxCkR8cNW17EhiogvRcQjEfGPVtfSVyJidETMbXUdzYqIt0bExa2uQyrD3K6Oud3/bSi5bQNdgYi4PyJWRMSTEfGPiJgWES9odV09EREHRcSa4jm1P37eh/sfGREZEYPWs8wpEfFsUdtjEfH7iHhtN/c3Avi/wOjM3KG7dQ9AXwS+3psbjIghEXFZ8XOREXFQh/kREV+LiKXF47SIiLr5IyPi2oh4KiLujIhD2udl5gxgj4h4dW/WrI2PuV3J/s3tvlFFbrf/39W/dj5bN3+jz20b6OoclpkvAPYC9gY+3dpyesXizHxB3eOwshuIiLYqCqtzSXHchwG/A66o/6FuRhH2LwWWZubDZQtY3y+L/iwiXgIcDPy0gs3/DngP0Ois0PHA24A9gVcD/w58oG7+RcCfgKHA/wMui4hhHeYf3/slayNkbjdgbvdfFec2wAvrXjtfrJu+0ee2DXTFMvMfwGxqgQxARHwqIu6JiCci4o6IOLxu3rER8buI+HpE/DMi7ouIN9fN3yUirivWvQrYtn5/xVsj84q/5H8TEbvXzbs/Ij4eEX+JiOUR8b2I2D4iZhXbuzoiXlT2OUbE7sW+Hiv2/da6edMi4tsRMTMilgMHR8SOEXF5RCwpnt+H6pYfFxFzI2JZRDwUEf9TzLq++Pex4i/h9Z6hyMxngQuAHYChEbFN8Xz/HhF/i9rbfG11x/yGiPhGRDwK/Aa4Ctix2Ne0Jo/tJyPiL8DyiNi1+Ov9uIhYWPxfTomI/Yrj/1hEnF23/ssj4tfFX/KPRMSPIuKFHbZ/crHu4xFxSURsVjd/YkTcWhy3eyJiQjG90+fdwBuBWzLz6Wb324zMXJmZZ2bm74DVDRZ5L3BGZi7KzL8BZwDHFvt/BbAP8PnMXJGZlwO3Ae+oW/83wFvK1CStj7ltbm/sud0EczszffTyA7gfOKQY3onaC+d/6+a/C9iR2h8w7waWAy8p5h0LPAu8H2gDTgAWA1HM/wPwP8CmwL8ATwA/LOa9otjWG4HBwCeABcCQurrmANsDw4GHgVuonWnZFPg1tRd8o+d0ELCowfTBxT4+AwwBxhc1vbKYPw14HHh98Xy3AG4GPlcs/zLgXuBNdc/v6GL4BcD+xfBIIIFB6znup9Qdi02B04GFxfhPge8AWwLbATcCH6g75quADwKDgM07Pt8mj+2twIhi/fZ6pwKbAf8GPF3UsV3d8T+wWH/XYtubUjsLcz1wZofX1I3UXjcvBuYDU4p544pj/MbiGA8HduvqeTc4fqcD5zR4LXe2352Bx9bzOLLBPhYBB3WY9jjwmrrxscATxfDhwPwOy58NnFU3/uLiWG/d6p99HwP3gbltbpvbj1Hkdt2x+Bu13P4+sG3dPjb63G55ARvio3jxPkktkBK4htrbIJ0tfyswsRg+FlhQN2+LYhs7FC/8VcCWdfN/zHPh81lget28TYoX/0F1dR1VN/9y4Nt14x8EftpJjQcBazr8oE0C3kDtbflN6pa9CDilGJ4G/KBu3muABzts+9PA94vh64FT639Qi+ntP8xdBfHKoraHqf1i2ZfaL55ngM3rlj0CuLbumHes6SCeH8TNHNv/aFDv8LppS4F3dzj+H+nkubwN+FOH19R76sZPA6YWw98BvtFgG+t93g2W/y7w1Qav5Yb77ebPRqMGejXFL45ifFRx7AI4GpjTYfkvA9PqxgcXy+/c3bp8+MDcNrfN7fptvIBaUzyoqOkyYHbd/I0+twfkNT8DxNsy8+qIOJBaWG5LLSCIiGOAj1H7YYXaC7X+Lb2114lm5lNRuxSsfZl/ZubyumUfoPbXM9T+2nygbt01EbGQ2l+27R6qG17RYHx9H5pZnJk71U+IiHdTO1uwpkNN9ftcWDf8UmpvsT1WN60N+G0x/D7gC8CdEXEfcGpm/mI9NXU0PTPf06HGcdR+WP8ez11Wt0mHuuqHG2nm2DbaRlPHOyK2A75J7RfbVkV9/+ywrfrrh58qaoLa///MBvt+KV0/73r/LPbdUWf77S1PAlvXjW8NPJmZGREd57XPf6JuvL3mx3q5Lm18zO3nmNs1G2VuZ+aTQPudPR6KiJOKmrbOzGWY214DXbXMvI7aX/NfB4iIl1L7i/EkYGhmvhC4ndpfbV35O/CiiNiybtrOdcOLqf3wUewrqP2Q/q37z6BLi4EREVH/Wtq5wz6zbnghcF9mvrDusVVmHgqQmX/NzCOovW31NWofPNiywzbKWkjtL/pt6/a5dWaO6aTGRpo5tj2p8SvF+q/OzK2pfeCu2Q/RLARe3sn0rp53vb9Qe8uzKRGxczz/E9odH0c1ual51D6I0m7PYlr7vJdFxFadzAfYHbi/CHWpx8xtwNxuxsaU2+3Hqf35bfS5bQPdN84E3hgRe1G7pimBJQARcRywRzMbycwHqP1FeGrUbg12AFD/ierpwFsi4l8jYjC12/k8A/y+l55HI3+kdo3ZJyJicNRuUXYY0Nk9Hm8ElhUf3Ng8ItoiYo+I2A8gIt4TEcOKMyOPFeuspna81lC79q6UzPw78CvgjIjYOiI2KT78cWCJzVR9bLei9hf9YxExHPh4iXW/BxxX1LZJRAyPiN268byvAvZp9sMmmflgPv/T/R0fP2pfNiI2rdvukIjYLJ47vfID4GNF3TtSO7bTin3cTe2t8s8X6xxO7RPfl9eVciAwq5mapRLOxNxuZ243tsHmdkS8JiJeWex/KLUz7b/JzMeLTW30uW0D3Qcycwm1F9tnM/MOap9W/QO1t4VeBdxQYnNHUrse7VHg88V22/dzF7W/gM8CHqEWiIdl5speeBoNFdt+K/DmYp/fAo7JzDs7WX51UddewH3FOucB2xSLTADmFW8B/S8wOTOfzsynqF1DdUPUPgm9f8lSj6H24Zc7qL3ldRnwkhLPs+pjeyq1Ty0/DvwSuKJEbTcCxwHfKNa/jufOujT9vDPzIWrXH07s1jNYv7uovfU5nNrdDVbU1fgd4OfUPrR1O7Xn/526dSdTuxbvn8BXgXcWP1PtjuiwvNRj5vbzlje3G9uQc/tlwJXULru4ndofHkfUzd/oc7v9E8KSRESMpnYbqXE5AMIhIg6j9un/Sa2uRZJawdxuDRtoSZIkqQQv4ZAkSZJKsIGWJEmSSrCBliRJkkqwgZYkSZJKGHDfRLjtttvmyJEjW12GJHXLzTff/EhmDmt1HX3FzJY0kHWW2QOugR45ciRz587tekFJ6oci4oGul9pwmNmSBrLOMttLOCRJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSBrW6AGkgWbbsiZbsd+utt2rJfiVpIGtVZgPsuefSluz3vvtGtmS/Gxsb6I1cq8KlVcECAzNcdtnl/pbteyAeL2lDZUMo9Q9ewiFJkiSVYAMtSZIkleAlHJIq08q3m71uXJJUFRtoSRukVl037vWakgYqr7FvnpdwSJIkSSXYQEuSJEkl2EBLkiRJJdhAS5IkSSXYQEuSJEkl2EBLkiRJJdhAS5IkSSXYQEuSJEkl2EBLkiRJJdhAS5IAiIgJEXFXRCyIiE81mL9NRPw8Iv4cEfMi4rhW1ClJrVZpA20YS9LAEBFtwDnAm4HRwBERMbrDYicCd2TmnsBBwBkRMaRPC5WkfqCyBtowlqQBZRywIDPvzcyVwMXAxA7LJLBVRATwAuBRYFXflilJrVflGWjDWJIGjuHAwrrxRcW0emcDuwOLgduAD2fmmo4biojjI2JuRMxdsmRJVfVKUstU2UAbxpI0cESDadlh/E3ArcCOwF7A2RGx9TorZZ6bmWMzc+ywYcN6u05JarkqG2jDWJIGjkXAiLrxnaid3Kh3HHBF1iwA7gN266P6JKnfqLKBNowlaeC4CRgVEbsUn0WZDMzosMyDwL8CRMT2wCuBe/u0SknqB6psoA1jSRogMnMVcBIwG5gPTM/MeRExJSKmFIt9EXhdRNwGXAN8MjMfaU3FktQ6g6racGauioj2MG4Dzm8P42L+VGphPK0I48AwlqSWycyZwMwO06bWDS8G/q2v65Kk/qayBhoMY0mSJG14/CZCSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSphEpvY9cKy5Y90ZL97rnn0pbsF+C++0a2bN+S1BNmtqSByDPQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkACJiQkTcFRELIuJTDeZ/PCJuLR63R8TqiHhxK2qVpFaqtIE2jCVpYIiINuAc4M3AaOCIiBhdv0xmnp6Ze2XmXsCngesy89E+L1aSWqyyBtowlqQBZRywIDPvzcyVwMXAxPUsfwRwUZ9UJkn9TJVnoA1jSRo4hgML68YXFdPWERFbABOAy/ugLknqd6psoA1jSRo4osG07GTZw4AbOnvHMCKOj4i5ETF3yZIlvVagJPUXVTbQhrEkDRyLgBF14zsBiztZdjLreccwM8/NzLGZOXbYsGG9WKIk9Q9VNtCGsSQNHDcBoyJil4gYQi2XZ3RcKCK2AQ4EftbH9UlSv1FlA20YS9IAkZmrgJOA2cB8YHpmzouIKRExpW7Rw4FfZebyVtQpSf3BoKo2nJmrIqI9jNuA89vDuJg/tVjUMJakfiAzZwIzO0yb2mF8GjCt76qSpP6nsgYaDGNJkiRtePwmQkmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEmSJKkEG2hJkiSpBBtoSZIkqQQbaEkSABExISLuiogFEfGpTpY5KCJujYh5EXFdX9coSf1BpQ20YSxJA0NEtAHnAG8GRgNHRMToDsu8EPgW8NbMHAO8q6/rlKT+YFBVG64L4zcCi4CbImJGZt5Rt8wLqYXxhMx8MCK2q6oeSdJ6jQMWZOa9ABFxMTARuKNumSOBKzLzQYDMfLjPq5SkfqDKM9BrwzgzVwLtYVzPMJak/mE4sLBufFExrd4rgBdFxG8i4uaIOKbRhiLi+IiYGxFzlyxZUlG5ktQ6VTbQhrEkDRzRYFp2GB8E7Au8BXgT8NmIeMU6K2Wem5ljM3PssGHDer9SSWqxyi7hoFwY/yuwOfCHiJiTmXc/b6XMc4FzAcaOHdtxG5KknlsEjKgb3wlY3GCZRzJzObA8Iq4H9gTuRpI2IlWegW42jK/MzOWZ+QjQHsaSpL51EzAqInaJiCHAZGBGh2V+BrwhIgZFxBbAa4D5fVynJLVclQ20YSxJA0RmrgJOAmZTy+HpmTkvIqZExJRimfnAlcBfgBuB8zLz9lbVLEmtUtklHJm5KiLaw7gNOL89jIv5UzNzfkS0h/EaDGNJapnMnAnM7DBtaofx04HT+7IuSepvqrwG2jCWJEnSBsdvIpQkSZJKsIGWJEmSSrCBliRJkkqwgZYkSZJKsIGWJEmSSrCBliRJkkqwgZYkSZJKsIGWJEmSSrCBliRJkkqwgZYkSZJKsIGWJEmSSuiygY6I7SPiexExqxgfHRHvq740SVJZZrYkVa+ZM9DTgNnAjsX43cBHKqpHktQz0zCzJalSzTTQ22bmdGANQGauAlZXWpUkqbvMbEmqWDMN9PKIGAokQETsDzxeaVWSpO4ysyWpYoOaWOZjwAzg5RFxAzAMeGelVUmSusvMlqSKrbeBjog24MDi8UoggLsy89k+qE2SVIKZLUl9Y72XcGTmamBiZq7KzHmZebtBLEn9k5ktSX2jmUs4boiIs4FLgOXtEzPzlsqqkiR1l5ktSRVrpoF+XfHvF+qmJTC+98uRJPWQmS1JFeuygc7Mg/uiEElSz5nZklS9Zr6JcJuI+J+ImFs8zoiIbfqiOElSOWa2JFWvmftAnw88AUwqHsuA71dZlCSp28xsSapYM9dAvzwz31E3fmpE3FpRPZKknjGzJalizZyBXhERB7SPRMTrgRXVlSRJ6gEzW5Iq1swZ6BOAC+quofsncGxlFUmSesLMlqSKNXMXjluBPSNi62J8WdVFSZK6pyeZHRETgP8F2oDzMvOrHeYfBPwMuK+YdEVm1t8uT5I2Cs3cheO/I+KFmbksM5dFxIsi4kvNbDwiJkTEXRGxICI+1WD+QRHxeETcWjw+150nIUmq6W5mF18Dfg7wZmA0cEREjG6w6G8zc6/iYfMsaaPUzDXQb87Mx9pHMvOfwKFdrWQYS1JLdCuzgXHAgsy8NzNXAhcDE6spUZIGtmYa6LaI2LR9JCI2BzZdz/LtDGNJ6nvdzezhwMK68UXFtI5eGxF/johZETGmZ6VK0sDUzIcIfwhcExHfp/Z1sP8BXNDEeo3C+DUNlnttRPwZWAycnJnzOi4QEccDxwPsvPPOTexakjZa3c3saDAtO4zfArw0M5+MiEOBnwKj1tmQmS1pA9fMhwhPi4i/AIdQC9gvZubsJrbda2GcmecC5wKMHTu24zYkSYUeZPYiYETd+E7UTmzUb3tZ3fDMiPhWRGybmY90WM7MlrRBa+ZDhFsCv8rMk6kF4qYRMbiJbTcVxpn5ZDE8ExgcEds2W7wk6fl6kNk3AaMiYpeIGAJMBmZ02PYOERHF8Dhqv0OW9uoTkKQBoJlroK8HNouI4cDVwHHAtCbWM4wlqe91K7MzcxVwEjAbmA9Mz8x5ETElIqYUi70TuL247O6bwOTM9AyzpI1OM9dAR2Y+FRHvA84q3h78U1crZeaqiGgP4zbg/PYwLuZPpRbGJ0TEKmrflGUYS1LPdCuzYe07gTM7TJtaN3w2cHavVitJA1BTDXREvBY4CnhfifUMY0nqe93ObElSc5q5hOPDwKeBnxRnkF8GXFttWZKkbjKzJalizdyF43pq19S1j98LfKjKoiRJ3WNmS1L1mjkDLUmSJKlgAy1JkiSV0Mx9oF/fzDRJUuuZ2ZJUvWbOQJ/V5DRJUuuZ2ZJUsU4/RFjcBul1wLCI+FjdrK2p3ddZktRPmNmS1HfWdxeOIcALimW2qpu+jNoXoEiS+g8zW5L6SKcNdGZeB1wXEdMy8wGAiNgEeEFmLuurAiVJXTOzJanvNHMN9FciYuuI2BK4A7grIj5ecV2SpO4xsyWpYs000KOLsxdvo/a13DsDR1dZlCSp28xsSapYMw304IgYTC2Mf5aZzwJZaVWSpO4ysyWpYs000N8B7ge2BK6PiJdS+1CKJKn/MbMlqWLruwsHAJn5TeCbdZMeiIiDqytJktRdZrYkVa+ZbyLcPiK+FxGzivHRwHsrr0ySVJqZLUnVa+YSjmnAbGDHYvxu4CMV1SNJ6plpmNmSVKlmGuhtM3M6sAYgM1cBqyutSpLUXWa2JFWsmQZ6eUQMpfgUd0TsDzxeaVWSpO4ysyWpYl1+iBD4GDADeHlE3AAMA95VaVWSpO4ysyWpYs000POAA4FXAgHcRXNnriVJfc/MlqSKNROqf8jMVZk5LzNvL27K/4eqC5MkdYuZLUkV6/QMdETsAAwHNo+IvamdyQDYGtiiD2qTJDXJzJakvrO+SzjeBBwL7AScwXNhvAz4TLVlSZJKMrMlqY902kBn5gXABRHxjsy8vA9rkiSVZGZLUt/p8hpog1iSBg4zW5Kq5yezJUmSpBJsoCVJkqQSmmqgI+J1EXFkRBzT/mhyvQkRcVdELIiIT61nuf0iYnVEvLPZwiVJjZnZklStLr9IJSIuBF4O3AqsLiYn8IMu1msDzgHeCCwCboqIGZl5R4PlvgbMLlu8JOn5zGxJql4z30Q4FhidmVly2+OABZl5L0BEXAxMBO7osNwHgcuB/UpuX5K0LjNbkirWzCUctwM7dGPbw4GFdeOLimlrRcRw4HBgaje2L0laV8szOyKOj4i5ETF3yZIl3ShFkvq3Zs5AbwvcERE3As+0T8zMt3axXjSY1vGMyJnAJzNzdUSjxYsNRRwPHA+w8847N1GyJG20Wp7ZmXkucC7A2LFjy54Jl6R+r5kG+pRubnsRMKJufCdgcYdlxgIXF0G8LXBoRKzKzJ/WL2QYS1LTTunmer2W2ZK0oeuygc7M67q57ZuAURGxC/A3YDJwZIdt79I+HBHTgF8YxJLUfWa2JFWvy2ugI2L/iLgpIp6MiJXFrYuWdbVeZq4CTqL2Se35wPTMnBcRUyJiSs9LlyR1ZGZLUvWauYTjbGpnIi6l9vbdMcCoZjaemTOBmR2mNfzwSWYe28w2JUnrZWZLUsWaaaDJzAUR0ZaZq4HvR8TvK65LktRNZrYkVauZBvqpiBgC3BoRpwF/B7astixJUjeZ2ZJUsWbuA310sdxJwHJqn9J+R5VFSZK6zcyWpIo1cxeOByJic+AlmXlqH9QkSeomM1uSqtfMXTgOA24FrizG94qIGRXXJUnqBjNbkqrXzCUcpwDjgMcAMvNWYGRVBUmSeuQUzGxJqlQzDfSqzHy88kokSb3BzJakijVzF47bI+JIoC0iRgEfArwlkiT1T2a2JFWsmTPQHwTGAM8AFwHLgI9UWJMkqfvMbEmqWDN34XgK+H/FQ5LUj5nZklS9LhvoiBgLfIbah1DWLp+Zr66uLElSd5jZklS9Zq6B/hHwceA2YE215UiSesjMlqSKNdNAL8lM7yEqSQODmS1JFWumgf58RJwHXEPtQykAZOYVlVUlSeouM1uSKtZMA30csBswmOfeDkzAMJak/sfMlqSKNdNA75mZr6q8EklSbzCzJalizdwHek5EjK68EklSbzCzJalizZyBPgB4b0TcR+16ugDSWyJJUr9kZktSxZppoCdUXoUkqbeY2ZJUsWa+ifCBvihEktRzZrYkVa+Za6AlSZIkFWygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBIqbaAjYkJE3BURCyLiUw3mT4yIv0TErRExNyIOqLIeSVLnzGxJak4zX+XdLRHRBpwDvBFYBNwUETMy8466xa4BZmRmRsSrgenAblXVJElqzMyWpOZVeQZ6HLAgM+/NzJXAxcDE+gUy88nMzGJ0SyCRJLWCmS1JTaqygR4OLKwbX1RMe56IODwi7gR+CfxHow1FxPHF24VzlyxZUkmxkrSRM7MlqUlVNtDRYNo6Zysy8yeZuRvwNuCLjTaUmedm5tjMHDts2LDerVKSBGa2JDWtygZ6ETCibnwnYHFnC2fm9cDLI2LbCmuSJDVmZktSk6psoG8CRkXELhExBJgMzKhfICJ2jYgohvcBhgBLK6xJktSYmS1JTarsLhyZuSoiTgJmA23A+Zk5LyKmFPOnAu8AjomIZ4EVwLvrPqAiSeojZrYkNa+yBhogM2cCMztMm1o3/DXga1XWIElqjpktSc3xmwglSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSbKAlSZKkEmygJUmSpBJsoCVJkqQSKm2gI2JCRNwVEQsi4lMN5h8VEX8pHr+PiD2rrEeS1DkzW5KaU1kDHRFtwDnAm4HRwBERMbrDYvcBB2bmq4EvAudWVY8kqXNmtiQ1r8oz0OOABZl5b2auBC4GJtYvkJm/z8x/FqNzgJ0qrEeS1DkzW5KaVGUDPRxYWDe+qJjWmfcBsyqsR5LUOTNbkpo0qMJtR4Np2XDBiIOphfEBncw/HjgeYOedd+6t+iRJzzGzJalJVZ6BXgSMqBvfCVjccaGIeDVwHjAxM5c22lBmnpuZYzNz7LBhwyopVpI2cma2JDWpygb6JmBUROwSEUOAycCM+gUiYmfgCuDozLy7wlokSetnZktSkyq7hCMzV0XEScBsoA04PzPnRcSUYv5U4HPAUOBbEQGwKjPHVlWTJKkxM1uSmlflNdBk5kxgZodpU+uG/xP4zyprkCQ1x8yWpOb4TYSSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJg1pdQG949tlnWbRoEU8//TRr1qxpSQ3f/W5r9gswf/78bq+7vuM1ZMgQhg7dlkGDNoiXiaR+wsyuJrPB3Jb6ygbxE7Zo0SK22morRo4c2bIwXr16VUv2C7D77pt2e93Vq1c3nJ6ZLF36KEuXPsL22+/Q7e1LUkdmdu9nNpjbUl/aIC7hePrppxk6dCgR0epSNhgRwdChL2blypWtLkXSBsbMroa5LfWdDaKBBgziCnhMJVXFfKmGx1XqGxtMA91f7L33FkyaNI63v30fTj75SFaseAqARx75B5/4xNG85S27c/jhe3HiiRO5//6/rl3vwgu/yX77bcMTTzzeqtIlaaNjZkvqDhvoXrbpppszffqNXHHFLQwePJhLL/0umclHP/puxo79F375y/n85Ce38sEPfoFHH31o7XqzZk1nzJh9+fWvf9bC6iVp42JmS+oOG+gK7b33ASxceA833vgbBg0axKRJ7187b7fd9mSffQ4AYOHCe1ix4klOOukUZs2a3qpyJWmjZmZLapYNdEVWrVrFDTfMZtSoPViw4A5Gj96n02VnzZrOhAmT2GefA3jggbtZuvThPqxUkmRmSyrDBrqXPfPMCiZNGseRR76OHXYYweGHH9vlOldeeSkTJkxik002Yfz4t3HVVVdUX6gkycyW1C0bxH2g+5P26+nq7brr7lx9deOAvfvu23jwwQVMmXIoUPuCgeHDRzJ58pTKa5WkjZ2ZLak7PAPdB8aNO5iVK1dy+eXfWzvt9tvnMnfu9cyadQknnPBfzJp1N7Nm3c3VV9/Hww8vZvHiB1pYsSRtvMxsSV2xge4DEcE3vnEJc+b8urgl0t5Mnfolhg3bkSuvvJTx4yc+b/nx4ydy5ZWXtqhaSdq4mdmSuuIlHL1szpylDadvt92OnH76j9aZPmvWXetM+/jHT+v1uiRJ6zKzJXVHpWegI2JCRNwVEQsi4lMN5u8WEX+IiGci4uQqa5EkrZ+ZLUnNqewMdES0AecAbwQWATdFxIzMvKNusUeBDwFvq6oOSVLXzGxJal6VZ6DHAQsy897MXAlcDDzvwrHMfDgzbwKerbAOSVLXzGxJalKVDfRwYGHd+KJiWmkRcXxEzI2IuUuWLOmV4iRJz2NmS1KTqmygo8G07M6GMvPczBybmWOHDRvWw7IkSQ2Y2ZLUpCob6EXAiLrxnYDFFe5PktR9ZrYkNanKBvomYFRE7BIRQ4DJwIwK99dSe++9BZMmjePtb9+Hk08+khUrngLgkUf+wSc+cXRxL9G9OPHEidx//1/Xrnfhhd9kv/224YknHm9V6ZIEZjZgZktqTmV34cjMVRFxEjAbaAPOz8x5ETGlmD81InYA5gJbA2si4iPA6Mxc1pN9L1/+VM+K72DLLbfocpn6r4P99Kffy6WXfpejj/4QH/3ouznssPdw2mkXAnDnnX/m0UcfYuTIUQDMmjWdMWP25de//hkTJx7Tq3VLUrPMbDNbUvMq/SKVzJwJzOwwbWrd8D+ovU24Qdl77wP4619v48Ybf8OgQYOYNOn9a+ftttuea4cXLryHFSue5GMf+ybnnXeaYSyppcxsM1tSc/wq7162atUqbrhhNqNG7cGCBXcwevQ+nS47a9Z0JkyYxD77HMADD9zN0qUP92GlkiQzW1J32ED3kmeeWcGkSeM48sjXscMOIzj88GO7XOfKKy9lwoRJbLLJJowf/zauuuqK6guVJJnZknqk0ks4Nib119O123XX3bn66sYBe/fdt/HggwuYMuVQAJ599lmGDx/J5MlTKq9VkjZ2ZraknvAMdIXGjTuYlStXcvnl31s77fbb5zJ37vXMmnUJJ5zwX8yadTezZt3N1Vffx8MPL2bx4gdaWLEkbbzMbEnNsoGuUETwjW9cwpw5vy5uibQ3U6d+iWHDduTKKy9l/PjnfUsu48dP5MorL21RtZK0cTOzJTVrg7yEo5lbGPW2OXOWNpy+3XY7cvrpP1pn+qxZd60z7eMfP63X65Kk/s7MljTQeAZakiRJKsEGWpIkSSrBBlqSJEkqwQZakiRJKsEGWpIkSSrBBlqSJEkqwQa6l+y//9C1w7/97ZUcdtgY/v73Byvb389+9gP++78/AsApp5zC8OHD2Wuvvdhjjz2YMWPG2uV+8IMfsMceezBmzBhGjx7N17/+9bXzVq1axfbbb89nPvOZyuqUpP7IzJbUExvkfaB33XVhr25vwYIRTS/7xz/+mq9+9aNMnfoLXvKSnZtaZ/Xq1bS1tXW3PAA++tGPcvLJJzN//nze8IY38PDDDzN79mzOPPNMfvWrX7Hjjjvy9NNPc+GFF65d51e/+hWveMUruOyyy/jyl79MRPSoBknqDjPbzJYGGs9A96Jbbvkdp576fzj77J8yYsTLAfjFL37MkUcewKRJ4/jCF05k9erVQO3sxznnnMpRR72BP/95DvvvP5Szzvoc73rXfrznPf/C0qUPAfDoo0v42Mcmc+SRr+fII1/Pn/70+/XWsPvuuzNo0CAeeeQRvvKVr/D1r3+dHXfcEYDNNtuM97///WuXveiii/jQhz7EiBEjmDNnThWHRJL6LTNbUnfZQPeSZ599hg9/+F2ceeal7LLLKwG49947mT37Mi644FqmT7+RtrY2Zs68CIAVK5az665j+NGPfss++7yeFSuW86pXvYZLL72Jffc9gMsvPx+A0077v7znPR/kxz++gTPOuIhTTz1hvXX88Y9/ZJNNNmHYsGHcfvvt7Lvvvg2XW7FiBddccw1vectbmDx5MpdcckkvHg1J6t/MbEk9sUFewtEKgwYNZs899+cnP5nGJz95BgB//OO1zJ//J4466vUAPP30Cl784mEAtLW1ccghh69df/DgIRx44KEA7L773syZcw0Ac+Zcy7333rl2uSeffILly59YZ//f+MY3+OEPf8hWW23FJZdc0uVbe7/4xS84+OCD2WKLLXj729/Ol7/8Zc4444wevy0pSQOBmS2pJ2yge0nEJpx++o/4wAcO5bzzvsZ//ucnyUwOO+woPvzhL62z/JAhmz0v+AYNGrw2QNva2li9ehUAmWv4wQ+uY7PNNl/v/tuvp6s3ZswYbr75ZsaPH7/O8hdddBE33HADL3957W3LpUuXcu2113LIIYeUe+KSNACZ2ZJ6wks4etHmm2/BWWddwS9/eTFXXPF9XvOag7n66p+wdOnDADz++KMsXvxAqW2+9rX/ysUXf3vt+J13/rnpdT/96U/ziU98gn/84x8APPPMM3zzm99k2bJl/O53v+PBBx/knnvu4Z577uGss87yLUFJGxUzW1J32UD3sm22eTHf/vYMvvvdr/Lggws48cRTOOGEf+ed7xzLBz7wFh555B+ltvfJT/4P8+bdwjvfOZbDD9+LSy/9btPrHnrooZx44okccsghjBkzhn333ZdVq1ZxxRVXMH78eDbddNO1y771rW/l5z//Oc8880yp+iRpIDOzJXVHZGarayhl7NixOXfu3OdNmz9/PrvvvjvA2k9M97V581a1ZL8Ar371pl0v1Imujtedd97JiBHN3dqpjD33XNrr22zWffeN7Pa6y5atey1jX/B4ldeqY9bV8YqImzNzbN9U03pm9rqqzGzY8HLbDCrH41VOdzPbM9CSJElSCTbQkiRJUgk20JIkSVIJG0wDPdCu5R4IPKaSqmK+VMPjKvWNDaKB3myzzVi6dKnB0Ysyk6VLH2XIkCGtLkXSBsbMroa5LfWdDeKLVHbaaScWLVrEkiVLWLNmTUtqePjh1uwXYP787v83ru94DRkyhKFDt+32tiWpETO7mswGc1vqK5U20BExAfhfoA04LzO/2mF+FPMPBZ4Cjs3MW8ruZ/Dgweyyyy5A627B8u//7m3GJA1sZnbfMLOlga+ySzgiog04B3gzMBo4IiJGd1jszcCo4nE88G0kSX3OzJak5lV5DfQ4YEFm3puZK4GLgYkdlpkI/CBr5gAvjIiXVFiTJKkxM1uSmlRlAz0cWFg3vqiYVnYZSVL1zGxJalKV10BHg2kdP3LdzDJExPHU3i4EeDIi7uphbVXYFnikFTuORkex//N4lePxKq8lx6yJ4/XSPiijO8zsPuLPVDker3I8XuV0N7OrbKAXASPqxncCFndjGTLzXODc3i6wN0XE3Ebfla7GPF7leLzK85iVZmZrvTxm5Xi8yhlox6vKSzhuAkZFxC4RMQSYDMzosMwM4Jio2R94PDP/XmFNkqTGzGxJalJlZ6Azc1VEnATMpnZLpPMzc15ETCnmTwVmUrsd0gJqt0Q6rqp6JEmdM7MlqXmV3gc6M2dSC9z6aVPrhhM4scoa+lC/fruyH/J4lePxKs9jVpKZrS54zMrxeJUzoI5X+FWqkiRJUvOqvAZakiRJ2uDYQPdQRJwfEQ9HxO2trmWgiIj7I+K2iLg1Iua2up7+ptFrKiJeHBFXRcRfi39f1Moa+5tGrymPmRoxs8szs9fPzC5vQ8hsG+iemwZMaHURA9DBmbnXQLplTR+axrqvqU8B12TmKOCaYlzP1/E15TFTI9Mws7vDzO7cNMzs7hjQmW0D3UOZeT3waKvr0Iajk9fUROCCYvgC4G19WdMA5THTOsxs9TYzu9cMqGNmA61WSOBXEXFz8Y1l6tr27ffbLf7drsX19DeNXlMeM6l3mNnlmT/rN+Azu9Lb2EmdeH1mLo6I7YCrIuLO4i94qbvWeU21uiBpA2Jmq7cN+Mz2DLT6XGYuLv59GPgJMK61FQ0ID0XESwCKfx9ucT39SievKY+Z1AvM7G4xf9ZjQ8hsG2j1qYjYMiK2ah8G/g3w0/BdmwG8txh+L/CzFtbSr6znNeUxk3rIzO4286cTG0pm+0UqPRQRFwEHAdsCDwGfz8zvtbSofiwiXkbtr02oXUL048z8cgtL6ncavaaAnwLTgZ2BB4F3ZaYfhKLz11REDMVjpg7M7HLM7K6Z2eVsKJltAy1JkiSV4CUckiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtBSPxMRH4mILVpdhySpa2b2xsnb2En9TETcD4zNzEdaXYskaf3M7I2TZ6DVZyJiZETcGREXRMRfIuKy9r/aI2K/iPh9RPw5Im6MiK2K5X8bEbcUj9d1st1jiu39OSIuLKa9NCKuKaZfExE7F9OnRcS3I+LaiLg3Ig6MiPMjYn5ETKvb5pMRcUax32siYlgxfa+ImFNs9ycR8aJi+m8i4mtF7XdHxBuK6W0RcXpE3FSs84Fi+kHFOpcVx+RHUfMhYEfg2qLGtqLm2yPitoj4aGX/QZJUx8w2s7UemenDR588gJFAAq8vxs8HTgaGAPcC+xXTt6b27URbAJsV00YBcxtscwxwF7BtMf7i4t+fA+8thv8D+GkxPA24GAhgIrAMeBW1PyZvBvYqlkvgqGL4c8DZxfBfgAOL4S8AZxbDvwHOKIYPBa4uho8H/qsY3hSYC+xC7VurHgd2Kvb9B+CAYrn7657PvsBVdc/3ha3+f/Thw8fG8TCzzWwfnT88A62+tjAzbyiGfwgcALwS+Htm3gSQmcsycxUwGPhuRNwGXAqMbrC98cBlWbx1ls997edrgR8XwxcW+2n386wl223AQ5l5W2auAeZR+4UBsAa4pL7OiNiGWhheV0y/APiXuu1eUfx7c912/g04JiJuBf4IDKX2iwXgxsxcVOz71rp16t0LvCwizoqICdR+eUhSXzGzzWw1MKjVBWij0/Gi+6R2ZqHRxfgfBR4C9qT2F//TDZbpbN317feZ4t81dcPt4539TDSzj/Ztra7bTgAfzMzZ9QtGxEEd9l2/znM7zfxnROwJvAk4EZhE7eyMJPUFMxszW+vyDLT62s4R8dpi+Ajgd8CdwI4RsR9AcS3dIGAbamc51gBHA20NtncNMCkihhbrvriY/ntgcjF8VLGfMjYB3lkMHwn8LjMfB/7Zfq1cUdN1jVauMxs4ISIGF/W9IiK27GKdJ4CtiuW3BTbJzMuBzwL7lHwektQTZraZrQY8A62+Nh94b0R8B/gr8O3MXBkR7wbOiojNgRXAIcC3gMsj4l3AtcDyjhvLzHkR8WXguohYDfwJOBb4EHB+RHwcWAIcV7LO5cCYiLiZ2nVv7y6mvxeYWnyQ5t4mtnsetbf5bomIKGp5WxfrnAvMioi/Ax8Bvh8R7X/sfrrc05CkHjGzzWw14G3s1GciYiTwi8zco9W1dCUinszMF7S6DklqFTNb6pyXcEiSJEkleAZakiRJKsEz0JIkSVIJNtCSJElSCTbQkiRJUgk20JIkSVIJNtCSJElSCTbQkiRJUgn/H/gd7W1u5v4KAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,[ax1,ax2]=plt.subplots(1,2,figsize=(12,6))\n",
    "sns.barplot(data=gs_rf_report[gs_rf_report['param_classifier__n_estimators']==100],\n",
    "            x=\"param_reduce_dim__n_components\", \n",
    "            y=\"mean_test_score\", \n",
    "            hue=\"param_reduce_dim\", \n",
    "            color=\"b\",\n",
    "            ax=ax1)\n",
    "ax1.set(ylabel='mean test score', xlabel='pca components', title='Random Forest Performance (n=100)')\n",
    "handles, labels=ax1.get_legend_handles_labels()\n",
    "ax1.legend(handles, ['PCA','KernelPCA'],title='PCA', loc='lower left');\n",
    "\n",
    "sns.barplot(data=gs_rf_report[gs_rf_report['param_classifier__n_estimators']==500],\n",
    "            x=\"param_reduce_dim__n_components\", \n",
    "            y=\"mean_test_score\", \n",
    "            hue=\"param_reduce_dim\", \n",
    "            color=\"b\",\n",
    "            ax=ax2)\n",
    "ax2.set(ylabel='mean test score', xlabel='pca components', title='Random Forest Performance (n=500)')\n",
    "handles, labels=ax1.get_legend_handles_labels()\n",
    "ax2.legend(handles, ['PCA','KernelPCA'],title='PCA', loc='lower left');\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Regression, Training score: 0.966\n",
      "Random Forest Regression, Test score: 0.862\n",
      "---------------------------\n",
      "Random Forest Regression, Training RMSE: 0.075\n",
      "Random Forest Regression, Test RMSE: 0.162\n"
     ]
    }
   ],
   "source": [
    "print(\"Random Forest Regression, Training score: %.3f\" %gs_rf.score(X_train, y_train))\n",
    "print(\"Random Forest Regression, Test score: %.3f\" % gs_rf.score(X_test, y_test))\n",
    "print('---------------------------')\n",
    "print(\"Random Forest Regression, Training RMSE: %.3f\" % logrmse(gs_rf.predict(X_train), y_train))\n",
    "print(\"Random Forest Regression, Test RMSE: %.3f\" % logrmse(gs_rf.predict(X_test), y_test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosting Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensemble of weak classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.inspection import permutation_importance\n",
    "#feature_importances_\n",
    "from sklearn.ensemble import GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_boost = Pipeline(steps=[('preprocessor',preprocessor),\n",
    "              ('classifier', GradientBoostingRegressor(loss='lad',min_samples_split=5))])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will find the effect of learning rate and maximum depth of weak classifiers on the Gradient Boosting Regression performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_estimators_gb = [500,1000, 1200]\n",
    "lr = [0.1, 0.05, 0.01, 0.001]\n",
    "max_depth = [3,4,5]\n",
    "param_grid = [\n",
    "    {\n",
    "        'classifier__learning_rate' : lr,\n",
    "        'classifier__n_estimators': n_estimators_gb,\n",
    "        'classifier__max_depth': max_depth\n",
    "    }\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_gb = GridSearchCV(pipe_boost,n_jobs=3, param_grid=param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_gb.fit(X_train,y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_gb_report=pd.DataFrame(gs_gb.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_classifier__learning_rate</th>\n",
       "      <th>param_classifier__max_depth</th>\n",
       "      <th>param_classifier__n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>45.829974</td>\n",
       "      <td>8.109150</td>\n",
       "      <td>0.999174</td>\n",
       "      <td>0.234132</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>500</td>\n",
       "      <td>{'classifier__learning_rate': 0.1, 'classifier...</td>\n",
       "      <td>0.847526</td>\n",
       "      <td>0.845442</td>\n",
       "      <td>0.769678</td>\n",
       "      <td>0.864916</td>\n",
       "      <td>0.903577</td>\n",
       "      <td>0.846228</td>\n",
       "      <td>0.043590</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>53.136551</td>\n",
       "      <td>8.436962</td>\n",
       "      <td>1.201640</td>\n",
       "      <td>0.373167</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>1000</td>\n",
       "      <td>{'classifier__learning_rate': 0.1, 'classifier...</td>\n",
       "      <td>0.830612</td>\n",
       "      <td>0.869893</td>\n",
       "      <td>0.783594</td>\n",
       "      <td>0.867740</td>\n",
       "      <td>0.910808</td>\n",
       "      <td>0.852529</td>\n",
       "      <td>0.042806</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52.428347</td>\n",
       "      <td>10.070871</td>\n",
       "      <td>0.966829</td>\n",
       "      <td>0.184031</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>1200</td>\n",
       "      <td>{'classifier__learning_rate': 0.1, 'classifier...</td>\n",
       "      <td>0.842007</td>\n",
       "      <td>0.842248</td>\n",
       "      <td>0.810565</td>\n",
       "      <td>0.851997</td>\n",
       "      <td>0.909455</td>\n",
       "      <td>0.851255</td>\n",
       "      <td>0.032281</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48.648168</td>\n",
       "      <td>8.558362</td>\n",
       "      <td>1.316960</td>\n",
       "      <td>0.121633</td>\n",
       "      <td>0.1</td>\n",
       "      <td>4</td>\n",
       "      <td>500</td>\n",
       "      <td>{'classifier__learning_rate': 0.1, 'classifier...</td>\n",
       "      <td>0.815260</td>\n",
       "      <td>0.863391</td>\n",
       "      <td>0.762700</td>\n",
       "      <td>0.884175</td>\n",
       "      <td>0.911568</td>\n",
       "      <td>0.847419</td>\n",
       "      <td>0.052787</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55.778568</td>\n",
       "      <td>8.766752</td>\n",
       "      <td>1.110395</td>\n",
       "      <td>0.270809</td>\n",
       "      <td>0.1</td>\n",
       "      <td>4</td>\n",
       "      <td>1000</td>\n",
       "      <td>{'classifier__learning_rate': 0.1, 'classifier...</td>\n",
       "      <td>0.822805</td>\n",
       "      <td>0.877851</td>\n",
       "      <td>0.787018</td>\n",
       "      <td>0.871793</td>\n",
       "      <td>0.898453</td>\n",
       "      <td>0.851584</td>\n",
       "      <td>0.040724</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0      45.829974      8.109150         0.999174        0.234132   \n",
       "1      53.136551      8.436962         1.201640        0.373167   \n",
       "2      52.428347     10.070871         0.966829        0.184031   \n",
       "3      48.648168      8.558362         1.316960        0.121633   \n",
       "4      55.778568      8.766752         1.110395        0.270809   \n",
       "\n",
       "  param_classifier__learning_rate param_classifier__max_depth  \\\n",
       "0                             0.1                           3   \n",
       "1                             0.1                           3   \n",
       "2                             0.1                           3   \n",
       "3                             0.1                           4   \n",
       "4                             0.1                           4   \n",
       "\n",
       "  param_classifier__n_estimators  \\\n",
       "0                            500   \n",
       "1                           1000   \n",
       "2                           1200   \n",
       "3                            500   \n",
       "4                           1000   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "0  {'classifier__learning_rate': 0.1, 'classifier...           0.847526   \n",
       "1  {'classifier__learning_rate': 0.1, 'classifier...           0.830612   \n",
       "2  {'classifier__learning_rate': 0.1, 'classifier...           0.842007   \n",
       "3  {'classifier__learning_rate': 0.1, 'classifier...           0.815260   \n",
       "4  {'classifier__learning_rate': 0.1, 'classifier...           0.822805   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0           0.845442           0.769678           0.864916           0.903577   \n",
       "1           0.869893           0.783594           0.867740           0.910808   \n",
       "2           0.842248           0.810565           0.851997           0.909455   \n",
       "3           0.863391           0.762700           0.884175           0.911568   \n",
       "4           0.877851           0.787018           0.871793           0.898453   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "0         0.846228        0.043590               19  \n",
       "1         0.852529        0.042806               10  \n",
       "2         0.851255        0.032281               15  \n",
       "3         0.847419        0.052787               17  \n",
       "4         0.851584        0.040724               13  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_gb_report.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAGDCAYAAAACpSdYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAACQe0lEQVR4nOzdeZxbdb3/8dfnnJNk9qUbXaalhQItXSjQllUWpVD2RQQEtMDlIiBw3cDdi/hTEGUTUS4XEBClKCBUQODCtVcQkLaCQAul29BON7q3syc5398f5yQ5yWRmMvtk5vN8PKaTnJxz8k0m/eadb76LGGNQSimllFJK5cbq6wIopZRSSimVTzRAK6WUUkop1QEaoJVSSimllOoADdBKKaWUUkp1gAZopZRSSimlOkADtFJKKaWUUh2gAXqAEJG9RORvIrJHRG7r6/L0NREpFJE/i8guEfljH5bjRhF51L88TkRqRcTuq/L0NhH5lIgs7+tyKNXfaJ2dTuvs/kHr7NxpgO5DIlItIg3+f9DNIvIbESnp5OmuALYCZcaYr3djMfPVucBewFBjzOey7SAi+4nIfBHZIiK7RWSFiNwtIlU9USBjzFpjTIkxJt7Vc4nIQhG5vI3bx4uI8V9bidfXr0Qk1NX7bqdcRkQmJq4bY141xhzQk/epVG/ROrtHaZ2tdXZe0QDd9043xpQAhwCzgO915GDxWMDewDLTiZVxRMTp6DF5YG/gI2NMLNuNfoXxD2ADcLAxpgw4ClgFHN3KMfn4PFX4r69pwBHAl/u4PErlO62ze4bW2R6ts/OFMUZ/+ugHqAZOCFz/GfCsf/lw4HVgJ/Av4LjAfguBHwN/BxqAR4Eo0AzUAicAEeBOvMpmg3854h9/HFADfBPYBPwWuBH4o3+uPcB7wP7At4FPgHXAiYEyXAp84O+7GvhS4LbE+b/uH7sRuDRweyFwG/AxsAt4DShs73Fnef4m+8/FTmApcIa//Yf+cxH1n49/y3Lso8Cf2/n7ZHueKoFngS3ADv9yVeCYCcD/+c/L/wC/BB71bxsPGMDxr5cDD/jPz3rg/wG2f9sl/vPyc/9+1gAn+7f9GIgDjf7j+2WWsqfdl7/tVuC+9p6/QNke8R/nx3ghwfJvm+g/xl14LWiP+9v/5t9nnV+u8xPPYcZr/hvAu/7xjwMFgdtv8J+PDcDl/vkm9vX/Vf3RH2O0zkbrbK2ztc5O/c36ugCD+YdAZQyM9f9D/AgYA2wDTsH7lmCOf324v+9CYC0wBXCAEPAQ8P8C574JeBMYAQzHq+B+5N92HBADfopXaRfiVcaNwEn+OR/xK4Dv+uf/d2BN4PynAvsCAhwL1AOHZJz/Jv/YU/zbK/3b7/EfwxjABo70y9Hm48547kLASuA7QBj4NF4FeIB/+434lWArz/0m4JJ2/j7ZnqehwGeBIqAU7w3s6cAxbwC3+/sf45eptcr4aeC/gGL/7/QW/psaXmUc9Z93G7gKr4KSwGvg8jbKnnlfo/He3C7L8fl7BHjGf4zjgY/w39SAx/zXhQUUAEcH7jet8iR7ZfyWX54heG/oV/q3zfX/LlP85/e3mefTH/3pyx+0ztY6W+tsrbMTz01fF2Aw//gvzFq8T5MfA7/y/8N/E/htxr4vAvP8ywuBmzJuf4j0yngVcErg+klAtX/5OLxP+8FPkTcC/xO4frpftsSn61L/P0ZFK4/laeA/AudvIP2T9Cd4LRWWf9tBWc7R5uPO2P4p/z+uFdj2GHBj4PG0VRnHgLmB69f4f4da4L9be56ynGcGsMO/PM4/b3Hg9t+TpTLG6+vXhN+K49/+eeCv/uVLgJWB24r8Y0cGXgO5VMY7/R+D94Zc1t7zh1f5NwEHBm77ErDQv/wIcB+BVpzAfrlUxhcHrt8K3OtffhC4OXDbxMzz6Y/+9OUPWmdnnkPrbK2zB22drX2g+95ZxpgKY8zexpirjTENeH3BPiciOxM/eH28RgWOW9fOeUfjVfAJH/vbErYYYxozjtkcuNwAbDWpwRMN/u8SABE5WUTeFJHtfvlOAYYFjt9m0vuy1fvHDsP7BLwqS5lzedzBx7fOGONmPMYxWfbNZlvwvMaYXxpjKvC+Ng0O2kh7nkSkSET+S0Q+FpHdeF+BVfijtEfjVcx1GWXKZm//fjYGHut/4bVqJGwKlK/ev9jRAUvD/MdVhPf18Qv+9raev2F4LRyZr5/Ec3sDXivWWyKyVEQu62CZNgUuJ14XyTIFbmvvNa5UX9A6O0XrbK2zB22drQG6f1qH96m+IvBTbIy5JbCPaeccG/D+wyeM87flenyrRCQCPInX12sv/z/783j/QduzFe9rx32z3JbL407YAIz1B+MkjMPrl5aLV4Bzctgv83n6OnAAcJjxBrEc428XvH5glSJSnFGmbNbhtRgMCzzWMmPMlBzL36G/n/8m/xBwhIgMo+3nbyveV5GZr5/1/rk2GWP+3RgzGq+V41fBUdxdsBEIjqYf2w3nVKo3aJ2tdXZHy9X2zlpn93saoPunR4HTReQkEbFFpEBEjuvgVD2PAd8TkeH+f74f+OftDmG8/mJbgJiInAycmMuB/qfnB4HbRWS0//iO8Cv4jjzuf+ANfLhBREIichzeV5jzc3wMNwKfEpHbRWQMgP88TW7nuFK8lp2dIjIE+M/AY/sYWAz8UETCInK0X6Zsz8NG4CXgNhEpExFLRPYVkWNzLP9mYJ8c9028gX4BryVhG208f34L1h+AH4tIqYjsDXwN//UjIp8L/E124L0xJFq9OlSuDH8ALhWRySJShPeaVSofaJ2tdXZ7tM4eYDRA90PGmHXAmXiDBbbgffK9no79vf4fXsXwLt7o7H/627qjfHuA6/D+8+wALgQWdOAU3/DLtAjYjjfgw+rI4zbGNANnACfjffr+FfBFY8yHOT6Gj/D691UB/xKRPXhfl20Avt/GoXfi9Xncijfg54WM2y8EDvMf13/i9T1rzRfx3tiW4T2PT5D9q89s7gLOFZEdIvKLNvbbKSK1eJXkEXijtk0Oz9+1eJX1aryR5b/HexMFb+quf/jnXYDXj3KNf9uNwMP+V5zn5fhYADDG/AX4BfBXvMEyb/g3NXXkPEr1Nq2ztc7OgdbZA0xidKhSSvUrIjIZeB9vKq+sc8MqpZTqHwZbna0t0EqpfkNEzva/Sq3Ea+X682CoiJVSKh8N5jpbA7RSqj/5Et5Xwavw+uhd1bfFUUop1YZBW2drFw6llFJKKaU6QFuglVJKKaWU6gAN0EoppZRSSnWA09cF6Khhw4aZ8ePH93UxlFKqU5YsWbLVGDO8r8vRW7TOVkrls9bq7LwL0OPHj2fx4sV9XQyllOoUEWltqeABSetspVQ+a63O1i4cSimllFJKdYAGaKWUUkoppTpAA7RSSimllFIdkHd9oJVSSuW3aDRKTU0NjY2NfV0UpSgoKKCqqopQKNTXRVF5RAO0UkqpXlVTU0NpaSnjx49HRPq6OGoQM8awbds2ampqmDBhQl8XR+UR7cKhlFKqVzU2NjJ06FANz6rPiQhDhw7Vb0NUh2mAVkop1es0PKv+Ql+LqjM0QCullFJKKdUBGqCVUkqpHC1cuJDTTjut2853+eWXs2zZMgD++Mc/MnnyZI4//ngWL17Mdddd123301sWLlzI66+/nrx+77338sgjj3TLuX/yk590y3mU6g46iFAppVRei8ViOE5+vp3df//9ycsPPPAAv/rVrzj++OMBmDlzZs7n6S/PwcKFCykpKeHII48E4Morr+y2c//kJz/hO9/5Ts77G2MwxmBZ2laoup++qpRSSvW56upqJk2axLx585g+fTrnnnsu9fX13HTTTcyaNYupU6dyxRVXYIwB4LjjjuM73/kOxx57LHfddRd//vOfOeywwzj44IM54YQT2Lx5MwA33ngj8+bN48QTT2T8+PE89dRT3HDDDUybNo25c+cSjUZbLdOiRYs48sgjOeigg5g9ezZ79uxJu/2tt97iyCOP5OCDD+bII49k+fLlACxdupTZs2czY8YMpk+fzooVK6irq+PUU0/loIMOYurUqTz++OPJx7F48WJuuukmXnvtNa688kquv/76tJbuuro6LrvsMmbNmsXBBx/MM888A8BDDz3E5z73OU4//XROPPHEnJ/rhQsXctxxx3HuuecyadIkLrroouTzms2SJUs49thjOfTQQznppJPYuHEjAL/4xS848MADmT59OhdccAHV1dXce++93HHHHcyYMYNXX32VG2+8kZ///OfJx/rVr36VY445hsmTJ7No0SLOOecc9ttvP773ve8l7++ss87i0EMPZcqUKdx3330AfOtb36KhoYEZM2Zw0UUXAXD77bczdepUpk6dyp133gl4r6PJkydz9dVXc8ghh7Bu3TouueQSpk6dyrRp07jjjjtyfp6UalPiE1q+/Bx66KFGKaXyFbDY9IO6tLd+stXZy5Yta7FtzZo1BjCvvfaaMcaYSy+91PzsZz8z27ZtS+5z8cUXmwULFhhjjDn22GPNVVddlbxt+/btxnVdY4wx//3f/22+9rWvGWOM+c///E9z1FFHmebmZvPOO++YwsJC8/zzzxtjjDnrrLPMn/70pxZlMcaYpqYmM2HCBPPWW28ZY4zZtWuXiUaj5q9//as59dRT07YZY8z//M//mHPOOccYY8w111xjHn300eR56uvrzRNPPGEuv/zy5Pl37tyZfByLFi1qcTl4P9/+9rfNb3/7W2OMMTt27DD77befqa2tNb/5zW/MmDFj0p6jXPz1r381ZWVlZt26dSYej5vDDz/cvPrqq1n3bW5uNkcccYT55JNPjDHGzJ8/31x66aXGGGNGjRplGhsbk+Uyxnu+f/aznyWPD14/9thjzQ033GCMMebOO+80o0aNMhs2bDCNjY1mzJgxZuvWrcYYk3w89fX1ZsqUKcntxcXFyfMuXrzYTJ061dTW1po9e/aYAw880Pzzn/80a9asMSJi3njjjeR+J5xwQvK4RDkzZXtNKmVM63V233/fo1QPM8YQj8dx4y6uMbjxOPG4m9ruGty4S9yNY1kWgiACYlmIeJct/zKkLqftgzeS27vuXQ4eo5Rq39ixYznqqKMAuPjii/nFL37BhAkTuPXWW6mvr2f79u1MmTKF008/HYDzzz8/eWxNTQ3nn38+GzdupLm5OW1O35NPPplQKMS0adOIx+PMnTsXgGnTplFdXZ21LMuXL2fUqFHMmjULgLKyshb77Nq1i3nz5rFixQpEJNmafcQRR/DjH/+YmpqaZAvrtGnT+MY3vsE3v/lNTjvtND71qU/l/Ly89NJLLFiwINmS29jYyNq1awGYM2cOQ4YMyflcCbNnz6aqqgqAGTNmUF1dzdFHH91iv+XLl/P+++8zZ84cAOLxOKNGjQJg+vTpXHTRRZx11lmcddZZOd3vGWecAXjP/ZQpU5Ln2meffVi3bh1Dhw7lF7/4BX/6058AWLduHStWrGDo0KFp53nttdc4++yzKS4uBuCcc87h1Vdf5YwzzmDvvffm8MMPT5539erVXHvttZx66qkdaqlX/Yvreu/brmswicuJQOua5O3ePi7GkLafJcKwEUPbv6McaYBW/VIq2Mb90OsSj8f90Ot6Idh1iUVj3k88TjQaIx6LE4vF/J848VgM1zWICMYYRMAYQTAYAZLfWgqWlfhGRgAXkORx/i6If9H4GwQD4l1O7Bc8RhDEAkssxLawLQuxxP9tYVmCZdnYtoBY2LaFZVlYIti2jdiCJRaWbXnXOxnuLcuCxHYN96qfynxNighXX301ixcvZuzYsdx4441p8/UmwhPAtddey9e+9jXOOOMMFi5cyI033pi8LRKJAN7/g1AolPb/JRaLZS2LV1+0/X/k+9//Pscffzx/+tOfqK6u5rjjjgPgwgsv5LDDDuO5557jpJNO4v777+fTn/40S5Ys4fnnn+fb3/42J554Ij/4wQ9yel6MMTz55JMccMABadv/8Y9/pD0HHZF4TgBs227zeZgyZQpvvPFGi9uee+45/va3v7FgwQJ+9KMfsXTp0pzv17KstDIk/hYLFy7k5Zdf5o033qCoqIjjjjsu6xzNyXo5i+BzUllZyb/+9S9efPFF7rnnHv7whz/w4IMPtltO1VIwwOKH1GCATQVXkwq6xgu7bqAhKx53Ma5LLB7HdV3/PT31vm5cl7i/3fiNW977OGBSb9uSfO8FMKS9pQf2Md7bObZtaYBW/U+iNde4hrjr/Udw/U+EwVbeWCxGLB4j1uyF3ngsFgi+cWLRmPefJvVfxAu7kPzP4QVU1w+Kfgi1vdBpWV4AdRybcDiUU1AsKy8l2hxN3qPj2GzZvA2xJHlOsSQZPDvz3BjXYAh+/eN9Mo7GDZiod934n5hJVUbJ45M/3R/ubcuGZKjv/+E++MaZ9iaaUXMa2rgteA43/UbvOL+WbnGc97qLNseIFEZwwiEaahsoLClstbwqd2vXruWNN97giCOO4LHHHuPoo4/m9ddfZ9iwYdTW1vLEE09w7rnnZj12165djBkzBoCHH364y2WZNGkSGzZsYNGiRcyaNYs9e/ZQWJj+dw7e50MPPZTcvnr1avbZZx+uu+46Vq9ezbvvvsukSZMYMmQIF198MSUlJWn7t+ekk07i7rvv5u6770ZEePvttzn44IO7/BhzccABB7Bly5bk3yUajfLRRx8xefJk1q1bx/HHH8/RRx/N73//e2prayktLWX37t2dvr9du3ZRWVlJUVERH374IW+++WbytlAoRDQaJRQKccwxx3DJJZfwrW99C2MMf/rTn/jtb3/b4nxbt24lHA7z2c9+ln333ZdLLrmk02Xra2kBNdHiih9sAwEWf1swwBoDcTdOPOaHVL9RyjWGWCzuhda46x/n3xY3uG48+R4OibeUVIOUVy7xw61f6waSrGsM4r/nIaTeJ0Tw3gH8kItgSfIKlggG/PchB2xve6Ludt04uOBiEDfwHgmYuB/wjQsGP4vEaWpqfbxDZ2iAHsRSn/yyt/ImPgVGm6PEY3GiaWHXa+WNx11i0aj36ZBEwE218hIMeQYsOxWQEmFXLO9yOBKioCCSDKs5P464S31DA7V76qmrq6e+roH6unrqEr9rvW11dfXU1wa2+7/vfuBm/u2CryTP98D8O9OuZ/LK7nXXsETSQl962BY/MEogPAYu+wFdkse2tr//W7Kd2y9D4Fhv38B5M263LPHDc2obiUrND9WJgC6WX8GRWeFJMpx7lSEIXvcVRLATfz+xsCSxv3ffgP+8pQK19w2A9wryfvtB3fJb7sX/kGRZOHbiQ5ONY9vehxzbxrZtLFtwLAvbdrD915XtOH4ZLfxHCpIYmW9Sz0XiMUuiZcO7H5JddCBSVEDFuJHU7q4jFAmBgNsUZ9TeIwlHwjm/ZlV2kydP5uGHH+ZLX/oS++23H1dddRU7duxg2rRpjB8/PtmdIpsbb7yRz33uc4wZM4bDDz+cNWvWdKks4XCYxx9/nGuvvZaGhgYKCwt5+eWX0/a54YYbmDdvHrfffjuf/vSnk9sff/xxHn30UUKhECNHjuQHP/gBixYt4vrrr0+2gv/617/OuSzf//73+cpXvsL06dMxxjB+/HieffbZLj2+XIXDYZ544gmuu+46du3aRSwW4ytf+Qr7778/F198Mbt27cIYw1e/+lUqKio4/fTTOffcc3nmmWe4++67O3x/c+fO5d5772X69OkccMABya4YAFdccQXTp0/nkEMO4Xe/+x2XXHIJs2fPBrzpAA8++OAWXXLWr1/PpZdemgyWN998c6efi8wAm+ga4Lqu3yKbCnItW2uN15KaaFGNpQKs934bJxZLBVjXdZPvycn3auNiGQsXF0k0tvjtJMn3YPEbBQKBErxtluW11ibqcoPBEguTOJ+I3xBjAvt5x4olfmhPNDAkHpsXkklrKPKPkUTgTaYDCFxreSX5ROMXKHBj4DyJoBFoLvJzt7+P38jkH5d4fur21HX6b5+NtPU1SH80c+ZMs3jx4r4uRp9oq5U3+DVILO53Ychs5Y17XR4Srb2Zrbz+qz3wMjVttvImgm9nW2YT4vF4KuDWNVBXW09dbR11tQ1++PVCcCLwJgJyfX0jDfUNNDS0vwRrpCBCYUGEgoIIBYUFFBYUUFBYwLDhlVx+3Rc4/9R/T+772IL/4oWnX+GTjVtxjZuqDN1EpelmrSjTv7IygWPb3jfxSTlR+QT3D1bS2c5h0rZnH8SVOJaM86RVdnlWD3RUotXEkmAglvT+68kPE6k3mOSHCv8DQ9mQMn5027e55HPXJs/9+DP3MW5CFcXluX2VLiJLjDG5z0+W57LV2R988AGTJ09O21ZdXc1pp53G+++/35vFy3u9/X+37bvru3ok89sok34hed0Yku9ziXrRAB999BESs/1WWpd4zMU1fott3MX1uxGkvUe63iUTuJPkewWJEOv3wwVItOKKf4wfEP3GWz8A+3WP63ph2HjJ0PvmTpLBORFGE1LnTHtWSDRypJ6eloE20TJMql3EP2dqO8b/LQYx/jegYpIxNdEYkTgukWiNcQPlcdP/Hql/8N+IEo8mvXzZLieetLQELoFfgedHIB5z2bF9JxdcdQEd1VqdrS3QPSxbK28w8GZr5fX67saJRqPE44muDemtvJYIrkn7r+tpr5U3HMLqRCtvW5qbmqmtradujxd2a/fUUV8fCL91DdTXNlBfnwi/DdTXN9BQ30B9QyPNTc1tnl8EIgUFXgD2w29ZWSl7jRhOJBKhIBImHA4TCYUIh0KEnBBh28GxbEKWjS2W92k/Gvc+VDTHiDVHiTfHOPuyswmH0lsOLSOcdeZc/vuG+7DbLpn/04XZIBOnANq5sx6V7C6SeINJXs7cHnjj8X8H35iS1xJvUH6F3urxaffR3vbs99Hm9haPL1mCtMeZdluivFmeG7c2hm2n/72Ly4spKivK9alWOWpujqb65XY4l7V+QE/kP9PWgT2QKVu7P2n5jhC4rfWitP1O0Mo5c3lcpp3nppXzBANpi9syE3xabk4/zrS8ki7xLRoQa46ydtnawLdQiW/gEh+mDSIWaaE0eV4/yHnNtskP3IkcCd7gdGyS3/iR/JX+LV/a9kAQTHu7Ni0uZITnzCBN58KsIfW4kPTHivfKCIbZREIxqSc3a5hNvy5pv7snlWQh3T9rswbobrT24/Vs37qjRSuv1+80VQ0FP1u1aOW1Ur9FBNu2CIUcpKigWyaDT4xMdWPeV07xeJzmpmige0M9dbUNqSCcbO1tSP401Dd6Pw2NNDY2Eo1mH3ySYFlCQUEBhYUFFBQUUFAYYUhlOZGRI4iEw0RCYcIhxwu+joNjOX7wtXGwEJdk+I1HY8kQHG+MEa+NA3GggSgNtNbDybItnLBDKGwTjoSIFDgUDi3inVeW8KkLj2fBK7+ldmctABVDy1n1jw857DOT/BZLkl/zQ+rTdaJyTOsCYUmyfkh2eUjuk35sap+W21q2CKRaSr19SN4ntHFM5n0HypN2f2mPJfU7uW+2+yNwW5bbsx2TbJkg/TEE33BaPBdkPIbE48ry/ASP6S7B8F02YX82b69Nu725oZnmhkYiRdoPuivGjx+f1vqcGHCU6PaTG8n43foeQRdccD7VH1enbfvRj/4fc06Yk9M7umTZqa3DsuSfwIaWUTczM2a7v/eXvs+///vladtqatZTVTUmbVs4HOGvr/w17T4TWemiiy/k448/TivND//zRj7z6ROylb7npDWwpj/WFo/dCl7MfF4kp78feO8RFUPLO1DAzIudDbOGZJg1qYya+jDQTstsWpJPhdn011E/C7MDhAbobrRh7UYiBeFua+VNTcPidRuIubHk1+3JqVnihqbGRvbsqae+to7a2lQQbqhvTOvqUF/vBd7GxiY//Hq/4/F4m+WwbSsZgBMtvsOGVBIJhwmHQoSdEKFA8HXEwsLCxsI2AnGTFoDj0TjE8H7qwPtPHsUQpQlo8u/Xsi3skIMTTv2EiyIUVRRjOzZ2yMZybKyQN5BNHO/Dh+PYFBWHKCsNU1FeSGVFEWWlBTh+y6HrGhqjLk1Rl1EHTqapIcr93/4N2zZsA+BTnz2aY887lurRlRnvXCb122T8TtxisjYTeB/e3YxtiQCYdo5ARZeoCE2wwkydO/VZP+M2CVadmW8+iX+yHOe/i7pi/K8Jg8U1yV0k+V1jaluyRdekvo5M1f9u6gwmUTcnKnu/lIk3k8ynLzNLCNneFrI8Pv96xoeGxLZU+A5+AMj44EDqQ0DiuELXwm2O8fgz91FcXkxzQzONtY3eoEJthO52icaFdKblpayZru0QGvTYY/PTXvCJi27cTd/W4rSmZT4y7ezfQybtdwCvLny1zfCZ4D2u9JsF+N1vf0falqyH53OsyvL8B6p1N9pMIsym//06GWbJ3JfU/hpm854G6G7S3NRMNBajyC4CY4hFY6m+T8k+p16YiMcC07Q0x2hoaEz1861vpLGugbr6BhoaG2msb6ShsYnGhkYam7zfwfDb2NiYHB3bGsdxvL6/kQiRSJhIOExxZaUXfG3H+7FsbD/82lhYRrBcwTbgRl2vxTrIQFra9UMwVhzCDlbYwQ5b2GEbp8DB8YOwnQjDget2yMZ2LMSxEdvCcrwuJ173MDfZ4gj+gETX9YOyQ9gRSorClBSEKC5wKAjbhJ1UEDIG4ljEcYi6FnEjuAiEbJwCi+YofPCPj3DCDv/xq2v55ytvs/rdNRx9zjFUDGt7uhtjEm+eid/BVgR/W8Y+wf1St3kF9boemNQ3a4GuA8mq2pDsiuZ9W5jqihCs8dNaQBIHJ04AiTEWaefHEq8xJ9Ef2EoEmIzWXwgMQPQHDlqpEdTJWTUS+yXOLRZiJwYimoxWc3/wYiJg+wP+Un/74P1L2mtCrEC5JDUPd1LaO0+Wt6F2woZ33/5zYNtUFkBl+RDMzs2YoiKsoXtRVBxp9VjVOfFojGhzzPsWiGCMzQwuCT0RTrO19abdnMMZApe6+duRdu6wEzIaDDr6lPbc54NeYDCJxiQNsyoHGqC7wY5tO6mtrWP4iKEkBhz94aGnqa9roLExFYAbGhtpbGhq0Qrc3iCQUCjUop9vUUk5oYqhXouvZeOI7bX4Isnwa7ngWA52a10/XO9H4uKFWifkBd5wCCfk/W71uv87rYXYbxFOjBr2RhunRhOn9U8Lfmi3hFDIIRQKeb/Dib7MNrZtIWKwEWzLEBIIOeCIwRYXCzdZtxmEuLFoMhaxuBCNQzxuMEZSWdISf5CGAXFp+vgT9j14X6YdM41wxOHY847jyLOOZtvGbUSTU95kf9NOn/kCxLJAvDmfEcGyE33hEqEycRup2R3EC334c0VjgYX/OzATR6rRItCaKunXU10iWt7W3vXOfFPS8nXb2huuaeWiabGtrf1TreCZx2V8YEgVMGu50u7XBG9zA/sEbwv04nRdioYNo/bjVbjRZizXpWivkYgTynwQqouM63ofoC07+YVM6v9fttdrP4o1WYvSWj3fj8rd1bL0p4fSCdKh7kJqsNMA3Q1qa+s49ZgLk9cfmH8nf3h8QfJ6OBRKht9EP9+KghKGF5V7rb6kWn1tY+HYNqFEdwjb9oJVBrEkGVztQItuq9cDLb92IAAnQm9rWkyp419OjioGongtRTR7ocx2vGmaIpEQTiiUCsThELY/Z7CdGNzoh1kTd4nHYph43JvfMR7HERfbihOyDSEbbCvVEBB3IRoTYq5FNIYXll2wLLBssB0HJxIiEg5jhbzHnZjqzA7ZafM6B2cTCRc6RIoKKKks6fbAOdC0fA5a+8q3leO7tTQ9LzhwsmziJBILbYgT0tdDT9H/a0qpfkoDdBc11DYQbUwfulYUKeDe+37Kk7f+EceyU1//BkNvRt/eFtezdHUIXm8r9LYmMQVaYrL0mBunuSFKPNHS0+KNymBZNqFQiHA4hFNgE7K9soYcxxvs6M9DjKQW4ghOZZNYNMTvs4CJusSbo8RcF1zjL8JhiISEsAVOyGDjtSwnWTY4YSQURkIRrFCYsONQ6N+HZVleOC6IYNmON/9xNwy4VCoobUCjpXM+q/5n2kHTKCkp8RoKHJv/+9//Y/uO7Vx62aWsXbeWcWPH8dBvHqKyohKA2+64jd8++lts2+anN/+UEz5zQjv3oJRK0ADdRYUlhUSK0vs/llSUsGHlBo65+Pi0AGzZ3bt8cmarcGLUenA6r0RfW+MaEON16QjZOLZDQSiMU2QTckI4ju0vOuG3DgdaiRNdF0TAsm1vPmjb29eyvVX/xPL6LnvzRXt9aY1xvQFnxvW6KYjBMi4mFsXEorjRJtymJkw8NYuHFQpjRYqwIwXYkQKsSASxbDBeq3eildOyHcR2vH7TgQ8pSinV38XrdmN2bYV4DGwHKR+GXVzWLed+dsGzDB2aGr9xx513cOyxx/K1r3yN2++8nTvuvIObbryJDz/8kKeeeop/vP4PNm7ayJlnn8k/F/0T2+7D+TSVyiMaoLuBbVk8MP9OQv7cw9H6Zqr2q2LT+k9yPkcqAMdTy1r7y20mFtkAkr8FsB0Lx/Fm/XBCNoXhCKGQg+M42LaF47cUOxGvBTkUDnkB2ba82SsSITjRncLO3q0huB0SwdyfCcT1lgA18TjG9Qdg+IPmTCyO29xMvNkLym5TY2ofwApHcIqKsfywbPthOTFQMHU/cSzbwQp7LczksDy3Ukr1R/G63Zgdm1N99OMxzI7NxKHbQnTQ8395nucWPAfAhRdcyKlnnMpNN97Ec395jnPOOYdIJML4vcezz4R9WLJkSXJlP6VU2zRAd1FDbQMvPf4KP7/rv/j+f36Vgmab6vfWcO4N57Fi6Sri8XhqUF1gtozkkDTx6lHLEsLhkDdNXFEBBZEwEX/e5FDEH1znB+FwOOz16XVaLpaS+LG6IWQav/XYuAYTjxGLu5h4zJuWLG2OH8GNRb2Q3NxIvKmReFMTJOe5FKxIBKekNNWyHI4ku1kYPyQbv281loXlOF5otm0NzEqpvBHf8Qkm2tT6Dk2NtBhQaAxm+2ZidbuyHiKhCHbliPbvXOCsz56FiHDpvEu59JJL2fLJFkaOHAnAyJEj2bJlCwAbN25k1szU0uijR49mw8YN7d+HUgrQAN1lhSWFxP0eHJ9Ub+aok49g5imzWL2imooRFRQURohEIkQKwkQKwoT88BsOh7yFPRwn2TLcVyExLSi7ca81OR73g3JqBgoRy+tHHW3GbW7yg3IjbnNTqjVFBDtSQKisPD0sBx6bN7Wfi4nF8aYhE6/rhuP4M1loYFZKDVStzcbR9TngXvrLS4waNYotW7Zw1jlnsf/++7deiiyzP2m9q1TuNEB3g3f/uQxbvJknJh98AOFImFHjRvZ1sVpIBWU31e0iGJT95nCRxHQX4oXlpsb0sJxgWdiRAsLllX5/5QKsUDg9LPvdMNzkYi0GsexAYLZ1wJ9SasBor6U4tmG11/e5xYEOzoixXbrvUaNGATB8+HBOO/U0lixZwvARw9m0aRMjR45k06ZNDB8+HPBanGvW1ySP3bBhA6NGjurS/Ss1mGhy6QbVqz6mJFJI5YgKIgWRPv8Ub4w3IbwbixJvaiLWUEe0djfRPbuJ1u4hVl9HvLERE4t5Y/IsGxELt6mJ6J7dNGzZRN26NdSu+Yj69R/TuHUzsfpaxHEIVw6lcOQYSvbel9IJ+1M8Zm8Khu1FqLQcO+w1xRvXu283FvUmpvdbpZ2iYkKlZYRKSnEKCrGckIZnpdSgIuXDWs54JOJt74K6ujr27NmTvPy/f/1fDpx8ICfPPZnfz/89AL+f/3tOOfkUAE6ZewpPPfUUTU1NVH9czarVqzj00EO7VAalBhNtge4GGzdtYUikhBHjcuij1o1Sg+28wYYmHvP7EgeXpvPnUbUsLMertN1YLK1VOd7UiImlpuITx/G6YZSWJQf4ie1k/WCQLEPg68fEgD+xdYYMpZQKsovLiEO3z8LxyZZPuPgLFwMQi8U499xzOeGEEzjkkEOYd9k8fvvob6mqquLh3zwMwOTJkznrrLOYfcRsHMfhtltv0xk4lOoADdBdtGP7TuobGthn6CiGDKvskfswxpsz2QvK8eSMF8Z1U6MQ/b7ESCooG2MwsRixxgav+0UiLAe+PpRQCLugADtSkeqGYbf+skibIcNfdzk14E9nyFBKqfbYxWXQzTNuTBg/gb+/+vcW24cMGcKfn/5z1mOu//r1XP/167u1HEoNFhqgu+jDZSsBqKwop6yyaxWit3StNz2c68Yg7nqr8iXmP4bAynhW2mIqxhhMNEq8qS7VX7nFtHFhnKKi1LRx4QKknRYHk5h/OTGY0PICs+2EEFsH/CmllFJq8NEA3UUfLfUC9LBhQykqLszpmMQsFLgGN9HtIh5PzWSRXDLa8n4yVh00xnjzKwdalePNjX7Q9njTxpX4s2D43TBy6G+cLJu/eqAO+FNKKaWUSqcBuouWvfcRtlgMGV5BcXFRcntyEZCsQRkS/ZOTQbmVvsLGuMQbG4n708a5jX5YDk4bFy5IDuJLrt4nOYTlwIIoyf7SloUVDmPZIV0SWymllFIqCw3QXbRy+RpKIoWUVJbihC1ijQ0Zq/KZ5IjrtoIyeK2/3qp9DYFuGIFp48SfNq6sMtkNwwqHc+5CEZzGLhnebRs75HXlENvKKXgrpZRSSg1mGqC7aEPNJirDxQwbNQSJRXFd12+5bXv2CROPE29uTOuG4TY3J28Xy8aKFBCuKEkuSCKhUIf6G6cP+MPrwqwzZCillFJKdYkG6C7YtnU7dfX1jBsxnGNOOxLLcUAkMOjO48azTBsXDUwbZ3vTxjnFZamw7GSfNq4tLWfI8AYaWnZIl8RWSimllOom+n19F3y4dAUAo0aOQLZtYNdHS9mz+iOM69K8Zxf1G9exp3oFtWtWUL9hHU3bthBvbMQOFxAZMpyiUWMpGb8fpRP2o2j0WAqGDidUUoqVY0uzcV0vnMeiuPEYGBdxQjiFRYRKSr0FSwqLscNhr8VZw7NSSg1IX77my+y7/74cfuThyW3bd2znzLPP5OCZB3Pm2WeyY+eO5G233XEbMw6dwaGzD+XlV15Obn/7nbc54qgjmHHoDG741g1Zl/xWSmmA7pKPPlwNwN5jhuFGve4XbrSZurWrcQoKcZubsQuKiAwdQdHocZRO2J/S8RMpGlVFZMgwnOISr9U6ByYxvV0s5v94LdhWKOyt8FdcSqikDKew0Avg2j1DKaX6per3q/nvb9xH7c7atMtdceGFF/LkH59M23bHnXdw7LHH8vbitzn22GO54847APjwww956qmn+Mfr/+DJPz7J16//OvG4N27na9/4GnfdcRdvL36bVatW8fLLL7e4L6VUDwdoEZkrIstFZKWIfCvL7eUi8mcR+ZeILBWRS3uyPN1t5YercSybqjFD07a70WbsSCEle+9L0cgxRCqH4hQVtzvnclB6YE4sie1PT1dUTKhEl8RWSql8U/1+NX/46eNsXb+VZ37xdPLya0++1qXzHnXkUVRWpi/m9fxfnufCCy4E4MILLuS5558D4Lm/PMc555xDJBJh/N7j2WfCPixZsoRNmzaxZ88eZs+ejYjw+Qs+z7PPP9ulcik1UPVYH2gRsYF7gDlADbBIRBYYY5YFdvsysMwYc7qIDAeWi8jvjDHNWU7Z76z8cA0l4UKG7pVeaVmhcHLdk1wlZ8gwJjnDnTfgL6wD/pRSKk/8z0Mvsfnjza3evnHVRmLN3mqwa5etTXaReOeVt9lasyXrMXvtvRdzLjmxw2XZ8skWRo4cCcDIkSPZssU7/8aNG5k1c1Zyv9GjR7Nh4wackMPo0aPTtm/cuLHD96vUYNCTgwhnAyuNMasBRGQ+cCYQDNAGKBUvGZYA24FY5on6q3Vr11MRKWb4pH2xQjZutBkrFKZk7339qeJal3VJbNtJLoutA/6UUmrgGTFuBFvXb6W5oTkZnkWEIaOG9FoZsvVrFpFWtyulWurJAD0GWBe4XgMclrHPL4EFwAagFDjfGNMieYrIFcAVAOPGjeuRwnbUJ5u3UVtbT9Xw4ezYuYcDZkz3VuuznRazcECiS0ZitcHUktiW7egMGUopNUC011Kc6MIRDKuWbTFu8jjmXn5yt5Zl+IjhbNq0iZEjR7Jp0yaGDx8OeC3LNetrkvtt2LCBUSNHMWb0GDZs2JC2PdGCrZRK15MdZ7OlwcyPtycB7wCjgRnAL0WkrMVBxtxnjJlpjJmZqAD62vJl3gwcZZEiisqLqa1eScPWT5JLcidnyIh7g/4wbmrAX0kJoeJSnMIinSFDKaUGkf956CXiMW/AnhN2sB2beCzOh29+2O33dfLck/n9/N8D8Pv5v+eUk08B4JS5p/DUU0/R1NRE9cfVrFq9ikMPPZSRI0dSUlLCokWLMMbw2PzHOPWUU7u9XEoNBD3ZAl0DjA1cr8JraQ66FLjFeB/FV4rIGmAS8FYPlqtbfPTBKgAqK8qpqCgB9mBZtheWMd5CKOEwlu14/Zd1kJ9SSg16n//ehbz25Gt8+MYHnHT5XD5+v5oP3/yQs75ydpfOe9nll/Ha319j27ZtTJ4ymW9/69t87StfY95l8/jto7+lqqqKh3/zMACTJ0/mrLPOYvYRs3Ech9tuvQ3bH+R++89v5+ovX01DYwNzTpjDnBPmdPkxKzUQ9WSAXgTsJyITgPXABcCFGfusBT4DvCoiewEHAKt7sEzdZuVHawjZDmXlJZSUFELdHsSxcYqK/BZlDcxKKaXSlVSUMPff5jL33+YCMPnwyd3SdePB+x/Muv3PT/856/brv34913/9+hbbDzn4EN58/c0ul0epga7HArQxJiYi1wAvAjbwoDFmqYhc6d9+L/Aj4CEReQ+vy8c3jTFbe6pM3Wn1io8pCRdSUFpIYUGIeJ23oqDYHV9BUCmllFJK5Y8eXcrbGPM88HzGtnsDlzcAHZ+bpx9YV72einARJUNKsMQlDp1aflsppfoLEZkL3IXX6HG/MeaWjNvLgUeBcXjvHz83xvym1wuqlFJ9TPsZdMInG7dQW1tHabiIihEViPEGhFhOqI9LppRSnROYu/9k4EDg8yJyYMZuibn7DwKOA24TkXCvFlQppfoBDdCd8IE/A0dJpJBhI4eBGwfL8uZvVkqp/JScu99fzCoxd39QXs/dr5RS3UUDdCes+MAb51gaKWLomGGYWCw5l7NSSuWpbHP3j8nY55fAZLwZld4D/qO1uftFZLGILE6sfqeUUgOJJr5OWLF8FWEnRGlZCeWVZbixKFg684ZSKq8N6Ln7lVKqO2ni64TVK9dSVlRMpDhCSXkxbjTqTV2nLdBKqfyV69z9TxnPSiAxd7/qY1++5svsu/++HH7k4clt3/vB95h52EyOPPpILvrCRezctTN522133MaMQ2dw6OxDefmVl5Pb337nbY446ghmHDqDG751Q9blvZVSGqA7zBjD+nUbKbIjFJYVU1BUgInHvP7POgOHUip/Jefu9wcGXgAsyNgnMXc/+TZ3/0B34YUX8uQfn0zbdvxxx/Pm39/k9ddeZ9999+X2O24H4MMPP+Spp57iH6//gyf/+CRfv/7rxOPeYPivfeNr3HXHXby9+G1WrVrFyy+/3OK+lFI9PI3dQLRpw2Zq99QxdvgwSoeV4tgWcWMQ287+BahSSuWBgT53f39yxme+wPZtO5PXhwytYMErv+3SOY868ig+Xvtx2rbPfPozycuzZs7imQXPAPDcX57jnHPOIRKJMH7v8ewzYR+WLFnCuHHj2LNnD7Nnzwbg8xd8nmeff5Y5c3Q1QqUyaYDuoOXLvCW8SyKFDBlRiS3e11s6B7RSKt8N5Ln7e8tdt/43K5a33SgfDM+J69f827db3X+/A/bhP2749y6V69HfPco5Z58DwMaNG5k1c1byttGjR7Nh4wackMPo0aPTtm/cuLFL96vUQKVdODpo+YdegC6NFDF01FBEvAHool04lFJK9UM/u+1nOI7DeZ87DyBrv2YRaXW7UqolbYHuoJUfriYSDhNxQgzZawgm7k2B6s0BrRWNUkoNZrm0FB894/QW2375wM09URx+/9jvefHFF1nw9IJkGB49ejQ162uS+2zYsIFRI0cxZvQYNmzYkLZ95MiRPVIupfKdtkB3UPXqtVSUlBIpLqB8SCkmFgXACof0k7pSSql2DRla0eb17vLyyy9z5113Mv/38ykqKkpuP2XuKTz11FM0NTVR/XE1q1av4tBDD2XkyJGUlJSwaNEijDE8Nv8xTj3l1B4pm1L5TlugOyAej1OzdiMjS4YQKSmgsLQIN9rodd3QVQiVUkrloKsDBrO57PLLeO3vr7Ft2zYmT5nMt7/1bW6/83aam5o565yzAJg5cyZ33n4nkydP5qyzzmL2EbNxHIfbbr0N27YBuP3nt3P1l6+mobGBOSfMYc4JOoBQqWw09XXAxvWbqautp6B0JMXlRYQLwrgNtYhlY1l2XxdPKaXUIPXg/Q+22PbFL3yx1f2v//r1XP/161tsP+TgQ3jz9Te7tWxKDUQaoDtg+QcrASi2I5QPryAcCRPfHfWnsNPuG0oppZRS/ck+UycSN+DG4946HkWFVHZDtykN0B2w4kNvaqKSSCFDRlbihByiMV2FUCmllFKqp7iuIRqN+j8xmpujNDd7173fscD1WGB7lH+bMIaLz7oqea6/vDZfA3RvW/HhGgoKIkTsEJXDK7FDNm4sihMOa4BWSiml1KDiuoZYLBVes4fbqL8tlgzBmSE3dUzm9ljyvJ0RCjl84arPd/Oj9miA7oDq1WsZWl6BiFAxohLLEnBdncJOKaU6yBijMxepfsEYQ8sZsPObMSYjjMbaaLGNtgjAmfu1dXtnhEIOoVCIcDjxO5T8XVJSFLg91GK/4L7B/RLXE7eHQt4Cd5UV5d387Ho0QOcoFouxft1Gxg0fRbg4QllFaXIOaLF1FUKllMpVQUEB27ZtY+jQoVp3qj5ljGHX7l3EmzoXBDtzf7FYvJUW2swW2OxdFTJDb3prbupyZ2QG22BQLS4ubDW8Jo5pO9x6vx3H8Roge4kl8NRLD+HG4xSXFlNUVNgt59UAnaMNNZuor2ugaGSYwrJCIoUR3Kj3AhVHVyFUSqlcVVVVUVNTw5YtW1rdp7G+Edd1e/WNVuUXJxRKazkWIBbtWHA0QLwpxu7Nu5JdEVpvoc3W7SB9W+tdGFLdF7Is+Nj+Y3XsVsNpYWGhH1KddsNrsIW2ZYttaED+f1v9/kpiMcOObdu54KoLuu28GqBztHyZNwNHxHUoqSwlXBROLqKiy3grpVTuQqEQEyZMaHOfhQv+yo5tOygrL+2lUqn+yhhDPB6nqSlKc3Mzzc1RmpqaOWLOkZx3yuXJ/R7783/x5ENPZ2mhzWy5bRmEOxtsbdtutftAYWFJlhAbbNnNDLPpXRGCIdfScVb9jgboHCVm4CiywlQMLydcECYerQe8Fmj9GlIppdRgluh329TUTHNzczLwetejadu8EJy43HLfzN+u67a4v6mHTU+7Xl/XwIIFLwNg21arLbClpcVpgTczxLbc3rKfbSIE27YG28FKA3SOVn60hqKiQsJ2iMoR3hR2ptFfxtsJ9XHplFJKqdy4rpsWTlsG2+Dv1sNt5v7Nzc0dasW1bZtIJEw4HEr7XVZWkrweDoeJRLzfwcuRSJjS0pK081VUlHHzzddrsFW9QgN0jqpXr2P40CHJGTickENT1JsD2rJ1FUKllFLdy+u2kL1FNjPcpi63H4Q7OnNCouU1FXTDydkSUiG3ZRAO/s62j93F985I2OFPLz2UvG4JFBREunROpXKlAToH0WiU9es2sf/ee4OByuEVyTmgxbJB+yYppdSglJhVofVA21YAzh6EE9vj8ZbdFlojQpZW2hAFBRHKy0uzhNlQMginB9v0Vt7+PLBs9fsr+7oIKg8UFhcxbuLevP7K6wA01DZQWNL1mTg0QOdg/bpNNNQ3UBQqJOJEKCwpxLIsTGIVQtEArZRS/VliJbO2+tlm66KQSzcH04F+C5ZlZW2lLS0tIhyuaKW7QtutvOFwODnnrVIqpbC4iH0nT2TFkhVMPXQqO7fsZNnry5g5dybhSLhL59YAnYPEDBwFxqaovJhIofcVkRuLYkcKdBVCpZTqJju27aSurp4x+49jVLwK47r87YXX0lprs19uOwh3dF7c1ECy9LBaUVGWtbW27Vbd1P6Oo13+lAoyxuDGXOLxOG7MxY17P/FY3Lscc4nHXdzE9bhLPLGfvy14Pe4f48ZdTvvS6Sx/azmvPPoK537jXP77+vtwwg4HHXeQBuje8NEHqwAIxyxKh5YSLgx7KxfFYkihrVPYKaVUN6mvb+CUT6WW3n1g/p388pePtLp/ZpeESCScHIjW0e4Kie06CE0NRsY1aSHWC6XxVIBtJbC2HmDb2TceJx5zMW7X1oG0bAvLsbBsC9uxk9dt2+IvDzzPp849hnO/cS6/u+lRAL7yX1+luLy4y8+XBugcrPxoDSWlxdiuRfmwCiKFEUzMX4VQp7BTSqkeU1JSzJVXXpg2eC0Rdvtz/1zV8wqLi9h7v7356N3lRAoLkpezTXmXL4wxGNe02wobDKGt3tba9UDAjQdafLscZB0L27YDYdbC8q/bIZtwYci7Hgi4lh947WQItv3jArcFrtu2lRaYLdtqM4MVFhex9+S9eeQHDye3/fN/lnDCvDnaAt0b1q6pYa8RwwCoHFGBE3Jw/UVULF1ERSmlekw4HOKAA/bp62KofibYt3X8pAkUFRezYskK9ho7ko1rN3T5/Ikg27IVto0W2vYCbJYuBvFAt4RENwa6kmOFjBBrpwVOJxIiHAywiYAbCLCp21qG2OT5MgKvWNIvGxP33m9vViz+CCfscNVdV7P01fdZ9a9VHNt4nAbontbc1MyG9ZuYOvkAqCU5B7QbawASy3j3cSGVUmqAKCoq5C+vzaexoRE37qINzCrBjbvEo3Fi0Tj7TdmfjxZ/xMuPvMx5N5zHEz99Aifs8Lnrz+Mf772Zcx/ZtH63iX64/uWuEEuSQTOzW4Hl2IQKQq23tGYJtXagxTU9FKffZmnXozQfvbucvcaN5PPfu5C333ibE+bN4djG47QLR2+oWbuBhvpGSiOFUAtlQ8uwwzbR+mbAX8ZbE7RSSnWLyqEVVA6t0KW885wxhngs7gXe5jjxWIx4sxd+49FYMghnvdwc94+NJS/HmmNpXQze/MObnHjJSXz2a5/ltzd6feSv+NkV/PFnf2Bz9ebkfmJJqitBRncAy7aIhMNZQmzLVlg70IrbsitBsNuB/1snF+gXXNdl49oNrFu9nl3bdxGOhLvc8pygAbody/0BhAUSJlJiESmMYNs2TYkuHKFQv/zaQimllMqVG3eJ+QE2GWibY8nwmrY9GvN/Z16Op50jV5bt9ZG1HRsnbGOHHOyQTaQonLzshGz/t3e9YkQl+83cj0d/+NvkeZa+vpTPzPs06z9e73VjsL2uBUr1BA3Q7UhMYReKWZRUlhAp9D65mGgUxPK6cCillFK9xBjTIrC2CLTJ8Jst6LYMvR0ZQBYMsnbYxnFsIsWR9O3J0Oskw2+Ly2E72R2hoybNmMyqt1fihB2+fPc1vPPXd6h+bw2zT53Nls1bOnw+pTpK0187Vq+oprSsBNMQo3TvMsJFqTmgdREVpZRSbUkMRku04ubUhSERbP0QHG9OD7zxWMdbd9OCbdjxA68faMNOoPU3uG8w8HrbLaftWQ96S6Jv64Xfu5A1H61m35n7MPuUWazyvzVWqqdpgG6DMYaPq9czpmoksaYYFcPKk31n3KgXoHWEi1JKda+G2gYsy6K4rJhJMyb36tRkidbd9D65bbfmpoXeZBeG1D45t+4KGS24fleGkgKKQ3ZGd4ZUC24iFNuOF4aTobeTrbv5ING3NTHjRt3uum6ZfUOpXGmAbkNTYxMb129m1qyDYDNUjKgkFAkBeMt4OyFtgVZKqW6UWGp33KS9KSgoYOU/V7Y6NZkxJjkzQ8tW3IyuDclW3NYCr3e5I7MvWI7lt9ymgm0oEqKgpCAZcoP9d4MtumldIPzL7c1pq5TqPzRAt6Fm7UYaGxopLSwG6qkYXo4T8p4yNxbD0WW8lVKq2zTUNrDs70t58aGX+Nz1n+OP//0HnLDDOV85hyV/XpI2I0OiK0OurbsiktaFIdFlobCgANtx/CCbpZW3lf67dkhnWlBqMNMA3YblH3gDCIvsCPXUUzqkFCfkYOJxMC6Wo1PYKaVUdyksKWTWKbOpHFnJw/7KYVf8/Eu88JsXcOOGUEGIwlBBeqAN59DKq627SqlupgG6DYkp7MJxC7ekgJBfUbvNTYA3B7RWyEop1X3qdtXx7L3PJq8vfX0pnzr/U9q/VSnVr/To908iMldElovIShH5VpbbrxeRd/yf90UkLiJDerJMHbF6RTVl5aXE66OUDiklXBBBRHCj3hzQ3iqEGqCVUqo7JLpwJKYmO+rso6h+bw1DRwzt66IppVSaHmuBFhEbuAeYA9QAi0RkgTFmWWIfY8zPgJ/5+58OfNUYs72nytQRruuytno9VXuPpmFXA8PHDCdS6E9hF02sQmhrgFZKqW5SWFLIzJNncdDxM1j2z6WMmVbF7FNn69RkSql+pye7cMwGVhpjVgOIyHzgTGBZK/t/HnisB8vTIU1NzWxav5mjjzuM2Jpmbwq7In8Ku+QqhGHtwqGUUt0osdRu3Z46dmzbwY5PtvV1kZRSqoWe7MIxBlgXuF7jb2tBRIqAucCTrdx+hYgsFpHFW7b0zgpD66praGxsoqKsDICK4RWECxJzQPst0JbdK2VRSimllFL9R08G6GxNs63NN3Q68PfWum8YY+4zxsw0xswcPnx4txWwLR8u874yLHYKACgdWpqaws5fRMWyNUArpZRSSg02PRmga4CxgetVQGvDqC+gH3XfAPjIn8IubLyQXDYkNQe0iSVWIdQ5QJVSSimlBpueTICLgP1EZIKIhPFC8oLMnUSkHDgWeKYHy9Jhq1d+THlFKbG6ZgpKCnEi3hR24C2iIpajqxAqpZRSSg1CPTaI0BgTE5FrgBcBG3jQGLNURK70b7/X3/Vs4CVjTF1PlaWj4vE4NR9voGrcaOp31FE6pJRQOJRcdcrEolgFRboKoVJKKaXUINSjC6kYY54Hns/Ydm/G9YeAh3qyHB3V2NjExg2bOX7O0TSsbWCvsXslp7AzxsXE4zqFnVJKKaXUIKVNqFmsXVNDU2Mze+01jFhj1JvCrtCbgcNEYwBYugqhUkoppdSgpAE6iw+XeQMISwqLASgdUpZaRCWmqxAqpZRSSg1mGqCzWOGvehUxXg+XsuAUdhqglVJKKaUGNQ3QWaxZtZbyyjKitU0AlA4tC8wBnVrGW7twKKWUUkoNPhqgM8RiMWrWbmTsuNHU7ailoKSAUNjBDvtT2EVTy3grpZRSSqnBRwN0hob6RjZt2Jycwq5saDmWbWPbgQBtWd4sHEoppZRSatDRAJ1hbXUNTU3NVI0bRcOueiqGlxMpTLU2u7Fmv/uGPnVKKaWUUoORpsAMHy71ZuAYOrSSaGOUsmHlyRk4wJvGTiwbLO3/rJRSSik1GGmAzrDiw9UAFNkFAJRWlhAOBGg3FvXngNanTimllFJqMNIUmGHNqo+pHFJO4556wJsDOhQJAWCMwcRjXhcOXcZbKaWUUmpQ0hQYEG2Osn7dRqrGjaZ2ay0AZYEp7Ew8DsboMt5KKaWUUoOYBuiA+voGNm34xAvQ22spKC4gFAllXURF54BWSimllBqcNEAHrKteT3Nz1JvCbmcd5cPLEQE75E1ZZ/w5oMV2AA3QSimllFKDkQbogA+WrgBg5JgRNO1poHxYOeGCSLK1Oe6vQmjpMt5KKaWUUoOW09cF6E9WfLgKgMqycprrmykbVpYxhZ3fAh0KaRcOpdSAIyJzgbsAG7jfGHNLxu3XAxf5Vx1gMjDcGLO9VwuqlFI5KqssZ9/JE3BCNm60GXG6J8NpC7TPGEP16nUMGVpB425vBo6SylLCRemLqCCCiK5CqJQaWMSr2O4BTgYOBD4vIgcG9zHG/MwYM8MYMwP4NvB/Gp6VUv1VWWU5Uw/eH2vHRmLrV7N75YfEGxswxnT53Bqgfd4MHJuoGjea3Vt3A1BaWUq4IBCgo1HEtrF0GW+l1MAzG1hpjFltjGkG5gNntrH/54HHeqVkSimVAxGwLcGxhbBjsd+UfWjauBbX74LrRpuprV6J8SeF6ArtwuGrr29g08ZPmDpjErXbElPYlSZn4AAwMS9Ao3NAK6UGnjHAusD1GuCwbDuKSBEwF7imF8qllMpjlgWWiP8DYqUuW5Yg4t9u+bcHbmv3GBEsK3VMZteM4qIIezY1p21zo83d0gKtAdpXvXod0eYoY8eNpu7jPRQUFxAuiKQFaDemi6gopQasbJ0CW3uXOR34e2vdN0TkCuAKgHHjxnVP6ZRS3SIVQv1A6odQS/xtycCbEVxF/OtZjmkjDHeGMQbXgOsaXP+y8S/HXYPrGozBu831b884xhjYd1QTViicbIEGsELhbukDrQHa96E/A8eoMXvx0XtbKB9eDgK2k+qu4caiOOGILuOtlBqIaoCxgetVwIZW9r2ANrpvGGPuA+4DmDlzZtebepTKoqyynIkHTiAcCdPc1MzKZWvYvWNXXxerw1KtqalWV0lezhZcJbdj0sKwIP7lzkiGU2P8IJsIr95tUdfgRkkG1xbBNssxbiDwZh7TXWLvr2bqwfsnu3FYoTAl4yciTqjL59YA7VuxfA0AI0YO593dDew1eQSOY2PZXlg2rguuq6sQKqUGqkXAfiIyAViPF5IvzNxJRMqBY4GLe7d4SqUkBoc1bVxLkx+Mph68P++//VGXQ7QkA2h6S2swhGZ2HUgLsW0dk2zVTYXhzgq2znohNHU56hpc46aHXWMwLtmP8YOtF4CDl/39uvSM9p3dO3bx/tsfJWfhKCkr6bZZODRA430S+njNOoYMq8SNuzTVNVE2rJxwYAq7xCqElq2rECqlBh5jTExErgFexJvG7kFjzFIRudK//V5/17OBl4wxdX1UVDUIZQbT/bMMDmvauJZJ0/al+t2lLVpfu9KPNlfBrgPBQOoaQzxuiBo3owU21eqaeUzrXRhSwVblZveOXSz629vs2LadC666oNvOqwEaaGpqZmPNJsaOG82uzTsBKKksSZsD2o2mlvHWFmil1EBkjHkeeD5j270Z1x8CHuq9Uqn+Llt3gmR4tTK3p3dLsIPBNqN1t62+tEWtDA4rLggzZlgRQFpLalf60aa19Ka14qaHYTW4aIAG6uvq2bThE6YfPCU5A0dpRSmRomCA9v6jiq0BWimlVH7oULgNdkVoEXxzD7ftadG6GgiksXiq60E8EHjT9zdMbGVwWF1dI/9atUMDrepxGqCBNavWEo3GqBo3irrtewAoHVqKEw7OwOG3QNu2duFQSinVZQJZWmhbD7dei23/CLeZ3QuC4ba144P9artqeZbBYZFR43j/7Y80PKte0W6AFpG9gJ8Ao40xJ/srUx1hjHmgx0vXSz5435uBY3TVSNa+voqC4gIiRelT2Bn/U64V6vrITaWU6in5XmcbYzCxKLOOOYSGugbWfLS2T2ZWyCXcigi2lQq32bss9E64jccNzZmDxlqE2d4Jt70hMThs4oETiPizcHTHAEKlcpVLC/RDwG+A7/rXPwIeB/KiMs7F6hXVAIwYOYyPdr9P2bByMCZ9DuhoFLFsbxYOpZTqvx4iT+tsYwzxxgZqq1fiRptxWplZIddwa7XWxzZLl4Wuhlsg2a822N82HhxA1ma4DW7Pflu+hNvesnvHLv7593f6uhhqkMolQA8zxvxBRL4NyZHa8R4uV69xXZe11TUMHVaJ4zg07Kpn+KRxWLbdYg5or/uGzgGtlOrX8rbONrFoMjxDamaFaQdPZNfqlV0Pt8HBYm6q5TbncNvGbRpulRpccgnQdSIyFH9FKhE5HBgw35E0NjaxoWYzVXuPpn5PA421jZQNLSNSGE7bz/irENKFORuVUqoX5G2dbYxJGxQGXogWy2FXXTTnbggabpVSPS2XAP01YAGwr4j8HRgOnNujpepF9XX1bN74CQcdOoXabd4AwpIhpWlT2IHXhcMuKNAWaKVUf5e3dbaIZJ1Zob6hiXVb6vuwZEopla7NAC0iNt6KU8cCB+B1PVtujIn2Qtl6xeoVHxONxhg7bnQqQFeWpC2iYozBxGOI7SCWBmilVP+U73W2OCFKxk9MduMIzqyglFL9SZtp0BgTB840xsSMMUuNMe/nS0Wcqw+XrQRg9NiR1O3w5oAuG1JGKJKabcPEYoDOAa2U6t/yvc4WEeyCQsomTiIybj9iZSN0ZgWlVL+USxeOv4vIL/FGcSeXbjXG/LPHStWLVn1UDcBeI4dT88ZqIkURIoXpU9illvHWOaCVUv1eXtfZIoKEwiz629/ZsW0HZeWlfV0kpZRqIZcAfaT/+6bANgN8uvuL07tisRjrPl7PsOFDsG2bhl0NlA8rB2gxhR34y3ijAVop1a8N2DpbKaX6i3YDtDHm+N4oSF9oamxm04bNVI0bTXNzlPpd9VTtX4UI2KHAFHa6jLdSKk8M5DpbKaX6i3ZHxIlIuYjcLiKL/Z/bRKS8NwrX0+rrG9i0YQtVe48m2hilYXc95UPLCBdE0rpqJFugQyHtwqGU6tcGcp2tlFL9RS5TSjwI7AHO8392461ylfdWfbSGWCxG1bjR1G7bDUDpkLIWU9iZWBTEwrJ0FUKlVL83YOtspZTqL3LpA72vMeazges/FJF3eqg8verDpd4MHFVjR1G3wQvQJRUlhIvSF1Fxo83eKoQ6hZ1Sqv8bsHW2Ukr1F7kkwgYROTpxRUSOAhpyObmIzBWR5SKyUkS+1co+x4nIOyKyVET+L7did49VK6oB2GvUMBr3NAJQMqSYcCQjQPvLeKMBWinV/3W6zlZKKZWbXFqgrwIeDvSh2wFc0t5B/oT+9wBzgBpgkYgsMMYsC+xTAfwKmGuMWSsiIzpW/M6LRmOsX7uB4XsNxbJsGnfVEymMUFBUmDaAEPxlvEMhbYFWSuWDTtXZSimlcpfLLBzvAAeJSJl/fXeO554NrDTGrAYQkfnAmcCywD4XAk8ZY9b65/4k96J3TVNjExvWb6Zq7Cii0Sj1O+spG95yCjtjDG4silNQqMt4K6X6vS7U2UoppXKUyywcPxGRCmPMbmPMbhGpFJH/l8O5xwDrAtdr/G1B+wOVIrJQRJaIyBdzL3rX1NfW88mmLVSNG40bN9Tu2EPF8HLApAVoXBeM8bpw6AwcSql+rgt1tlJKqRzl0qR6sjFmZ+KKMWYHcEoOx2VLmybjugMcCpwKnAR8X0T2b3EikSsSUzJt2bIlh7tu34rlq4nF4lTtPZrmpiYadjdQNqwcJ+Rg2amnJbUKoaNT2Cml8kFn62yllFI5yiVA2yKSnNdNRAqBSBv7J9QAYwPXq4ANWfZ5wRhTZ4zZCvwNOCjzRMaY+4wxM40xM4cPH57DXbdv+Qf+DBzjRlO3vQ5jDKVDSglnTGGXnANaF1FRSuWHztbZSimlcpRLgH4UeEVE/k1ELgP+B3g4h+MWAfuJyAQRCQMXAAsy9nkG+JSIOCJSBBwGfJB78TvHGMPqFR8DsNfI4TTu9gaol1SUtJgDOrkKoaMBWimVFzpbZyullMpRLoMIbxWRd4ET8Lpl/MgY82IOx8VE5BrgRcAGHjTGLBWRK/3b7zXGfCAiLwDvAi5wvzHm/S48npxEm6Osr9nEiL2GYdkWTbu8AF1aWUqkKHsLtHbhUErlg87W2UoppXLXboAWkWLgJWPMCyJyAHCAiISMMdH2jjXGPA88n7Ht3ozrPwN+1rFid01jYxMb12+matxoos1RGnbVEy4IEymK4ITTn5K0FmillOrnulJnK6WUyk0uXTj+BhSIyBjgZeBS4KGeLFRPq6utZ/PGLYwZN4p43KVuey3lw8sRS9Jn4CCxiIqDZesy3kqpvDDg6myllOpvcgnQYoypB84B7jbGnA0c2LPF6lnLP1hFPB5n7LjRGNdlz45aKkZUgDEtArRJrEKo3TeUUvlhwNXZSinV3+QUoEXkCOAi4Dl/W173Z1i5fDUAVXuPJtoUo35nHWXDyrFsG9tJb2lOLeOtAVoplRcGXJ2tlFL9TS4B+j+AbwN/8gcB7gP8tWeL1XOMMaxZ+TEiwsjRI2jc05Ccwi5SGG65fyyGWDaWpV04lFJ5YUDV2Uop1R/lMgvH3/D61CWurwau68lC9aSmpmY21Gxi+F5DsUSI1nqDBEsrS1tMYWdcFxOP6xR2Sqm8MdDqbKWU6o9yaYEeUJoam9iwfjNjx42muTlKw856AIorilsuohKLAd4UdhqglVJKKaUUDMIAvWdPLZ9s2krVuNHE43Hqd9QSioSIFEYIRUJp+5pYahVCnQNaKaWUUkpBDgFaRI7KZVu+WPHhauLxOFV7jwYDe7btpnx4BZa0nMIunpwD2sZbj0Appfq3gVZnK6VUf5RLC/TdOW7LCx8tWwVA1dhRuK5L7fZaKkaUQ5YAnWqB1mnslFJ5Y0DV2Uop1R+1OojQnwbpSGC4iHwtcFMZ3tLcecd1XapXr/Vm4BizF3t21FK7s5b9h+2PCNihjCns/BZoKxTWLhxKqX5tINbZSinVX7U1C0cYKPH3KQ1s3w2c25OF6ineEt6fMGLkMCwR4o1RjGsoG1qOEwm1CMluNAqWhcig6yqulMo/A67OVkqp/qrVAG2M+T/g/0TkIWPMxwDiJckSY8zu3ipgd2puambD+k1UjRtNNBqjeU8TAKWVJS2msIPUIipiaYBWSvVvA7HOVkqp/iqXZHiziJSJSDGwDFguItf3cLl6xO5dtXyy2ZuBo7kpSv2OOgCKK7IHaBONIpYNGqCVUvljwNTZSinVX+WSDA/0Wy/OAp4HxgFf6MlC9ZTlH6zEjbuMHTcaBPZs3e1NYVcUbqUFOoZlO9oCrZTKJwOmzlZKqf4ql2QYEpEQXmX8jDEmCpgeLVUPWfHhagCq9h6NCOzeupvyYeWIWC0GEBpjvGW8bVv7QCul8smAqbOVUqq/yiUZ/hdQDRQDfxORvfEGpeSVWCzG2jU1iCWMGj0CN26o3bGH8hEVAC2nsIvHAaPLeCul8s2AqLOVUqo/azdAG2N+YYwZY4w5xXg+Bo7vhbJ1q+amZjau38xeew1HLAvHsdmzfQ8Vw8sB0yJAu7oKoVIqDw2UOlsppfqzXFYi3EtEHhCRv/jXDwTm9XjJulljoxegq/YeRXNzM6YpjnENpUPLcEIOlp3+VJioF6AtbYFWSuWRgVJnK6VUf5ZLF46HgBeB0f71j4Cv9FB5eszOnbuSM3BEm6M07KwHvCnswlkGECaX8bY1QCul8spDDIA6Wyml+rO2FlJJGGaM+YOIfBvAGBMTkXgPl6vbrVi2Gtd1qRo3GmOgdsseAEoqSludwg7wBxFqgFZK5Y0BUWcrpRSA6xpvYgfjJi+7ruv/Dl7Hz2sGkLS2z3jcEA6HurVcuQToOhEZ6pcIETkc2NWtpegFKz7yZuAYO240Auzasgsn7BApihApyjaFXTOIeC3QSimVPwZEna2Uyk9eqG097CZug1TCFcEPwN7vIMsSbNv2f7wxbMHrtu3gOBa23+BpWRaW5f1OXAehuaGhWx9nLunwa8ACYF8R+TswHPhct5aiF6xfu8GbgaNqJLW7awNT2AlOuOXT4Ea9VQgt285yNqWU6rcGRJ2tlOp5XqDNbMkNbkuFXy/wpn6LCCaZdlO3WZbgOI4fbm3C4XDysuPYWJbth2ArLeQmQm/mtu7oBeC6hlhT905JnEuAXgocCxyA9wwtJ7e+0/3KhvWb2WvkcCwRQuEQu7ftZnjVMJCWU9gBGH8Zb3QOaKVUfhkQdbZSqqXWwm7Lll5ID7mJgJtJAuHWIhRysCwv3HqXLT8M24GAK4hYGaE3sa17Am8+yCVAv2GMOQSvUgZARP4JHNJjpeoBG2o2MXbv0TQ3N1MQjlC7Yw8TD94XTMsp7MBbhVBsW1chVErlmwFRZys1EGTrv5u9D6/BsoJhN9GdIT38ikig64KdEXK935Y/VW9rLbu2HWzh1YzTWa0GaBEZCYwBCkXkYFKdVcqAol4oW7dpbmpmy+ZtHPGpmTQ3xxABN+5SNrQcy7axnZbdNNxYFCccQazB8UlKKZXfBlKdrVRfyey20NqAtWD/3YSW/XdNMqQmui9khtxUt4b0gJuta0OqP6/qD9pqgT4JuASoAm4j9WrZDXynZ4vVvarXrPNm4Bg7GuO61O3wZuAoG1JGKNJyVKZxXXBdvwuHBmilVF4YMHW2UrlKBNrcBqy19jslMWAtEWqDfXkzB6y1FXKD19XA1GqANsY8DDwsIp81xjzZi2Xqdqs+qgagatwoDLDrE29AekllSdYp7FxdREUplWcGUp2tlOu6xOMu8XiceNzFdeO4bsv9ggPWvNDbcsCa182h9f673T1gTQ0O7faBzveKeMe2nUzYdxwPzL+TYSOGIiJ8/NoKnJBDpLC1Kex0GW+lVH7K9zpbDWyJUOwFZC8cG9NycJtlWUQiYQoKCgiHQ4TDYSKRUKArROq3Un1hwE9yXF/fwHmnXJ68/vQrj7Dzk52UDy8HMYQj4RbHuMFVCLP0c1JKKaWUxxiTNRin895LQyGHcDhEYWEiGIdwHMcPw05gjl/t66v6twEfoDOJCLu37WHo6CGA4ISzDCBMBGhH+0ArpZQanFzX4LrxtK4U8bjrvy2mzwwRCoWIREL+7zDhcJhQyGnRYqzf6qqBIqcALSJHAuOD+xtjHumhMvUoEWHP9t3sc9A+ANjZ5oBO9oEO6X92pVTeGUh1tup+He1GEYlEsnajCM4koe+VarBpN0CLyG+BfYF3gLi/2QB5URkXFRXyl9fm09TU7M35bFm4cddfhTD7Iiquv4iKWNq3SimVX7pSZ4vIXOAuwAbuN8bckmWf44A7gRCw1RhzbDcUW3VRYgaK9NbiOOlzCgMYHCfVjSLYWqzdKJTKXS4t0DOBA022j6d5oHJoBZVDK3jj1UXEYnHsOq9fVtmQMpxI9hZmNxZFLP1ErZTKS52qs0XEBu4B5gA1wCIRWWCMWRbYpwL4FTDXGLNWREZ0X7FVNp3pRpEZjIPdKBJzDCuluiaXAP0+MBLY2MNl6XkGdm7aCbQ+hR1409iJbYNWMkqp/NPZOns2sNIYsxpAROYDZwLLAvtcCDxljFkLYIz5pOvFHZzSu1G0PU1bIhiHw8V+MM4+G4U2+ijVe3IJ0MOAZSLyFtCU2GiMOaPHStVDQiGHnZt3YIdsCooKWg3QJhbDKijQZbyVUvmos3X2GGBd4HoNcFjGPvsDIRFZCJQCd2XrWy0iVwBXAIwbN66j5c9b2o1CqcEjlwB9Y08XorcUFhd6U9gNK8fgZg3QxhhMPObPAa0Vl1Iq79zYyeOyNV9mdgNxgEOBzwCFwBsi8qYx5qO0g4y5D7gPYObMmXnZ/S8oWzcK1zV+IE48vOzTtEUi4cA0banWYu1GoVR+y2Uhlf/rjYL0hqLCAnZv203lyEpAsEMtBwmaWAwA0VUIlVJ5qAt1dg0wNnC9CtiQZZ+txpg6oE5E/gYcBHxEHurIanfajUIpFZTLLByHA3cDk4Ew3ujsOmNMWQ+XrVuJWIRDYfZs38OEaROA1mfgALB0FUKlVB7qQp29CNhPRCYA64EL8Po8Bz0D/FJEHP/chwF3dGPxu6ytbhTBKt0Yg23bRCIhXe1OKdVhuXTh+CVeRfpHvNHdXwT268lC9YRQOETjngbisTjlwysAkz1AB1ch1ACtlMo/naqzjTExEbkGeBEvdD9ojFkqIlf6t99rjPlARF4A3gVcvKnu3u+hx5FZvizzFxvSe5loNwqlVO/IaSEVY8xKEbGNMXHgNyLyei7HtTenqD+f6DPAGn/TU8aYm3Ise4eEQw47N24HoGxIKU7IwcoyOEMDtFIq33W2zjbGPA88n7Ht3ozrPwN+1m2FbYUgNDY2gQjGJLpROITD4WQojkTC/kA7Xe1OKdW7cgnQ9SISBt4RkVvxpkYqbu+gXOYU9b1qjDmtg+XusOF7DeOdt9cCUDKklHAbU9iBt4y3VsJKqTzUqTq7vxk9Zi/KS4soKinWbhRKqX4nl++vvuDvdw1QhzfI5LM5HJecU9QY0wwk5hTtEyNHj2D7xm3Yjk1hcWHrc0DHoiAWlp1T47xSSvU3na2z+5VIJExRUWGylVkppfqTXGbh+FhECoFRxpgfduDcucwpCnCEiPwLb7T3N4wxSztwHzlz4y47t+ykbFgZxhgiRa3MAR1NLOOtfeOUUvmnC3W2UkqpHLWbEkXkdOAd4AX/+gwRWZDDuXOZU/SfwN7GmIPwRo0/3UoZrhCRxSKyeMuWLTncdUux5hi7t+2hYngFxrhZBxCCv4y3bYPOAa2UykNdqLOVUkrlKJeUeCNed4ydAMaYd4DxORzX7pyixpjdxpha//LzeCtcDcs8kTHmPmPMTGPMzOHDh+dw1y1Fm6Ls2b6b8uHlgLQaoE0spi3QSql8diOdq7OVUkrlKJeUGDPG7OrEuZNzivoDWi4A0lpBRGSk+CP1RGS2X55tnbivdu34ZAfxaGIKO7IvomKM1wJtOYilAwiVUnmps3W2UkqpHOUyUu59EbkQsEVkP+A6oN0pkXKZUxQ4F7hKRGJAA3CBSa2L2q02V28GoHxYOZZlZW+Bdl0wBnFsncJOKZWvOlVnK6XUQGJcgzEmOYd8d8slQF8LfBdoAh7DC8Q/yuXk7c0paoz5Jd6k/z1ua43Xd7qksoRQQSjrPokp7CydA1oplb86XWcrpVRfMcakhd7kZf+36xowJjWYzpA22k4QDCa5XUSwHQvLsrAci7Li0m4tby6zcNTjVcbf7dZ77mXbNm3Hsi0KS9qaws5fRMXRZbyVUvlpoNTZSqn+rbWgm7nN29c/qI1oJQKWZXuh17aww96Cd7ZtYTk2lm3h+IFYLAvLEsQSxLIQEf+6hVj+5UCO88oS79bH326AFpGZwHfwBqEk9zfGTO/WkvSw3Vt2UT6sHOO2PoVdchEV26bNv7JSSvVTA6XOVkp1P9d1MQZM5m/jpoVeEWkxbVomASzbwrK90GuHbD/oCrbtX7a80JsIuolgmxZ0AwE4n+TSheN3wPXAe0D3dyLpJbu37fYCtHEJR8JZ90ks461dOJRSeWxA1NlKKa8fr+sl3GQXBtc1XuANBGDwQ29m6hXSJhC2bAvbshDb8pa9tx1sx/ZCcCAMB4Nuay29+RZ4u1suAXqLMSav5xA1xrB7227GThoLCE44+6pWqWW8tQuHUipv5X2drVS+aqsrgxu43v6JAMvr15vo0pAKt1Yg9EqypTct9NqBlt5k6BXNNt0olwD9nyJyP/AK3qAUAIwxT/VYqbrZnu17iEVjgSns2lpExUEsXTZWKZW38r7OVqq3tNePN23gWk79eMUPuF6/XduxAgPZ/Ot+GG6t325r/XhV/5JLgL4UmASESH0daIC8qYy3rt8KQNmwMkRoYxGVxDLe+oJVSuWtvK+zlcqFG3dxXRc37qYH4WQrr7Q7nMnrx+sNULOdjIFr/uA127/eWr/dzO4NanDIJUAfZIyZ1uMl6UHFZcU4IYdhY4YxrGp4q5/odBlvpdQAkPd1thrcEqE4GZBjfj/fjLdu27YJRRwiRQWpgWt+625w4FpbLb0aeFVn5RKg3xSRA40xy3q8ND1g55adrPjnR5x46UkMrxrGB29+wKyTZ2UdSGiiMayisC7jrZTKZ3ldZ6uBy+sSkQrH8Xg81VIcYNneYmeRogihcAgnHCIUtlP9fh0vJGv4VX0plwB9NDBPRNbg9acTwOTDlEgNtQ0s+/tS/ueRlzn36+fy4HcexAk7zDh+RosAbVwX48a9PtDaAq2Uyl95W2er/GSMSW8xjrt+32FS2diAWEIo7OCEQ4QLQsnLtu1PgZacCULfg1X/l0uAntvjpeghhSWFzDplNpUjK3n4Bw8D8B/3foXi8uIW+7qxGACiU9gppfJb3tbZqv+JxzO7U8RbvEd6Y4v8QBwJEYqECIcdf9CcnVwIw9ZgrAaQXFYi/Lg3CtJT6nbV8ey9zyav//PlJcyZd2LLFmh/FUJLp7BTSuWxfK+zVe9I62fs/7SYXE3AcWyccIhIYZhwQRgn7OAkulKEUi3G+r6pBptcWqDzVqILhxN2uPZX1/H2K2+z+l+riTZGWwToeGAOaG2BVkoplY+Ma4jH4+kzVLi0OQAvFHEIR0LYjuNPu5bqa6zBWKnsBnSALiwpZObJszjo+BlsrdnKjONncNz5x2XtwmH8VQjFsjVAK6WU6lfS+hnHXeL+1G3pO4HleAPwwoVhQpEwIX8AXnK5ZT8c6wA8pbpmQAdogHAkTDgSZsPKDUizUFJRknU/XYVQKaVUbzP+0sxuLB5oMTYtpm0TITAjhUOoIETIH4CX1tdY+xkr1SsGfIAOihS2nLouwY1FwbKwdBVCpZRS3SDRjSIeS3WnaEEglByA5xAKhwhHQslBd8FwrJTqPwZVgA4XRVq9zY0mlvHWSkoppVTrsg7Ay7bQh2MTioQoLAknp25znMzuFNrPWKl8NHgCtECksPUAbWJRLF2FUCmlBq3gQh+JgXjZFvqwbYtQJOQNwAs7hCIhnJCTXOBDF/pQauAbNAHasixC4VCrt7uxGHYkoi3QSik1wLQ6AC9joQ/LtnDCNqGCEEXhomQ4DoZi7WeslIJBFKCdkI0Tyv5wjTGYWBQpKkK0xUAppfJCcgBeMhzHvQF4GfuJ4A++Sy32EY6E0hb40IU+lFIdMWgCtOV4k75nY+L+KoSWzgGtlFL9RTwWp7mxObkaXgsCIcdbDrqgOOIF5Eioxep32s9YKdXdBk2AHjJqSKst0MllvHURFaWU6hciRRGiTU2ECyLeALyIgxNy0rtT6EIfSqk+MmgCdHFZy8VTEtxmXcZbKaX6k6EjKxkyosxb3EoppfoZ7fAFuDF/FUJbVyFUSimllFJt0wANmMQqhLZDi4k8lVJKKaWUCtAAjb+Mt4h24VBKKaWUUu3SAI23jLfoIipKKaWUUioHmhhJBGhH54BWSimllFLt0gCN1wdaW6CVUkoppVQuNDECbjyGWLYu462UUkoppdo16BOjcePguojjaIBWSimllFLtGvSJ0Y36qxDqFHZKKaWUUioHGqD9RVQs29Yp7JRSSimlVLs0QCcWUXFCugqhUkoppZRqlwboqC7jrZRSSimlcjfoA3RqGW/twqGUUkoppdo36AN0YhVCy7L7uihKKaWUUioPaID2F1HRKeyUUkoppVQuBn1qdGNRxHJAA7RSSimllMrBoE+NJhbz+z8P+qdCKaWUUkrloEdTo4jMFZHlIrJSRL7Vxn6zRCQuIuf2ZHkyGWMwcT9AWzqAUCmllFJKta/HArSI2MA9wMnAgcDnReTAVvb7KfBiT5WlNSaWmAPa0SnslFJKKaVUTnqyBXo2sNIYs9oY0wzMB87Mst+1wJPAJz1YlqzcmLeMt2VrgFZKKaWUUrnpyQA9BlgXuF7jb0sSkTHA2cC9bZ1IRK4QkcUisnjLli3dVkC3ObWIis4BrZRSSimlctGTATpbIjUZ1+8EvmmMibd1ImPMfcaYmcaYmcOHD++u8gVWIdQWaKWUUkoplRunB89dA4wNXK8CNmTsMxOY77f+DgNOEZGYMebpHixXkhtLrUKYPe8rpZRSSimVricD9CJgPxGZAKwHLgAuDO5gjJmQuCwiDwHP9lZ4Bm8RFSwLy3G0C4dSSimllMpJjwVoY0xMRK7Bm13DBh40xiwVkSv929vs99wbTCyKWDboHNBKKaWUUipHPdkCjTHmeeD5jG1Zg7Mx5pKeLEs2biyqc0ArpZRPROYCd+E1etxvjLkl4/bjgGeANf6mp4wxN/VmGZVSqj/o0QDd37mxKFYooi3QSqlBLzB3/xy8MSyLRGSBMWZZxq6vGmNO6/UCKqVUPzJok6MxJrWMtzVonwallErIde5+pZQa9AZtcjRuHIzBsh0N0EoplcPc/b4jRORfIvIXEZmS7UQ9NXe/Ukr1F4M2OZpoYBlvncJOKaVymbv/n8DexpiDgLuBp7OdqKfm7ldKqf5i0AboeCJA2zqFnVJKkcPc/caY3caYWv/y80BIRIb1XhGVUqp/GLQB2sS8VQgtx9ZVCJVSKjB3v4iE8ebuXxDcQURGit/iICKz8d5DtvV6SZVSqo8N2lk43EQLtKXLeCulVI5z958LXCUiMaABuMAYk9nNQymlBrxBHqAFbEu7cCilFO3P3W+M+SXwy94ul1JK9TeDtgtHYhEVy7L7uihKKaWUUiqPDNoAbZKrEA7ap0AppZRSSnXCoE2PbtQL0GiAVkoppZRSHTBo06O3CqGD6DLeSimllFKqAwZlejSui3Hj2oVDKaWUUkp12KBMj24stYiKTmGnlFJKKaU6YlAGaOMHaMtxdBVvpZRSSinVIYNyHujUMt62zgE9gESjUWpqamhsbOzroihFQUEBVVVVhEKhvi6KUkqpbjYoA7SJest4axeOgaWmpobS0lLGjx+vH4xUnzLGsG3bNmpqapgwYUJfF0cppVQ3G5RdONxAC7T24Rg4GhsbGTp0qIZn1edEhKFDh+q3IUopNUAN3gBt2VjahWPA0b+n6i/0taiUUgPX4AzQsSiWLqKilFJKKaU6YVAmyOQy3rqIiuonFi5cyGmnndZt57v88stZtmwZAH/84x+ZPHkyxx9/PIsXL+a6667rtvvpLQsXLuT1119PXr/33nt55JFHuuXcP/nJT7rlPEoppQaPQTmI0I1FsSMFOoBQtSkWi+E4+flf5P77709efuCBB/jVr37F8ccfD8DMmTNzPk9/eQ4WLlxISUkJRx55JABXXnllt537Jz/5Cd/5zndy3t8YgzEGS7/BUkqpQWvQvQMYY1LLeOsb4IBXXV3NpEmTmDdvHtOnT+fcc8+lvr6em266iVmzZjF16lSuuOIKjDEAHHfccXznO9/h2GOP5a677uLPf/4zhx12GAcffDAnnHACmzdvBuDGG29k3rx5nHjiiYwfP56nnnqKG264gWnTpjF37lyi/kDVbBYtWsSRRx7JQQcdxOzZs9mzZ0/a7W+99RZHHnkkBx98MEceeSTLly8HYOnSpcyePZsZM2Ywffp0VqxYQV1dHaeeeioHHXQQU6dO5fHHH08+jsWLF3PTTTfx2muvceWVV3L99dentXTX1dVx2WWXMWvWLA4++GCeeeYZAB566CE+97nPcfrpp3PiiSfm/FwvXLiQ4447jnPPPZdJkyZx0UUXJZ/XbJYsWcKxxx7LoYceykknncTGjRsB+MUvfsGBBx7I9OnTueCCC6iurubee+/ljjvuYMaMGbz66qvceOON/PznP08+1q9+9ascc8wxTJ48mUWLFnHOOeew33778b3vfS95f2eddRaHHnooU6ZM4b777gPgW9/6Fg0NDcyYMYOLLroIgNtvv52pU6cydepU7rzzTsB7HU2ePJmrr76aQw45hHXr1nHJJZcwdepUpk2bxh133JHz86SUUir/9X3TUi8z8RiALuM9iCxfvpwHHniAo446issuu4xf/epXXHPNNfzgBz8A4Atf+ALPPvssp59+OgA7d+7k//7v/wDYsWMHb775JiLC/fffz6233sptt90GwKpVq/jrX//KsmXLOOKII3jyySe59dZbOfvss3nuuec466yzWpSlubmZ888/n8cff5xZs2axe/duCgsL0/aZNGkSf/vb33Ach5dffpnvfOc7PPnkk9x77738x3/8BxdddBHNzc3E43Gef/55Ro8ezXPPPQfArl270s71gx/8gP/93//l5z//OTNnzmThwoXJ23784x/z6U9/mgcffJCdO3cye/ZsTjjhBADeeOMN3n33XYYMGdKh5/rtt99m6dKljB49mqOOOoq///3vHH300S32i0ajXHvttTzzzDMMHz6cxx9/nO9+97s8+OCD3HLLLaxZs4ZIJMLOnTupqKjgyiuvpKSkhG984xsAvPLKK2nnC4fD/O1vf+Ouu+7izDPPZMmSJQwZMoR9992Xr371qwwdOpQHH3yQIUOG0NDQwKxZs/jsZz/LLbfcwi9/+UveeecdwAv1v/nNb/jHP/6BMYbDDjuMY489lsrKSpYvX85vfvMbfvWrX7FkyRLWr1/P+++/D3ivGaWUUoPHoAvQySnsHAedwm5wGDt2LEcddRQAF198Mb/4xS+YMGECt956K/X19Wzfvp0pU6YkA/T555+fPLampobzzz+fjRs30tzcnDan78knn0woFGLatGnE43Hmzp0LwLRp06iurs5aluXLlzNq1ChmzZoFQFlZWYt9du3axbx581ixYgUikmzNPuKII/jxj39MTU1NsoV12rRpfOMb3+Cb3/wmp512Gp/61Kdyfl5eeuklFixYkGzJbWxsZO3atQDMmTOnw+EZYPbs2VRVVQEwY8YMqqurswbo5cuX8/777zNnzhwA4vE4o0aNAmD69OlcdNFFnHXWWVk/hGRzxhlnAN5zP2XKlOS59tlnH9atW8fQoUP5xS9+wZ/+9CcA1q1bx4oVKxg6dGjaeV577TXOPvtsiouLATjnnHN49dVXOeOMM9h77705/PDDk+ddvXo11157LaeeemqHWuqVUkrlv0HXBJsI0JYd0mmmBonMv7OIcPXVV/PEE0/w3nvv8e///u9p8/UmwhPAtddeyzXXXMN7773Hf/3Xf6XtF4lEALAsi1Ao9XqyLItYLJa1LMaYdl933//+9zn++ON5//33+fOf/5y8zwsvvJAFCxZQWFjISSedxP/+7/+y//77s2TJEqZNm8a3v/1tbrrpppyfF2MMTz75JO+88w7vvPMOa9euZfLkyS2eg45IPCcAtm23+TxMmTIled/vvfceL730EgDPPfccX/7yl1myZAmHHnpoq+fIdr+WZaWVIfG3WLhwIS+//DJvvPEG//rXvzj44IOzztHcVpeT4HNSWVnJv/71L4477jjuueceLr/88nbLqJRSauAYhAHaX4XQsXUQ4SCxdu1a3njjDQAee+yxZIvosGHDqK2t5Yknnmj12F27djFmzBgAHn744S6XZdKkSWzYsIFFixYBsGfPnhYBMXifDz30UHL76tWr2Weffbjuuus444wzePfdd9mwYQNFRUVcfPHFfOMb3+Cf//xnzmU56aSTuPvuu5Oh8e233+7io8vdAQccwJYtW5J/l2g0ytKlS3Fdl3Xr1nH88cdz6623snPnTmprayktLW3RV7wjdu3aRWVlJUVFRXz44Ye8+eabydtCoVCylf+YY47h6aefpr6+nrq6Ov70pz9lbdXfunUrruvy2c9+lh/96Ecdet6VUkrlv0HXhcMkunBYuoz3YDF58mQefvhhvvSlL7Hffvtx1VVXsWPHDqZNm8b48eOT3SmyufHGG/nc5z7HmDFjOPzww1mzZk2XyhIOh3n88ce59tpraWhooLCwkJdffjltnxtuuIF58+Zx++238+lPfzq5/fHHH+fRRx8lFAoxcuRIfvCDH7Bo0SKuv/76ZCv4r3/965zL8v3vf5+vfOUrTJ8+HWMM48eP59lnn+3S48tVOBzmiSee4LrrrmPXrl3EYjG+8pWvsP/++3PxxReza9cujDF89atfpaKigtNPP51zzz2XZ555hrvvvrvD9zd37lzuvfdepk+fzgEHHJDsigFwxRVXMH36dA455BB+97vfcckllzB79mzAmw7w4IMPbtElZ/369Vx66aW4rgvAzTff3PknQymlVN6Rtr6y7I9mzpxpFi9e3Onj62o+pmnHVkrG70e4tGX/U5W/Pvjgg2QXhITq6mpOO+205GAvpXpTttekiCwxxuQ+l2Ce62ydHauvw7hxxLJ7oFRKqcHEGINx44RLyzt8bGt19uDrwhGLelPYaeuzUkoppZTqhEHXhcONRnUKu0Fk/Pjxfdb6fPbZZ7fo8vHTn/6Uk046qU/K0xnvvfceX/jCF9K2rVu3jrFjx6Zti0Qi/OMf/8h6joHwPCillFJBgy5Am1gUCYVAA7TqYYkp0/LZtGnTknMkd9ZAeB6UUkqpoEGXIt14DLEcRAbdQ1dKKaWUUt1gUKVI48bBdRFHu3AopZRSSqnOGVQpMrkKoa1zQCullFJKqc4ZlAHacnQWDqVyMX78eKZNm8aMGTOYOdObxWf79u3MmTOH/fbbjzlz5rBjx47k/jfffDMTJ07kgAMO4MUXX+yrYiullFI9anAF6Ji/CqGti6iogadpxzZ2fvAu299dzM4P3qVpx7ZuOe9f//pX3nnnHRJz+d5yyy185jOfYcWKFXzmM5/hlltuAWDZsmXMnz+fpUuX8sILL3D11VcTj8e7pQxKKaVUfzK4ArR24VADVNOObdTVfJxcqt6NNvuLBnVPiA565plnmDdvHgDz5s3j6aefTm6/4IILiEQiTJgwgYkTJ/LWW291+/0rpZRSfW1QTWOXCBe6spXKN3Ub1hJvqG/19lh9HWSuKmpc6mqqadq+JesxdmERxaPHtXm/IsKJJ56IiPClL32JK664gs2bNzNq1CgARo0axSeffAJ4y1sHl8iuqqpi/fr1uTw8pZRSKq/0aIAWkbnAXYAN3G+MuSXj9jOBHwEuEAO+Yox5rafKY2LeIiqWbWsfaDWwZIbn9rbn6O9//zujR4/mk08+Yc6cOUyaNKmNIrS8L/1/ppRSaiDqsQAtIjZwDzAHqAEWicgCY8yywG6vAAuMMUZEpgN/AFp/h+4iNxrz+j/rFHYqz7TXUrzzg3eT37AEWaEwZft2/r/U6NGjARgxYgRnn302b731FnvttRcbN25k1KhRbNy4kREjRgBei/O6deuSx9bU1CSPV0oppQaSnkySs4GVxpjVxphmYD5wZnAHY0ytSTVbFQNday5rhxuLIpati6ioAadw5BjIfF2L5W3vpLq6Ovbs2ZO8/NJLLzF16lTOOOMMHn74YQAefvhhzjzT+299xhlnMH/+fJqamlizZg0rVqxg9uzZnb5/pZRSqr/qyS4cY4B1ges1wGGZO4nI2cDNwAjg1GwnEpErgCsAxo1ruyWuLSYWwyoo1AGEasCJVA4FoGHTetxoM1YoTOHIMcntnbF582bOPvtsAGKxGBdeeCFz585l1qxZnHfeeTzwwAOMGzeOP/7xjwBMmTKF8847jwMPPBDHcbjnnnuwbR1voJRSauDpyQCdLaW2aGE2xvwJ+JOIHIPXH/qELPvcB9wHMHPmzE61UhtjMHGvC4euQqgGokjl0C4F5kz77LMP//rXv1psHzp0KK+88krWY7773e/y3e9+t9vKoJRSSvVHPZkka4CxgetVwIbWdjbG/A3YV0SG9URhTCywiIoGaKWUUkop1Uk9mSQXAfuJyAQRCQMXAAuCO4jIRPGH6YvIIUAY6P6Ja8mYAzpr47hSSimllFLt67EuHMaYmIhcA7yIN43dg8aYpSJypX/7vcBngS+KSBRoAM432ebC6gapAK3LeCullFJKqc7r0XmgjTHPA89nbLs3cPmnwE97sgwJyUVUdBlvpZRSSinVBYOmM7Dr94EWR5fxVkoppZRSnTd4AnQ0CpaF6CqESimllFKqCwZNgE4s4y06gFCpnF122WWMGDGCqVOnJrdt376dOXPmsN9++zFnzhx27NiRvO3mm29m4sSJHHDAAbz44ovJ7UuWLGHatGlMnDiR6667Luuy30oppVS+GDQB2o1GsSydwk4NXKveWcUdV9zOnu170i53xSWXXMILL7yQtu2WW27hM5/5DCtWrOAzn/kMt9xyCwDLli1j/vz5LF26lBdeeIGrr76aeDwOwFVXXcV9993HihUrWLFiRYtzKqWUUvlk0KRJN+61QKMBWg1Aq95ZxUPf/w1b1m1h/i2PJS+/8ruXu3TeY445hiFDhqRte+aZZ5g3bx4A8+bN4+mnn05uv+CCC4hEIkyYMIGJEyfy1ltvsXHjRnbv3s0RRxyBiPDFL34xeYxSSimVj3p0Fo7+whiDicWQgiJENECr/PPnXy9gw6qNrd5es3wd0SZvoOzqf61OdpF46/m32PzxJ1mPGb3vKE6/6owOl2Xz5s2MGjUKgFGjRvHJJ975169fz+GHH57cr6qqivXr1xMKhaiqqmqxXSmllMpXgyNAu3EwBtFVCNUANXKfUWxZ+wlN9U3J8CwiDBvTIwt7ZpWtX7OItLpdKaWU6grv/cWAAYzBJC6Dt2Ze4P1H7O6NvIMjQAcWUdEp7FQ+aq+lONGFIxhWLdtiwvR9OPu6s7u1LHvttRcbN25k1KhRbNy4kREjRgBey/K6deuS+9XU1DB69GiqqqqoqalpsV0ppZQKSr6HGdMyEBNsjJHkotIiltc4KoKIIJbl9Tbwr3v7Sbfnv0HRHJucA1pXIVQD1IJfPUM85g3YC4VD2I5NPBbn/Vff6/b7OuOMM3j44YcBePjhhznzzDOT2+fPn09TUxNr1qxhxYoVzJ49m1GjRlFaWsqbb76JMYZHHnkkeYxSSqmByxjj/bguxo1j4nHcWAw3FvV/Yt5P3Ptt4nEwLliCODZWKIxdUIBTWIRTVIJTXEKopNT/KSNcWk6opBSnqNjbp6AQOxzBCoWwHMebfc2y/VDdvflvULRAJ5bxtmxdREUNTJff8u+88ruXee9v73HWtWex8p1VvP/qe1z43Yu6dN7Pf/7zLFy4kK1bt1JVVcUPf/hDvvWtb3HeeefxwAMPMG7cOP74xz8CMGXKFM477zwOPPBAHMfhnnvuwbZtAH79619zySWX0NDQwMknn8zJJ5/c5ceslFKq9xm/dTjRdcJrJTbgTxRshLRWY6912G8VtiwssRDLbxEW8Y5KXM6jjCb5Nh/rzJkzzeLFi3Pe3xhDvKEe47qIbWNFCrC0H/SA9MEHHzB58uS+LoZSSdlekyKyxBgzs4+K1Os6WmcnxOrrMG4cseweKJVSKiF7P2IvEJMMw4G+xJYFeF0l8LtLJAMxiW4T+ReIW9NanT2gW6CNMcQbG6j9eBVutBkrFKZk/ESkoHBA/FGVUkoppYLa7EecMbDO22Yl+w4jgliS6lecFohJXVcDPEDHotRWr8SNNgPgRpuprV5J2cRJSCjcx6VTSimllGpbWiDG+F0oyAjDgVAr+N0lBBG7xcA673ZJ7KiBuJMGdoA2JhmeE9xosy4jrJRSSqk+kRmIvR4T/mUgOMNEYpPXRcJKBuHUzBMEQrAG4t40oAO0iGCFwmkh2gqF9cWllFJZiMhc4C7ABu43xtzSyn6zgDeB840xT/RiEZXql1ID62jZjxha9CVO9SO2B9TAusFkYAdoJ0TJ+InJbhzJPtBOqK+LppRS/YqI2MA9wBygBlgkIguMMcuy7PdT4MXeL6VSvaP9BTqA4LzEloUkBtaJheWH4Zb9iL0wrKE4/w3sAC2CXVBI2cRJ3tyClqUt0Eopld1sYKUxZjWAiMwHzgSWZex3LfAkMKt3i6dU53VogQ6M3/KbwwIdwZkn1KAy4OdzS3TjiEejPTKRtlID2WWXXcaIESOYOnVqctv111/PpEmTmD59OmeffTY7d+5M3nbzzTczceJEDjjgAF58MdVAuWTJEqZNm8bEiRO57rrrdBxC/zQGWBe4XuNvSxKRMcDZwL29WC41SCUX4Uj7cf1FOYI/2RboiHZ+gY7S8twX6BDNFYPVgA/QCRL46kQplZtLLrmEF154IW3bnDlzeP/993n33XfZf//9ufnmmwFYtmwZ8+fPZ+nSpbzwwgtcffXVxOPe6ohXXXUV9913HytWrGDFihUtzqn6hWwVZOYnnTuBbxpj4m2eSOQKEVksIou3bNnSXeVT3aBDodQPpokfNx5L/SRWkMsMrMFtLfYLHpvtejTtsonH/HK4Xvg1bqAbhXhh2LK8IBsKYUcSgbjYC8TFJYSKSwmVlhEuKydUUkaoqASnsLhFILZsp8dWrFMD04DuwpFGO+KrAe74Q89i29YdyetDh1Xy1yVPd+mcxxxzDNXV1WnbTjzxxOTlww8/nCee8MaQPfPMM1xwwQVEIhEmTJjAxIkTeeuttxg/fjy7d+/miCOOAOCLX/wiTz/9tK5G2P/UAGMD16uADRn7zATm+3XpMOAUEYkZY54O7mSMuQ+4D7yFVHqqwP1F9m9UTMuPH4kNJrjFtLi5ZZeCwDaRjP0yF7vIcj3znMmBahn3kZjiLG1KtMTsDqkzSWBb2vVgcdPOm/iVuY9k7Jr+Hq3v2ao/G0QBOvmPUnnnpz+8m+XLVra5TzA8J65fdv5/tLr/AQdO5Jv/eW2XyvXggw9y/vnnA7B+/XoOP/zw5G1VVVWsX7+eUChEVVVVi+2q31kE7CciE4D1wAXAhcEdjDETEpdF5CHg2czw3K1MZjjthlDaIoB628QEj+uLUGplHJYtlGYGTg2lSvWVQROgLcvWLhxKdaMf//jHOI7DRRddBGRvhRORVrer/sUYExORa/Bm17CBB40xS0XkSv/23u33bFuYeAyM234oTQbFQCgVEDoeSiVLSG0rlOprWanBadAEaLugsK+LoFSn5dJSPH3vY1tse/Dxu3qiODz88MM8++yzvPLKK8kAUVVVxbp1qTFoNTU1jB49mqqqKmpqalpsV/2PMeZ54PmMbVmDszHmkp4sixMphIjW20qp/mnQDCJUaqAbOqyyzevd5YUXXuCnP/0pCxYsoKioKLn9jDPOYP78+TQ1NbFmzRpWrFjB7NmzGTVqFKWlpbz55psYY3jkkUc488wze6RsSimlVG8YNC3QSg10XR0wmM3nP/95Fi5cyNatW6mqquKHP/whN998M01NTcyZMwfwBhLee++9TJkyhfPOO48DDzwQx3G45557sG0bgF//+tdccsklNDQ0cPLJJ+sAQqWUUnlN8m0+1pkzZ5rFixf3dTFUP/TBBx8wefLkvi6GUknZXpMissQYM7OPitTrtM5WSuWz1ups7cKhlFJKKaVUB2iAVkoppZRSqgM0QCullFJKKdUBGqDVgJJvffrVwKWvRaWUGrg0QKsBo6CggG3btmlwUX3OGMO2bdsoKCjo66IopZTqATqNnRowEgt2bNmypa+LohQFBQVpS5grpZQaODRAqwEjFAoxYcKEvi6GUkoppQY47cKhlFJKKaVUB2iAVkoppZRSqgM0QCullFJKKdUBebeUt4hsAT7OcfdhwNYeLE5f08eX3/Tx5bfOPr69jTHDu7sw/VUH6+yg/vT66S9l6S/lAC1LNv2lHNB/ytJfygHdXGfnXYDuCBFZnG398oFCH19+08eX3wb64+tr/en57S9l6S/lAC1Lfy4H9J+y9JdyQPeXRbtwKKWUUkop1QEaoJVSSimllOqAgR6g7+vrAvQwfXz5TR9ffhvoj6+v9afnt7+Upb+UA7Qs2fSXckD/KUt/KQd0c1kGdB9opZRSSimluttAb4FWSimllFKqW+VtgBaRuSKyXERWisi3stwuIvIL//Z3ReSQXI/tDzr7+ERkrIj8VUQ+EJGlIvIfvV/6tnXlb+ffbovI2yLybO+VOnddfG1WiMgTIvKh/zc8ondL374uPr6v+q/L90XkMREp6N3Sty+HxzdJRN4QkSYR+UZHjlUgIgUi8paI/Mt/Lfwwyz49Xn/nWI6L/Pt/V0ReF5GDArdVi8h7IvKOiCzubDk6UJbjRGSXf3/viMgPArd12+sux7JcHyjH+yISF5Eh/m3d9rz452u1vu+N10mO5eiV10mOZemV10mOZenN10mb5+uR14oxJu9+ABtYBewDhIF/AQdm7HMK8BdAgMOBf+R6bF//dPHxjQIO8S+XAh/1p8fXlccWuP1rwO+BZ/v68XT34wMeBi73L4eBir5+TN342hwDrAEK/et/AC7p68fUicc3ApgF/Bj4RkeO1R+D/7oo8S+HgH8Ah+f4Guq25zjHchwJVPqXT874v1oNDOvF5+S4bHVed7/ucilLxv6nA//bE8+Lf75W6/veeJ3kWI5eeZ3kWJZeeZ3kUpZefp20eb6eeK3kawv0bGClMWa1MaYZmA+cmbHPmcAjxvMmUCEio3I8tq91+vEZYzYaY/4JYIzZA3yAF1z6i6787RCRKuBU4P7eLHQHdPrxiUgZcAzwAIAxptkYs7MXy56LLv39AAcoFBEHKAI29FbBc9Tu4zPGfGKMWQREO3qsAv91UetfDfk/mYNxerz+zqUcxpjXjTE7/KtvAlWdua/uKEsbuvV114myfB54rLP315Yc6vteeZ9vrxy99TrJpSxt6Pb6qYNl6bHXSY66/bWSrwF6DLAucL2GliGxtX1yObavdeXxJYnIeOBgvBaE/qKrj+1O4AbA7aHydVVXHt8+wBbgN/5XYveLSHFPFrYTOv34jDHrgZ8Da4GNwC5jzEs9WNbO6Er9kA91S7/gf+37DvAJ8D/GmMw6qlfq7xzKEfRveC1YCQZ4SUSWiMgVnS1DB8tyhN+14i8iMsXf1u2vu1yfFxEpAuYCTwY2d+fzcidt1/e99T7fXjmCevR1kmNZeuV1kmNZeuN1ksv5uv21kq8BWrJsy/yE3No+uRzb17ry+LwbRUrwXqxfMcbs7saydVWnH5uInAZ8YoxZ0v3F6jZd+ds5wCHAr40xBwN1QH/rR9uVv18l3if7CcBooFhELu7m8nVVV+qHfKhb+gVjTNwYMwOvpW62iEzN2KVX6u8cyuEVRuR4vGD0zcDmo4wxh+B9Zf9lETmms+XIsSz/xFtS+CDgbuDpRPGyna6Hy5JwOvB3Y8z2wLZueV5yrO97/HXSkfednn6d5FiWXnmddPD9uMdeJx04X7e/VvI1QNcAYwPXq2j5VXBr++RybF/ryuNDREJ44fl3xpinerCcndGVx3YUcIaIVON9zfJpEXm054raKV19bdYEWnuewAvU/UlXHt8JwBpjzBZjTBR4Cq/vYH/SlfohH+qWfsXvorQQr3UqqFfr7zbKgYhMx/uK+kxjzLbAMRv8358Af8L7KrjLWiuLMWZ3omuFMeZ5ICQiw+jB111bz4vvAjK+lu/G5yWX+r43Xic5ve/00uuk3bL04uukI+/HPfk6yfV83f9aMd3Ysb23fvBa6lbjtWQlOn1PydjnVNI7jL+V67F9/dPFxyfAI8Cdff04uvuxZexzHP1zEGGXHh/wKnCAf/lG4Gd9/Zi68bV5GLAUr++z4A2YvLavH1NHH19g3xtJH0TY7+uW/vADDMcfHAsU+q/503J8DXXbc5xjOcYBK/9/e/cWY1dVx3H8+9MWtC2WpCCgMWlCiw3hUmwtUmvTh9r4pG1CFSEapAGpAUVCMSGISkW8PIDGVEgMXhKgXCWGkBZCaGrvgLW3QFBDX4wPXoBSasG2Px7WGmYznpmc086cOWN/n2Ry9ll7r8s+2ee/1+y99lnA3AHpE4GTGsubgM+M8GdyOv1zN8yhDIXScB937bSlrpsM/BuYOFKfS6PcBbR+MK6r5/kh2tGV46TNtnTlOGmnLd06TtopbySOlXGMQbYPSboGWEt5gvIe23skXV3X3wU8QXnq8i/AAeArQ+Udhd0Y1LHsH+W/wi8Bu+oYNoCbXP4THXXHuG89bxj271rgXkknUL7UPbXvx/jd2yrpYcotxkPAdnprlqq29k/S6cBzwAeAI5Kuozy1va/XY0uPOAP4jaT3Uu6CPmj78VGI3+204xZgCrBKEsAh27OB04Df1bRxwH221xxlO9pty8XAckmHgP8Al7j0AIb7nNZOWwCWAE/afqORd7g/l//RK+f5UTpO2mlLt46TdtoC3TlOWpY30sdKZiKMiIiIiOjAWB0DHRERERExKtKBjoiIiIjoQDrQEREREREdSAc6IiIiIqID6UBHRERERHQgHejoOZL2d6GOqyV9eaTrGVDnYklnd7POiIjjhaR1kmYfZd53xedjKSuOD+lAx/+t+jumLdm+y/Zvu1knsBhIBzoiovcsJvE5OpAOdPQ0SSskPStpp6TvNdIfk/S8pD2Srmqk75d0q6StwEX1/W2SdkjaIum0ut13Jd1Ql9dJ+pGkbZJekvSpmj5B0oO17gckbW11RULSXkm3SNoALJV0ZW3zDkmP1HLmAp8FfiLpT5LOrH9r6n78QdKMkf00IyK6Q9JUSS9K+qWk3ZLulbRQ0kZJf5Y0p243R9ImSdvr60dr+vWS7qnL59YyJgyo4/2SVvfFaMrMiX3rFknaLOmPkh6SNKmm723E+22SprWKz7WYpQPPCxF90oGOniVpETCdMh3pTGCWpPl19RW2ZwGzga9LmlLTJwK7bV9oe0N9v8X2+cB64MpBqhtnew5wHfCdmvY14BXb5wErgVlDNPeg7Xm2VwOP2v54rfMFYJntTcDvgRW2Z9r+K2UWvmvrftwArGr/04mI6HnTgJ8C5wEzgEuBeZR4d1Pd5kVgvu0LKDP6/aCm3wlMk7QE+BXwVdsHBpS/HDhQY/Rt1Bgt6RTgZmCh7Y9RZg69vpFvX433PwfuHCQ+Q+vzQgTA2JzKO44bi+rf9vp+EqVDvZ7SaV5S0z9S0/8FHAYeaZTxFvB4XX4e+PQgdT3a2GZqXZ5HCf7Y3i1p5xBtfaCxfI6k7wMn1zavHbhxvRoyF3ioTj8KcOIQ5UdEjDUv294FIGkP8LRtS9pFf5ydTJk6fDpgYDyA7SOSLgd2Anfb3tii/PnAz+r2Oxsx+hOU4Rgba3w9AdjcyHd/4/WOIdrf6rwQAaQDHb1NwO22735XorQAWAhcZPuApHXA++rqg7YPNzb/r/vnqz/M4Mf8my220SDbtvJGY/nXwGLbO+oJYEGL7d8DvGp7Zgd1RESMJW82lo803h+hP86uBJ6xvUTSVGBdI890YD/woSHqcIs0AU/Z/mIbeVrl79PqvBABZAhH9La1wBWNsWsflvRByhWLV2rneQblasNI2AB8vtZ9NnBum/lOAv4uaTxwWSP99boO2/uAlyUtreVL0vnD1fCIiDFiMvC3unx5X6KkyZQ7gPOBKZIubpF3PTXGSjqHMlQEYAvwSUnT6roJks5q5PtC47XvyvQ78TmiHelAR8+y/SRwH7C53vJ7mBLg1gDj6u26lZRgORJWAafWer5FuZX4Whv5vg1sBZ6ijO/rsxpYUR+WOZMS+JdJ2gHsAT43nI2PiBgDfgzcLmkj0PwVozuAVbZfApYBP6wXUJp+AUyqMfpGYBuA7X9QOuP313VbKGOw+5xYHzT/BvDNmjYwPkcMSf13tyOiSeUn6cbbPlgD6tPAWbbfGuWmRUTEUZC0F5ht+5+j3ZYY2zKmJ2JwE4Bn6lAMAcvTeY6IiIhcgY6IiIiI6EDGQEdEREREdCAd6IiIiIiIDqQDHRERERHRgXSgIyIiIiI6kA50REREREQH0oGOiIiIiOjA2xePa1QCVTLnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,[ax1,ax2]=plt.subplots(1,2,figsize=(12,6))\n",
    "sns.lineplot(data=gs_gb_report, \n",
    "             x='param_classifier__learning_rate',\n",
    "             y='mean_test_score',\n",
    "             hue='param_classifier__n_estimators',\n",
    "             style=\"param_classifier__n_estimators\",\n",
    "             ax=ax1, markers=True, dashes=False,\n",
    "             )\n",
    "ax1.set(xlabel='learning rate',ylabel='mean test score', title='Performance of Gradient Boosting')\n",
    "sns.lineplot(data=gs_gb_report,\n",
    "             x='param_classifier__max_depth',\n",
    "             y='mean_test_score',\n",
    "             hue='param_classifier__n_estimators',\n",
    "             style=\"param_classifier__n_estimators\",\n",
    "             ax=ax2, markers=True, dashes=False)\n",
    "ax2.set(xlabel='max depth',ylabel='mean test score', title='Performance of Gradient Boosting');\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gradient Boosting is robust for different parameter setting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('preprocessor',\n",
       "                 ColumnTransformer(transformers=[('num',\n",
       "                                                  Pipeline(steps=[('imputer',\n",
       "                                                                   IterativeImputer(estimator=BayesianRidge(),\n",
       "                                                                                    max_iter=100,\n",
       "                                                                                    min_value=0,\n",
       "                                                                                    random_state=1,\n",
       "                                                                                    sample_posterior=True)),\n",
       "                                                                  ('scaler',\n",
       "                                                                   StandardScaler())]),\n",
       "                                                  ['YrSold', 'BsmtUnfSF',\n",
       "                                                   'GrLivArea', 'MiscVal',\n",
       "                                                   'YearRemodAdd',\n",
       "                                                   'ScreenPorch',\n",
       "                                                   'EnclosedPorch',\n",
       "                                                   'OpenPorchSF', 'PoolArea',\n",
       "                                                   '...\n",
       "                                                   'RoofStyle', 'RoofMatl',\n",
       "                                                   'Exterior1st', 'Exterior2nd',\n",
       "                                                   'MasVnrType', 'ExterQual',\n",
       "                                                   'ExterCond', 'Foundation',\n",
       "                                                   'BsmtQual', 'BsmtCond',\n",
       "                                                   'BsmtExposure',\n",
       "                                                   'BsmtFinType1',\n",
       "                                                   'BsmtFinType2', 'Heating',\n",
       "                                                   'HeatingQC', 'CentralAir',\n",
       "                                                   'Electrical', 'KitchenQual', ...])])),\n",
       "                ('classifier',\n",
       "                 GradientBoostingRegressor(learning_rate=0.05, loss='lad',\n",
       "                                           max_depth=5, min_samples_split=5,\n",
       "                                           n_estimators=1200))])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_gb.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression, Training score: 0.983\n",
      "Gradient Boosting Regression, Test score: 0.897\n",
      "---------------------------\n",
      "Gradient Boosting Regression Training RMSE: 0.051\n",
      "Gradient Boosting Regression, Test RMSE: 0.147\n"
     ]
    }
   ],
   "source": [
    "print(\"Gradient Boosting Regression, Training score: %.3f\" %gs_gb.score(X_train, y_train))\n",
    "print(\"Gradient Boosting Regression, Test score: %.3f\" % gs_gb.score(X_test, y_test))\n",
    "print('---------------------------')\n",
    "print(\"Gradient Boosting Regression Training RMSE: %.3f\" % logrmse(gs_gb.predict(X_train), y_train.values))\n",
    "print(\"Gradient Boosting Regression, Test RMSE: %.3f\" % logrmse(gs_gb.predict(X_test), y_test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBOOST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBOOST is an efficient implementation of Gradient Boosting of decision trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find best hyperparameter set\n",
    "eta = np.arange(0.01,0.21,0.01).tolist()\n",
    "min_child_weight = np.arange(1,11).tolist()\n",
    "max_depth  = np.arange(2,11).tolist()\n",
    "gamma = np.arange(2,11).tolist()\n",
    "lambdaa = np.arange(2,11).tolist()\n",
    "alpha = np.arange(2,11).tolist()\n",
    "colsample_bytree = np.arange(0.4,1.1,0.1).tolist()\n",
    "\n",
    "parameters = {'booster': ['gbtree'],\n",
    "              'eta': eta, \n",
    "              'min_child_weight': min_child_weight,\n",
    "              'max_depth': max_depth, \n",
    "              'gamma': gamma,  \n",
    "              'colsample_bytree': colsample_bytree\n",
    "             }\n",
    "\n",
    "model = xgb.XGBRegressor()\n",
    "gs_xg= RandomizedSearchCV(model,parameters, n_iter = 30)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_pr = preprocessor.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                          colsample_bylevel=None,\n",
       "                                          colsample_bynode=None,\n",
       "                                          colsample_bytree=None, gamma=None,\n",
       "                                          gpu_id=None, importance_type='gain',\n",
       "                                          interaction_constraints=None,\n",
       "                                          learning_rate=None,\n",
       "                                          max_delta_step=None, max_depth=None,\n",
       "                                          min_child_weight=None, missing=nan,\n",
       "                                          monotone_constraints=None,\n",
       "                                          n_estimators=100, n_jobs...\n",
       "                                        'colsample_bytree': [0.4, 0.5, 0.6, 0.7,\n",
       "                                                             0.7999999999999999,\n",
       "                                                             0.8999999999999999,\n",
       "                                                             0.9999999999999999],\n",
       "                                        'eta': [0.01, 0.02, 0.03, 0.04, 0.05,\n",
       "                                                0.060000000000000005,\n",
       "                                                0.06999999999999999, 0.08, 0.09,\n",
       "                                                0.09999999999999999, 0.11, 0.12,\n",
       "                                                0.13, 0.14, 0.15000000000000002,\n",
       "                                                0.16, 0.17, 0.18000000000000002,\n",
       "                                                0.19, 0.2],\n",
       "                                        'gamma': [2, 3, 4, 5, 6, 7, 8, 9, 10],\n",
       "                                        'max_depth': [2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                      10],\n",
       "                                        'min_child_weight': [1, 2, 3, 4, 5, 6,\n",
       "                                                             7, 8, 9, 10]})"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_xg.fit(X_train_pr, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_pr = preprocessor.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters of the best model:\n",
      "{'min_child_weight': 9, 'max_depth': 5, 'gamma': 3, 'eta': 0.17, 'colsample_bytree': 0.7999999999999999, 'booster': 'gbtree'}\n",
      "XGBOOST Train accuracy: 0.99 \n",
      "XGBOOST Test accuracy: 0.89 \n",
      "---------------------------\n",
      "XGBOOST Train RMSE: 0.05 \n",
      "XGBOOST Test RMSE: 0.15 \n"
     ]
    }
   ],
   "source": [
    "#Predicting and accuracy results\n",
    "print(\"Parameters of the best model:\")\n",
    "print(gs_xg.best_params_)\n",
    "print(\"XGBOOST Train accuracy: %.2f \" % (gs_xg.score(X_train_pr,y_train)))\n",
    "print(\"XGBOOST Test accuracy: %.2f \" % (gs_xg.score(X_test_pr,y_test)))\n",
    "print('---------------------------')\n",
    "print(\"XGBOOST Train RMSE: %.2f \" % logrmse(gs_xg.predict(X_train_pr),y_train))\n",
    "print(\"XGBOOST Test RMSE: %.2f \" % logrmse(gs_xg.predict(X_test_pr),y_test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy and RMSE comparison of different methods\n",
    "acc,err = [],[]\n",
    "for m in [gs_lr_wpca,gs_lr_wop,gs_rf,gs_gb,gs_xg]:\n",
    "    if m == gs_xg:\n",
    "        acc.append(m.score(X_test_pr,y_test))\n",
    "        err.append(logrmse(m.predict(X_test_pr),y_test))\n",
    "    else:\n",
    "        acc.append(m.score(X_test,y_test))\n",
    "        err.append(logrmse(m.predict(X_test),y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkFUlEQVR4nO3debhdVX3/8ffHMMgUxjgwhGAZNCogBJCCDBaVyA9xrghiVaS0WkFrFasVFFsRsXUARYpMVcRaQQYZ1D4ig6AkgCAoGhnDUBJkFGT8/P5Y+4bD9Sb3JPeevfc55/N6nvPk7umslXC/fPdeew2yTURERNs8q+kKREREjCUJKiIiWikJKiIiWikJKiIiWikJKiIiWikJKiIiWikJKiIiWikJqkaSLpR0r6QVm65LxCCQdLOk3XpcxoWS/iTpIUkLJZ0u6fkdxw+TZEkfGHXdwdX+wzr2/bOkm6rvmi/pO4spZ+Rzdi//bm2XBFUTSTOAVwAGXldjucvVVVbEAHu/7VWBjYFVgaNGHf8t8M5R+/ar9gMg6Z3AO4Ddqu+aBfzvWOV0fPaczL9Ev0mCqs9+wOXASXT8IkvaoLojWyDpHklHdxx7r6RfS3pQ0vWStqr2W9LGHeedJOkz1c+7VHdmH5V0F3CipDUlnVOVcW/18/od168l6URJd1THv1/t/5WkPTvOW766g9yyR/9GEZNC0oqSvlj9Tt9R/bxix/GPSLqzOrb/6JhaHNv3Ad8Hthx16ApgZUkvrr7/xcBK1f4R2wAX2P599V132T5uAn/NgZcEVZ/9gG9Vn9dIeq6kKcA5wC3ADGA94DQASW8BDquum0p56rqny7KeB6wFbAgcQPnvfGK1PR14BDi64/z/AlYGXgw8B/iPav8pwL4d570WuNP21V3WI6IpHwdeTkkkWwDbAp8AkLQ78CFgN8oT0c7dfqmktYE3AvPGOPxflHiFchN6yqjjlwP7SfonSbOq+I8lsZ1Pjz/AjsDjwDrV9m+ADwLbAwuA5ca45gLgoMV8n4GNO7ZPAj5T/bwL8Bjw7CXUZ0vg3urn5wNPAWuOcd66wIPA1Gr7f4CPNP3vmU8+Ix/gZkqT2ej9vwde27H9GuDm6ucTgM92HNt4dEyN+q4LgYeB+6vzrgamdxw/DPgm5ebvVmD56s8Nqv2HdZy7D/Bj4I+UG85Dxijnvo7P4U3/Gzf5yRNUPd4J/ND2wmr71GrfBsAttp8Y45oNKEG2LBbY/tPIhqSVJX1d0i2SHgAuAtao7uA2AP5g+97RX2L7DuBS4E2S1gBmU54AI9puXUrLxIhbqn0jx27rONb58+J8wPbqwObAmsD6o0+wfSvlyerfgN/Z/rPvtf0t27sBawAHAp+W9JpR5azR8fmXLuo2sJKgekzSSsBbgZ0l3VW9F/ogpdnh/4Dpi+nIcBvwF4v52ocpTXIjnjfq+Ogp6v8R2AzYzvZUYKeR6lXlrFUloLGcTGnmewtwme3bF3NeRJvcQWnSHjG92gdwJ89MMBt0+6W2rwU+AxwjSWOccgol3kY3743+nsdtfxe4BnhJt+UPmySo3ns98CQwk9K0tiXwIuDi6tidwBGSVpH0bEk7VNcdD3xY0tYqNpY0EnBXA2+XNKVqTx+vDX01ynun+yStBRw6csD2ncB5wFerzhTLS9qp49rvA1sBBzFO0EU0ZPkqdkY+ywHfBj4haZqkdYBPUprbAP4beJekF0lauTq2NE6mvKsdqzfud4BXV2U8g6S/kbSHpNUkPUvSbMp7358vZflDIwmq994JnGj7VpdeO3fZvovSSWFvYE9KG/itwHzgrwGqu6t/pTQHPkhJFGtV33lQdd19lDbt749Thy9SehQtpLyoPX/U8XdQ3pH9BrgbOHjkgO1HgO8BGwGnd//XjqjNuZQbsJHPYZSnnDmUJ5RrgSurfdg+D/gy8BNKk9xl1fc82k1hth+rrv+z5jfbj9j+cRU3oz0A/DMl1u8DjgT+zvYlHeccPWoc1Nxu6jSoVL2ci1gsSZ8ENrW977gnR/QZSS8CfgWsuJj3wdGQPEHFElVNgu8BMl4jBoakN0haQdKawOeAs5Oc2icJKhZL0nspnSjOs31R0/WJmER/Sxni8XvKO+K/a7Y6MZY08UVERCvlCSoiIlqpsYlE11lnHc+YMaOp4iO6Nnfu3IW2pzVdj6WVGIt+sbgYayxBzZgxgzlz5jRVfETXJN0y/lntkxiLfrG4GEsTX0REtFISVEREtFISVEREtFISVEREtFISVEREtFISVEREtFJj3cyj/WYc8oNayrn5iD1aWf6gy79vtF0S1GIkeCMimpUmvoiaSNpd0g2S5kk6ZIzj+0i6pvr8TNIW3V4bMYiSoCJqIGkKcAwwm7K68t6SZo467SZgZ9ubA4dTLXHS5bURAycJKqIe2wLzbN9Yrch6GrBX5wm2f2b73mrzcmD9bq+NGER5B9VidbwHyzuw2qxHWVtrxHxguyWc/x7gvKW9VtIBwAEA06dPX9a6RrRCnqAi6qEx9o25GJukXSkJ6qNLe63t42zPsj1r2rS+m4A94hnyBBVRj/nABh3b6wN3jD5J0ubA8cBs2/cszbURgyZPUBH1uALYRNJGklYA3gac1XmCpOnA6cA7bP92aa6NGER5goqoge0nJL0fuACYApxg+zpJB1bHjwU+CawNfFUSwBNVc92Y1zbyF4moURJURE1snwucO2rfsR0/7w/s3+21EYMuTXwREdFKXT1BSdod+BKleeF420eMOr468E1gevWdR9k+cSIVy1RDERHDbdwnqC5Hsb8PuN72FsAuwBeql7kRERHLpJsmvm5GsRtYTeXN7qrAH4AnJrWmERExVLpJUGONYl9v1DlHAy+ijM24FjjI9lOTUsOIiBhK3SSobkaxvwa4GlgX2BI4WtLUP/si6QBJcyTNWbBgwVJWNSIihkk3CaqbUezvAk53MY8yK/MLR39RpmGJiIhudZOguhnFfivwVwCSngtsBtw4mRWNiIjhMm438y5HwB8OnCTpWkqT4EdtL+xhvSMiYsB1NQ6qixHwdwCvntyqRUTEMMtMEhER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpJUBER0UpdzcUXETFoZhzyg1rKufmIPWopZxDlCSoiIlopCSoiIlopCSoiIlopCSoiIlopCSoiIlopCSqiJpJ2l3SDpHmSDhnj+AslXSbpUUkfHnXsZknXSrpa0pz6ah3RnHQzj6iBpCnAMcCrgPnAFZLOsn19x2l/AD4AvH4xX7Or7YU9rWiN0s07xpMnqIh6bAvMs32j7ceA04C9Ok+wfbftK4DHm6hgRNskQUXUYz3gto7t+dW+bhn4oaS5kg5Y3EmSDpA0R9KcBQsWLGNVI9ohCSqiHhpjn5fi+h1sbwXMBt4naaexTrJ9nO1ZtmdNmzZtWeoZ0Rp5BxVRj/nABh3b6wN3dHux7TuqP++WdAalyfCiSa1h1C7v4ZYsCSqiHlcAm0jaCLgdeBvw9m4ulLQK8CzbD1Y/vxr4dM9qGkOj7QmyqwQlaXfgS8AU4HjbR4xxzi7AF4HlgYW2d16mGkUMINtPSHo/cAEljk6wfZ2kA6vjx0p6HjAHmAo8JelgYCawDnCGJCgxe6rt8xv4a0TUatwE1U33WElrAF8Fdrd9q6Tn9Ki+EX3L9rnAuaP2Hdvx812Upr/RHgC26G3tItqnm04S43aPpTRVnG77Vijt5JNbzYiIGDbdJKhuusduCqwp6cKqG+x+k1XBiIgYTt28g+qme+xywNbAXwErAZdJutz2b5/xRWX8xgEA06dPX/raRkTE0OjmCaqb7rHzgfNt/7GaiuUixmgzzxiNiIjoVjcJalH3WEkrULrHnjXqnDOBV0haTtLKwHbArye3qhERMUzGbeLrpnus7V9LOh+4BniK0hX9V72seEREDLauxkGN1z222v488PnJq1pERAyzzMUXERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVERGtlAQVURNJu0u6QdI8SYeMcfyFki6T9KikDy/NtRGDKAkqogaSpgDHALOBmcDekmaOOu0PwAeAo5bh2oiBkwQVUY9tgXm2b7T9GHAasFfnCbbvtn0F8PjSXhsxiJKgIuqxHnBbx/b8at+kXivpAElzJM1ZsGDBMlU0oi2SoCLqoTH2ebKvtX2c7Vm2Z02bNq3rykW0UVcJqtsXtJK2kfSkpDdPXhUjBsJ8YIOO7fWBO2q4NqJvjZugun1BW533OeCCya5kxAC4AthE0kaSVgDeBpxVw7URfWu5Ls5Z9IIWQNLIC9rrR533D8D3gG0mtYYRA8D2E5LeT7mBmwKcYPs6SQdWx4+V9DxgDjAVeErSwcBM2w+MdW0jf5GIGnWToMZ6Qbtd5wmS1gPeALySJKiIMdk+Fzh31L5jO36+i9J819W1EYOum3dQ3byg/SLwUdtPLvGL0sMoIiK61M0TVDcvaGcBp0kCWAd4raQnbH+/8yTbxwHHAcyaNavbHkwRETGEuklQi17QArdTXtC+vfME2xuN/CzpJOCc0ckpIiJiaYyboLp5udvjOkZExBDq5glq3Je7o/b/zcSrFRERwy4zSURERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUVERCslQUXURNLukm6QNE/SIWMcl6QvV8evkbRVx7GbJV0r6WpJc+qteUQzlmu6AhHDQNIU4BjgVcB84ApJZ9m+vuO02cAm1Wc74GvVnyN2tb2wpipHNK6rJ6gu7vz2qe74rpH0M0lbTH5VI/ratsA82zfafgw4Ddhr1Dl7Aae4uBxYQ9Lz665oRFuMm6A67vxmAzOBvSXNHHXaTcDOtjcHDgeOm+yKRvS59YDbOrbnV/u6PcfADyXNlXTA4gqRdICkOZLmLFiwYBKqHdGcbp6gxr3zs/0z2/dWm5cD609uNSP6nsbY56U4ZwfbW1FuFN8naaexCrF9nO1ZtmdNmzZt2Wsb0QLdJKhu7vw6vQc4b6wDubuLITYf2KBje33gjm7PsT3y593AGZQbx4iB1k2C6ubOr5wo7UpJUB8d63ju7mKIXQFsImkjSSsAbwPOGnXOWcB+VW++lwP3275T0iqSVgOQtArwauBXdVY+ognd9OLr5s4PSZsDxwOzbd8zOdWLGAy2n5D0fuACYApwgu3rJB1YHT8WOBd4LTAPeBh4V3X5c4EzJEGJ2VNtn1/zXyGidt0kqEV3fsDtlDu/t3eeIGk6cDrwDtu/nfRaRgwA2+dSklDnvmM7fjbwvjGuuxFIz9gYOuMmqC7v/D4JrA18tbrLe8L2rN5VOyIiBl1XA3W7uPPbH9h/cqsWERHDLFMdRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREKyVBRUREK3WVoCTtLukGSfMkHTLGcUn6cnX8GklbTX5VI/rbROJovGsjBtG4CUrSFOAYYDYwE9hb0sxRp80GNqk+BwBfm+R6RvS1icRRl9dGDJxunqC2BebZvtH2Y8BpwF6jztkLOMXF5cAakp4/yXWN6GcTiaNuro0YOMt1cc56wG0d2/OB7bo4Zz3gzs6TJB1AuTMEeEjSDUtV2/GtAyxcmgv0uZQ/yZquQy/K33BZK9NhInHUzbVAYmwIym9DHWqLsW4SlMbY52U4B9vHAcd1UeYykTTH9qxefX/Kb38dmi5/CSYSR13FFyTGBr38NtShzvK7SVDzgQ06ttcH7liGcyKG2UTiaIUuro0YON28g7oC2ETSRpJWAN4GnDXqnLOA/apeSC8H7rd95+gvihhiE4mjbq6NGDjjPkHZfkLS+4ELgCnACbavk3RgdfxY4FzgtcA84GHgXb2r8hL1rGkj5Xet6To0Xf6YJhJHi7u2gb8GNP/vO+zlQ/N1qK182WM2ZUdERDQqM0lEREQrJUFFREQrJUFFREQrddPNPJZA0pqUqWmePbLP9kU1lb2D7UvH2xfRzxJjw6uvO0lI2gT4LGV+ss5f3hfUVP7+wEGUcSlXAy8HLrP9yprKv9L2VuPt63EdPjTG7vuBubavrqH8t9j+7nj7YtkkxhJjTcZYvz9BnQgcCvwHsCulW+5Yo+575SBgG+By27tKeiHwqV4XKml74C+BaaN+eadSuiHXaVb1Obva3oMybudASd+1fWSPy/8YMDpQxtoXyyYxlhhrLMb6PUGtZPt/Jcn2LcBhki6mBFQd/mT7T5KQtKLt30jarIZyVwBWpfz3W61j/wPAm2sov9PawFa2HwKQdCjwP8BOwFygJ8EjaTZlzNB6kr7ccWgq8EQvyhxSibHEWGMx1u8J6k+SngX8rhrIeDvwnBrLny9pDeD7wI8k3UsNU9DY/inwU0knVf/ToPp3WNX2A70uf5TpwGMd248DG9p+RNKjPSz3DmAO8DpKkI54EPhgD8sdNomxxFhjMdbv76C2AX4NrAEcTsnsn6+WKqi7LjsDqwPnV0si1FHmqcCBwJOUX6DVgX+3/fk6yq/q8C/AG4Azq117Uqbh+QJwnO19elz+8rYfr35eE9jA9jW9LHOYJMYSY03GWF8nqKZV86VdZ/vBans1YKbtn9dU/tW2t5S0D7A18FHKi9PN6yi/ox6zgB0o7yYusT2nxrIvpNzhLUd5ib4A+KntsV4sR59JjC2qx1DGWF+Pg5L0o+rxf2R7TUkX1FiFrwEPdWz/kXpXE15e0vLA64Ezq7ucJu44rqK8MD0duFvS9BrLXr1qcnkjcKLtrYHdaix/oCXGEmM0GGP9/g5qHdv3jWzYvldSne3jcscjqO2nJNX5b/p14Gbgl8BFkjakvMStjaR/oLww/z9KM4goAVzXHeZyKqvOvhX4eE1lDpPEWGKssRjr9wT1lKTptm8FqH556ry7uVHSB3j6ju7vgRvrKtz2l4HO3jW3SNq1rvIrBwGb2b6n5nJHfJoyy/eltq+Q9ALgdw3VZRAlxhJjjcVYX7+DkrQ7Zer3n1a7dgIOsF1LE0R1J/llYGTQ4I+Bg23fXVP5zwX+DVjX9mxJM4HtbX+jjvKrOvwEeJXtdO0eQImxxFiT+jpBAUhahzK6HMpgvoVN1qdOks6jDKT8uO0tqqaPq2y/tMY6fAPYDPgBsKjLq+1/r6n89YGvUF4gG7gEOMj2/DrKHwaJscQYDcVYX3eSqOxEubvaFXhFnQVLeoGksyUtkHS3pDOrx9+6rGP7v4GnoCxsR2mjrtOtwI8oAxtX6/jU5URKl9t1gfUoo+1PrLH8YZAYS4w1EmN9/Q5K0leBjYFvV7v+VtJutt9XUxVOBY6hjFGAshT3t4Htair/j5LWpnonUHXJvb+msgGw3fNpZ8YxzXZnsJwk6eCmKjNoEmOJMRqMsb5OUMDOwEtGevlIOhm4tsbyZfu/Ora/WY22r8uHKHc2fyHpUmAaNU3DIumLtg+WdDZjvDS3/bo66gEslLQvT/8PdG+gqZfJgygxlhhrLMb6PUHdQJkG5JZqewOgzlkEfiLpEOA0yi/QXwM/kLQWgO0/9KpgSVMo//PYmdI+LeCGkRHfNRj5n8ZRNZW3OO8GjqZMZgpwabUvJkdiLDHWWIz1dScJST+lzHT8i2rXNsBlwMPQ+zsMSTct4bDd4yUJJF1oe5deltFFHQ6y/aXx9kV/SowlxprU7wlq5yUdryZ8HFiS/pUyN9h3KCPsAbB9ZY11GGu9nKtsv6ym8tOLr4cSY4mxJmOsrxPUsKvGR4xm17CYm6S9gbcDOwIXdxyaCjxhu5apUCT9iPIifaQ5ZF9gH9uvqqP8GGyJsWZjLAkqlkk1o8BGlNVWD+k49CBwTV2DClVN5jnevoh+kxgbjHFQQ0vS6pL+XdKc6vMFSavXUbbtW2xfSJk08uKqqedOytLcda64ulDSvpKmVJ99SS++mCSJMaDBGOvrJyhJr6SMbH+4wTq8jjKQEcoU9Gcv6fxJLvt7wK+Ak6td7wC2sP3GGuswlzJ4c03gcsoCZw+7x2vUdJQ/ndLDaPtq16WU9vFbFn9VdCsxlhhrMsb6PUGdQpmC5R5KG+3FlLVS7q2p/M8C2wLfqnbtDcyx/bGaym+8eWvkBa7KjMsr2T6yzhe40VuJscRYk/q6ic/2frY3Bd4EzKeMOF9QYxX2oEzieILtE4Ddq311eUTSjiMbknYAHqmx/KpYbQ/sQ5krDGocX9eCqXAGWmIsMdZkjPX1QN2qLfQVwEuBhZTH0IuXeNHkWwMYGSxYS9t0hwOBUzraxO8F3llzHQ4GPgacYfu66hd3rJ5PvdL0VDgDLTGWGKPBGOv3Jr6FwO+BY4Gf2L655vL3Bo6g/LKI0k7+Mdun9bjcg2x/SdIOti+VNBXAZdXLRqgsxW3bD4178uSW+3Pb243ad7ntly/umuheYiwx1mSM9XWCApD0Ysov7Y7AJpSpSN7R4zKXG+niqbLS5DaU4Pm57bt6WXZV5tW2txxrAF/dJL0UOAVYi/JvsADYz/Z1PS53rerHjwD38cypcFa0fXgvyx8mibHEGA3FWL838U2lzBO2ITCD8vj/VA1FXy5pPnA+cL7ts2oos9OvJd0MTJPUOS+aKHdYdS0FDWVJ7A/Z/gmApF2A/wT+ssflzqUEy0h327/tOGYgCWoSJMYSYzQYY339BFX94lxSfS6qc3qbahDdbMpL2/WqOpxH6Qb76JKunaTyn0dZhvnP5kKrs4u1pF/a3mK8fdGfEmOJsSb1dYJqC0nLU14k7w7sAiywXWdPo8ZIOgO4kmdOgzLL9utrKn+s8Sj3A9e6pmXBo/cSY8MZY0lQE7C4QYyS1rN9e0PVqpWkNYFPUd5PCLgIOKzGcTI/oAwgHOnVtAtlMOOmwKf9zLWEos8kxoY7xpKgJmAxgxgvtn1fk/VqQvWu4qkGehidDexv+/+q7ecCXwP2pzRJvaTO+sTkSow9bRhjrK8H6jZtMYMYFzZbq3pJeqmkqyirrF4naa6kOpPCjJHAqdwNbOqykF1dC8tFjyTGhjvG+r0X38g6JTtSehbVuhZQ04MYJc0CPk7pYbUc7elhdBy972E04mJJ5wDfrbbfDFwkaRVK19iYgMRYYowGY6yvm/jU8FpALRjEeAPwT5Q7q0Vdf4eph1F1J7kZT7fPXwIsdJkFOiYoMZYYazLG+j1BtWEix9oHMXaUfYntHcc/s6d1aLqH0a+qso8EVgI+V5W//RIvjK4kxhJjTcZYv7+DanQtoAYHMY44VNLxkvaW9MaRT43lA7wbmAacDpxR/fyuGsvfDtgA+BnwC+AOytLUMTkSY4mxxmKsr99BUf7DHQ38R7V9abWvLpd0fI6ucxBj5V3AC4HleTpoTflFrkXV1fUDKpNpPmX7wbrKrjxOmV16JeDZwE226/wf2KBLjCXGGouxvm7iG3aSrrX90obrsA1wArBatet+4N2259ZU/i+BMynTrqxNeaH8uO0311F+DLbEWLMx1tdNfJKOlDRV0vKS/lfSwqoJYlhcLmlmw3X4BvD3tmfYngG8DzixxvLfY/uTth+3fZftvSjBFJMgMZYYo8EY6+snKD094/AbgNcDH6T09Bn4OaoAJP0a+AvgJuBRGugCK+lS2zuMty/6U2IsMdakfn8HtXz152uBb9v+g6QlnT9odm+qYEkjSxD8QtLXKQuYjUzFf2FT9YpJlxhrSGKs/xPU2ZJ+Q3mB9/eSpgF/qqvwpgcx2r5F0haUgYxQpoD5ZR1lA18YtX1ox8/9+1geoyXGEmON6esmPlg0keIDtp+UtDIw1TUsaFaV3fQgxoOA9/J0j6I3AMfZ/kod5cdwSIwlxprS1wlK0lsoi5k9KOkTwFbAZ2xfWVP5jQ5iVFmrZ3vbf6y2VwEuq3kalhhgibHEWJP6uhcf8C9V4OwIvAY4mTLLbl0aHcRIeWH7ZMf2kzy9+mXEZEiMJcYa0+8JauQXZw/ga7bPBFaosfx3A28F7qo+b6beQYwnAj+XdJikwyhrtHyjxvKRtGI3+6JvJcYSY43p9ya+c4Dbgd2ArSkvcn8xLF1gYVFPn0ULmdm+qubyr7S91Xj7oj8lxhJjTer3XnxvpXQDPcr2fZKeT5l5uBaSjgQ+Qwna84EtgINtf7PH5a7VsXlz9Vl0zGWdlp6S9DxgPWAlSS/j6WaPqcDKvS4/apMYS4w1pq+foEZIeg5ljigAbN9aU7mNDGKUdBOlm6koE2neW/28BnCr7Y16WX5Vh3cCfwPMAuZ0HHoQOMl2bXOVRe8lxhJjTejrJyhJr6OMFViXssrjdOA3wItrqkIjgxhHgkPSscBZts+ttmdTmmLqqMPJwMmS3mT7e3WUGfVLjCXGmtTXT1DVJIavBH5s+2WSdgX2tn1ATeUfQbmrewTYlnJ3dY7t7Woqf67trUftm2N7Vh3lV+WtSFmOewYdNzy2P11XHaJ3EmOJsSb1ey++x23fAzxL0rNclkTesq7CbR8CbE9ZvOtx4I/AXnWVT+mC+wlJMyRtKOnj1NsFF8qkkXsBT1D+/iOfGAyJscRYY/q6iQ+4T9KqwEXAtyTdTfmPWIuOQYxPdg5ipHSHrcPelOlPzqi2L6r21Wl9243NVxY9lxhLjDWm35v4VqHMCyZgH8pqm9+q7vjqKP8a25tXgxg/CxwF/HNdzQ9tIOk44Cu2r226LjH5EmPNG+YY6+sE1TRJV1Xt8p8FrrV96si+msrfFPgwf942/co6yq/qcD2wMQ0uRxCDKzE23DHWlwlK0oOMPZvvyH+4qTXVo9FBjNUL7GOBuXRMx+KaVtqs6rDhWPtt31JXHWLyJcYWlZ8Ya1BfJqi2UJnZeXfKnd3vqkGML7X9w5rK/7MeRk2oml82sX2iynIMq9q+qel6Rf9LjC2qx1DGWBLUJGhwEONhlLEpZ1Ae/UfK7/ko9446HEoZSLiZ7U0lrQt810Ow2mfUJzE2nDGWBDUBixvEaLuWQYzVaPfRbPsFdZRf1eFq4GXAlSPvBUZebNdVhxhcibHhjrF+72betMOBlzNqEGNdhdcx3UoXHrNtSYZFvb4iJktibIhjLAlqYh63fY+kRYMYJX2uzgpIegkwk2c2f5xSYxX+W9LXgTUkvZeyFMJ/1lh+DLbE2BDHWBLUxDQ9iPFQYBdK8JwLzAYuAWoLHttHSXoV8ACwGfBJ2z+qq/wYeImxIY6xvIOagBYMYryWsvzAVba3kPRc4Hjbe9ZRfkSvJcaGW56gJsB253xYJzdQhUdsPyXpCUlTKS+Ra3l5K+kS2zuOMV6m1nEyMdgSY8MdY0lQy6AtgxiBOZLWoLRHzwUeAn5RR8G2d6z+XK2O8mK4JMYSY5Amvr4lSZRJJG+rtmcAU21fU1P5ay3peJ3jRCJ6ITHWvCSoPtbkKHe1YMXRiF5LjDWr39eDGnaXS9qmiYJtb1QNVrwA2NP2OrbXBv4fMPBLUcfQSIw1KE9Qfaya5XhT4BbKAma1z3I81h2mal5xNKJXEmPNSieJ/ja76QpQrTgKfJPSHLEv9a84GtEribEG5QkqJqR6kXsosFO16yLgU8PwAjeiDsMcY0lQERHRSmniiwmp1qb5CPBinjlXWW0rjkYMsmGOsfTii4n6FvAbYCPgU8DNwBVNVihiwAxtjKWJLyZkpIdR5/o0kn5qe+em6xYxCIY5xtLEFxP1ePXnnZL2AO4A1m+wPhGDZmhjLAkqJuozklYH/hH4CjAV+GCzVYoYKEMbY0lQscwkTQE2sX0OcD+wa8NVihgowx5j6SQRy8z2k8Drmq5HxKAa9hhLJ4mYEEn/SllE7juUqWAAsH1lY5WKGCDDHGNJUDEhkn4yxm4PwxiNiDoMc4wlQUVERCulk0RMiKQPjbH7fmCu7atrrk7EwBnmGMsTVEyIpFOBWcDZ1a49KKPcXwh81/aRTdUtYhAMc4wlQcWESLoAeJPth6rtVYH/Ad5AucOb2WT9IvrdMMdYupnHRE0HHuvYfhzY0PYjwKPNVClioAxtjOUdVEzUqZRlsc+stvcEvi1pFeD65qoVMTCGNsbSxBcTJmlrYEfKctiX2J7TcJUiBsqwxlgSVEREtFLeQUVERCslQUVERCslQUVERCslQUVERCv9f4MXytjcZOg5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "modelnames=['lasso w pca','lasso w/o pca','random forest','gradient boosting', 'xgboost']\n",
    "fig,(ax1,ax2) = plt.subplots(1,2)\n",
    "ax1.bar(np.arange(len(modelnames)),acc)\n",
    "ax1.set_title('Accuracy')\n",
    "ax2.bar(np.arange(len(modelnames)),err)\n",
    "ax2.set_title('Log RMSE')\n",
    "ax1.set_xticks(range(len(modelnames)))\n",
    "ax2.set_xticks(range(len(modelnames)))\n",
    "ax1.set_xticklabels(modelnames,rotation=90)\n",
    "ax2.set_xticklabels(modelnames,rotation=90)\n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best model in terms of smallest log RMSE is  xgboost\n"
     ]
    }
   ],
   "source": [
    "print('Best model in terms of smallest log RMSE is ',modelnames[np.argmin(err)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best model in terms of highest accuracy is  gradient boosting\n"
     ]
    }
   ],
   "source": [
    "print('Best model in terms of highest accuracy is ',modelnames[np.argmax(acc)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test on Submission Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sub = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove columns that were mostly empty in training set\n",
    "df_sub.drop(columns=drop_cols,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_final_test = df_sub.iloc[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n"
     ]
    }
   ],
   "source": [
    "X_final_test_pr = preprocessor.transform(X_final_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = gs_xg.predict(X_final_test_pr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n",
      "/home/esin/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_bayes.py:331: RuntimeWarning: invalid value encountered in sqrt\n",
      "  y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))\n"
     ]
    }
   ],
   "source": [
    "predictions = gs_gb.predict(X_final_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission = pd.DataFrame(columns=['Id','SalePrice'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission['Id'] = df_sub['Id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission['SalePrice'] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission.to_csv('submission.csv',sep=',',columns=['Id','SalePrice'],index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1459, 2)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_submission.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}