BERT_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torch.backends.cudnn as cudnn\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from datasets import load_dataset\n",
    "from transformers import BertForSequenceClassification, Trainer, TrainingArguments\n",
    "\n",
    "from bert_utils import TextSentiment, compute_centers_cov, mahalanobis_score, metrics_eval\n",
    "\n",
    "imdb = load_dataset('imdb')\n",
    "yelp = load_dataset(\"yelp_polarity\")\n",
    "tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'tokenizer', 'bert-base-cased')\n",
    "\n",
    "trainset = TextSentiment(imdb['train'], tokenizer)\n",
    "ID = TextSentiment(imdb['test'], tokenizer)\n",
    "OOD = TextSentiment(yelp['test'], tokenizer)\n",
    "\n",
    "batch_size = 1\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)\n",
    "ID_loader = torch.utils.data.DataLoader(ID, batch_size=batch_size,shuffle=True, num_workers=2)\n",
    "OOD_loader = torch.utils.data.DataLoader(OOD, batch_size=batch_size,shuffle=True, num_workers=2)\n",
    "\n",
    "model = torch.hub.load('huggingface/pytorch-transformers',\n",
    "                       'modelForSequenceClassification', \n",
    "                       'bert-base-cased',\n",
    "                       output_attentions=False).cuda()\n",
    "#model = torch.load('./models/bert/IMDB_finetuned10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First computing class centers and covariance matrices\n",
    "\n",
    "model.eval()\n",
    "\n",
    "start_layer = 0\n",
    "layers = 13 # Number of layers\n",
    "num_classes = 2\n",
    "num_per_class = None\n",
    "\n",
    "centers_cov_matrices = [compute_centers_cov(trainloader, model, layer, num_classes,\n",
    "                                            num_per_class, quantity_target=None)\n",
    "                        for layer in range(start_layer,layers)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computing the inverses of covariance matrices. It may be necessary to manually change the value of eps\n",
    "\n",
    "eps = 0.0000001 # This tiny quantity added to all covariance matrices is here to prevent divisions by zero\n",
    "                # from happening\n",
    "try:\n",
    "    inv_cov_matrices = [torch.inverse(S+eps*torch.ones(S.size()).cuda()) for _, S in centers_cov_matrices]\n",
    "except RuntimeError:\n",
    "    eps = eps * 1.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building score dataset for logistic regression training\n",
    "\n",
    "X = []\n",
    "y = []\n",
    "\n",
    "quantity_target = 5000 # \n",
    "\n",
    "for data_in, data_out in tqdm(zip(ID_loader,OOD_loader)):\n",
    "    \n",
    "    txt_in, txt_out = data_in['input_ids'].cuda(), data_out['input_ids'].cuda()\n",
    "    \n",
    "    features_in = np.zeros((len(txt_in),layers-start_layer))\n",
    "    features_out = np.zeros((len(txt_out),layers-start_layer))\n",
    "    \n",
    "    for k in range(0, layers-start_layer):\n",
    "        \n",
    "        centers, _ = centers_cov_matrices[k]\n",
    "        inv_cov_matrix = inv_cov_matrices[k]\n",
    "        \n",
    "        scores_in = mahalanobis_score(txt_in, model, centers, inv_cov_matrix, layer=start_layer+k)\n",
    "        scores_out = mahalanobis_score(txt_out, model, centers, inv_cov_matrix, layer=start_layer+k)\n",
    "        \n",
    "        features_in[:,k] = scores_in.cpu().numpy()\n",
    "        features_out[:,k] = scores_out.cpu().numpy()\n",
    "        \n",
    "    X = X + list(features_in) + list(features_out)\n",
    "    y = y + [0]*len(scores_in) + [1]*len(scores_out)\n",
    "    \n",
    "    if len(y) >= quantity_target:\n",
    "        break\n",
    "\n",
    "np.save(f'./data/scores_IMDB_Yelp_{len(y)}',X)\n",
    "np.save(f'./data/labels_IMDB_Yelp_{len(y)}',y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 0\n",
      "0.5\n",
      "{'auroc': -0.0, 'auprIN': 29.856134510190106, 'auprOUT': 31.5250327984636, 'detection accuracy': 50.0, 'tnr at 95 tpr': 0.0}\n",
      "layer 1\n",
      "0.9091781061371993\n",
      "{'auroc': 90.91781061371995, 'auprIN': 91.96986247859621, 'auprOUT': 89.67054833162481, 'detection accuracy': 84.03392686891996, 'tnr at 95 tpr': 48.31636648394675}\n",
      "layer 2\n",
      "0.9484060083072358\n",
      "{'auroc': 94.84060083072357, 'auprIN': 95.66162989511476, 'auprOUT': 93.82085021915283, 'detection accuracy': 88.57306865090978, 'tnr at 95 tpr': 67.42364917776038}\n",
      "layer 3\n",
      "0.9620795878525085\n",
      "{'auroc': 96.20795878525085, 'auprIN': 97.02555204269537, 'auprOUT': 95.02661309443121, 'detection accuracy': 91.76703882963635, 'tnr at 95 tpr': 76.11589663273297}\n",
      "layer 4\n",
      "0.9488868726593078\n",
      "{'auroc': 94.8886872659308, 'auprIN': 95.87248539928626, 'auprOUT': 93.62058184082966, 'detection accuracy': 89.56860512840872, 'tnr at 95 tpr': 65.54424432263117}\n",
      "layer 5\n",
      "0.9329876147015153\n",
      "{'auroc': 93.29876147015153, 'auprIN': 94.6079212286858, 'auprOUT': 91.83005321122589, 'detection accuracy': 87.33040887556498, 'tnr at 95 tpr': 55.05090054815975}\n",
      "layer 6\n",
      "0.9217250800533497\n",
      "{'auroc': 92.17247599039807, 'auprIN': 93.50610961938004, 'auprOUT': 90.72684014043978, 'detection accuracy': 85.38124347295475, 'tnr at 95 tpr': 48.70790916209867}\n",
      "layer 7\n",
      "0.922695452790454\n",
      "{'auroc': 92.26954527904539, 'auprIN': 93.64508081738524, 'auprOUT': 90.55550140657462, 'detection accuracy': 85.42619244434684, 'tnr at 95 tpr': 52.07517619420516}\n",
      "layer 8\n",
      "0.8907375024891613\n",
      "{'auroc': 89.07375024891614, 'auprIN': 90.85332446816217, 'auprOUT': 87.25572721854186, 'detection accuracy': 81.60280860638339, 'tnr at 95 tpr': 40.328895849647616}\n",
      "layer 9\n",
      "0.8540535072043213\n",
      "{'auroc': 85.40535072043212, 'auprIN': 87.5088853208249, 'auprOUT': 83.63880831294557, 'detection accuracy': 77.807085673892, 'tnr at 95 tpr': 33.594361785434614}\n",
      "layer 10\n",
      "0.8330177727720645\n",
      "{'auroc': 83.30177727720645, 'auprIN': 84.9874814748647, 'auprOUT': 81.73362209034086, 'detection accuracy': 75.9836109135078, 'tnr at 95 tpr': 29.130775254502737}\n",
      "layer 11\n",
      "0.8341114030161912\n",
      "{'auroc': 83.41114030161913, 'auprIN': 84.74995840076186, 'auprOUT': 82.34778355774047, 'detection accuracy': 75.39559256766837, 'tnr at 95 tpr': 30.540328895849644}\n",
      "layer 12\n",
      "0.8189689781664534\n",
      "{'auroc': 81.89689781664534, 'auprIN': 83.63919563772266, 'auprOUT': 78.95939541053303, 'detection accuracy': 75.2798585708148, 'tnr at 95 tpr': 24.902114330462023}\n",
      "\n",
      "All layers logistic Regression\n",
      "{'auroc': 88.8143652302418, 'auprIN': 85.57496834947229, 'auprOUT': 89.15176713880989, 'detection accuracy': 82.4453777154269, 'tnr at 95 tpr': 52.85826155050901}\n"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "\n",
    "load_data = True # This flag is used to either load from disk or from memory.\n",
    "\n",
    "if load_data:\n",
    "    X = np.load(f'./data/scores_IMDB_Yelp_5000.npy')\n",
    "    y = np.load(f'./data/labels_IMDB_Yelp_5000.npy')\n",
    "    layers = 13\n",
    "    start_layer = 0\n",
    "else:\n",
    "    X = np.array(X)\n",
    "    y = np.array(y)\n",
    "    \n",
    "for k in range(layers-start_layer):\n",
    "    X[:,k] = X[:,k]/max(X[:,k].max(), -X[:,k].min()) # Normalizing Mahalanobis scores\n",
    "\n",
    "y = 1-y # Inverting ID/OOD convention so that the LR's coefficients are positive\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(np.array(X),y, test_size=0.5, shuffle=True)\n",
    "\n",
    "LR = LogisticRegression(max_iter=200000, verbose=0, fit_intercept=False)\n",
    "LR.fit(X_train[:,1:],y_train)\n",
    "\n",
    "y_pred = LR.predict_proba(X_test[:,1:])[:,1]\n",
    "\n",
    "\n",
    "# Converting results to the format used in the metrics_eval function\n",
    "for l in range(layers):\n",
    "    scores_ID, scores_OOD = [], []\n",
    "    for k in range(len(y_test)):\n",
    "        if not y_test[k]:\n",
    "            scores_OOD.append(X_test[k,l])\n",
    "        else:\n",
    "            scores_ID.append(X_test[k,l])\n",
    "        \n",
    "    print(f'layer {l}')\n",
    "    print(roc_auc_score(y_test,X_test[:,l]))\n",
    "    print(metrics_eval(scores_ID, scores_OOD))\n",
    "\n",
    "scores_ID, scores_OOD = [], []\n",
    "for k in range(len(y_test)):\n",
    "    if not y_test[k]:\n",
    "        scores_OOD.append(y_pred[k])\n",
    "    else:\n",
    "        scores_ID.append(y_pred[k])\n",
    "\n",
    "print('\\nAll layers logistic Regression')\n",
    "print(metrics_eval(scores_ID, scores_OOD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAHLCAYAAAC5/f4FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAASdAAAEnQB3mYfeAAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd7h0VXn38e9PEAsqKMWOIBZQNCrWaASNvRc0sUSxd9Ro8hIbD2IhGjtijA3UWGIlCqIBxY4oioqIivqIgkpTkWrhfv9Y+/AMw8xpe86Z8zDfz3XNNees3e7Zs2fPPWuvtXaqCkmSJM2Gy007AEmSJK0ekz9JkqQZYvInSZI0Q0z+JEmSZojJnyRJ0gwx+ZMkSZohJn+SJEkzxORPkiRphpj8SZIkzRCTP0mSpBli8idJkjRDTP42UkmOSrLiN2ZOUkmOWuntSH0luVeSryX5XXfcfnLaMa0FSdYnWT/tONaCJAd1x8b2047lsiLJum6f7j7tWFbKZfE1mvytgO4gGXz8NclZXcK2Z5JMO8Y5G9sXw1zSe1n6EI6zsb0309R9mR8C7AC8B9gX+NACy+w+4rNaSc5L8oMk+ye5xojl1o1ZbvBx1NAy64emX5TkD0mOTvK8JJdfwroHH+snsgM1cUlukuStSU5Mck6Sc5P8KMmBSW66iOXvkeTDSU5OckH3o+abSfZJcvUxy+w54hg5J8mvkhyR5OVJbjz5V9vfNBPzgf2252pve1o2nXYAl3H7ds+XB24EPBTYDbgt8OxpBbVEOwPnTTsIaQH3AK4IvKCqPrDEZX8BHNT9HWBr4D7A/wMemmTXqjpnxHJfBI4as871Y8rfBPwe2ATYDngY8Abg74EHjlnfrYAHA98Fhmszfz9mOxrt34D9gVNWciNJ9gJeT6tg+RLwaaCAXYGnA09N8s9V9eYRy14BeCfwWOB84DPAj4GrAHcH1gHPTvLwqvrSmBAGj5UrAdsCdwBeCrw4yVuAF1bVX/q/2plwAO3H5MnTDmRSTP5WUFWtG/w/yZ1pJ4JnJnldVf18KoEtQVWdOO0YpEW4Tvd86jKWXT/is7oZ8DXal/UebEgOBx01vNwivLGq1g9sZz/gOOABSXarqqMYSgC72ogHA8ctY3saUFW/Bn69kttI8jhakn8W8NDhBC3J39ESszcl+X1VvXdoFW+jJX7fBh5SVb8cWDbAs7r1H5rk9lX1wxFhjDxWktyNdiw/l/Zj6enLepEzpqrOAM6YdhyT5GXfVVRVXwVOpNUu7Do8Pckdknw0yW+S/CnJL5O8Pcl1LrWyEZJsluTZSQ5L8oskF3aXm49Ict+heXdPazN4A+AGQ5cJDhqY7xKXsLp4KsmDxsRwx276R4bKr5zk35Ic113+OCfJ15M8ajGvbRGvfX33uEqSN3T77vxuew/p5tk0yYuS/KS7jPLTJJeqgc2Gy4Hrktyp239/SPLHJJ9NctsxMWyR5NXdpZ25yzSfTXKPBbZx+ySHdu9VzV2CYOH35iFJ3p/kxwP79NgkeyW51Gd78LJKkqcl+X4X52+T/FeSLca8ruslefPAfjsryTFJXjpm3gOS/Kw7/s5M8r9JbjfP2zdSkkcm+VK378/v4v23tJqRS+xHNtSyf2FgX+2+1G3Oqao/0Wr2ALZZ7noWsZ2TBraz5H20XN2x+i9JPp92SfBPSU7v3qs7Ds179bRL4T9NRjdZSfLpbp/vOlS+6HNaNjTp2CzJy7rP0YVzx3xXvleSb3efrfO6z/whoz5jY+K81KXF7vNQ3bTtk3woyRndsf6tJA9YzLq7dV0VeGP376NH1cxV1ZeBx3T/vqFbZm75uwBPAH4HPGAw8euWrao6AHgtrSbwUjWH86mqLwD3Bv5Eq328zWKXTbJrksO78+DZ3XnxTgsss1O3X3/ZvZe/TfKBDF327j7Dj+/+/XnGNGtIco20c+wPu3PCH5IcmeRe88TwD908Z3Xv6fokH0x3Hk/7fntPN/t7csnz7fbdPGPb/CX5+26/zK3/x2lNRi51Ph04xge/iy7s9s+/p/3oXBXW/K2+uZPnny9RmDwBeAdwIfC/wC+BGwNPBh6Y5I5VtVCV8zVovwi/BvwfcDpwbdrlpMOSPKWq3tnNu572hfm87v83DqznuHm2cRDwVNoH9X9HTH9c93zwwGvbEvg8cGvar9l303543Bv4QJKbV9VLFnhti3F52uu+Bq3912bAo4CPdSeHZ9IufXyGtp8fAbwlyelV9eER67sD7TLREcBbaZfuHwbcNcm9upP44Gv8KnAz4Ju0/bk18Ejgc0meUVVvH7GNO3Xb+Aptv2xNu8SzmPdmf+Ai4Bu0y1hb0C4LvYmWSPzTmP30Gtq+/xTwOeBuwFO613f3wRm7E+Rnafv0S8DHgSt3r3MdsN/AvLfp1neNbpmPd6/nIcBXkjy0qg4bE9MlJHkVbb+cAXwAOAe4L/Aq4N5J7llVf2bDcbw7rUnFwWy45LqeZUprg7db9++3lruexW6ue/7zvHNN1s7AK2nv6aG0ZGM74EHAfZM8sKoOB6iq3yX5EC0puQftM3axJNejXSY/tqqOHShf7jntY7Tj9zO0GrLTuvKDaJ/n44H30i6JXge4S7f9I5a/O4D2Y+sY4GfA+2jH8T8AhyS5R5c4LWQP4OrAMVX12XEzVdXhSb5Je517sCH5eEr3/I6ulnKcf6edH+6RZIelXEWqqhOT/A+tdvFRtHPyvJL8LW3/bkb7XJ9Ea45wFO3cPmqZ+3TzXp52rjkJuB7tHHr/JHerqrlt70s7T/wNG5pGMPBMkht029se+DJwOLA58ADg8CRPq6p3DMwf2n59PO088nHad+L1aOe8H9E+2wd123kw7Xtj8Bw7b7OKJE+j1dSeC3yEdqzuTmsy8sAkd66qUev4APB3tGP8bOB+wL/SLs8/Yb5tTkxV+Zjwg9a2o0aU3xX4K+1keO2B8pvQfomdBFx3aJm7d8t8Yqj8qOFtAFcArjdiu1vQTphnAVcamraedtlrvtdy1FDZj7rXsNWI7Z8F/BbYdKD8oG49/zo0/xVpH+CLgFstct8e1a1r9xGvo2gnmSsMlP9dV34WLSnbcmDaDbv9/p2hde0+9x4Czx6a9uCu/CfA5QbK396Vvx3IQPmNgT90+2v7Mdt42pjXutB7s+OIssvREqAC7jA0be59OBnYbqB8U1oSUMDtB8o3A37elT96xLauP7SOk4ALgN2G5rsOLTn99eB7M8/rutNAnNca2sanumkvGlpm3ajjYoHtzL0H67vl19G+hA7o3t8LgVePWG5uW0cNLDf8uOOY43P7ofKb0r44Cth1TJx7dtMPWuxrW+g4op0Tth4x7/Vol85/OFR+2y6Gj86zP54yULbscxrwveHYungvon1ZbzIihq2Gy8bsi4OG3wdaMjH3WdxnaP57d+WHLXL97+rmf+Ui5n1lN+87B8p+2pXdcxHLf7Wb97FLPVaAJ3XzfXER2wntilUBDx6a9tyBfbf7QPnVaT8ozgBuNrTMzWk/5r690Hsz4vi4CPjHofItaQnb+cA1B8qf2q3vGGCLoWU24ZLfwXP7bc8x2547xgdf4w1o54izgZ2G5j+wm/+/xhzjxwLXGCjfvPus/JWBc95KPlZ8A7P4GPgwrOserwQ+TDsZXgQ8Z2j+N3Tz33/M+j4B/AW46vBBtISY/rnbxl2Hytez9OTvRV35s4bK9+jKXz9QtlUX+zfHrP9vumVes8jXMffh2X3E6yhGJ0Q/66bdfcS0L3TxbTJQtjsjErwRMezW/X952hf4Hwc/0APz79fN/7IR2/jOPK913vdmnuVuM7y9rvygrvzJI5Z5AkPJLvDwruyQRWxzLil+7Zjpc18S91vEut7RzfvUEdNuQjtB/myofN2o42KB7cy9B+Men2EoiRva1nyP5405Pt/IhhrTg2lfgmP3W7fsnkw4+Vtg/jd329tuqPybtNrJwYR8E1qN3tnAVQbKl31OYyjB6KZdrZv2VQZ+XC1jX8x9BrYfKNueDT8CRiWWvwDOWOT6D+vW9fRFzPt0hhJLWue6YiiZGLP8hxj6Ub3YY4VWU1rACYvYzp0Zkyh27/9Jw589NnzenzVmnXPHx80Gyi713gxMm/ue+MiY9c2df545UPb9ruzWi3iNc/ttzzHT1414jS/uyl41Yv6rd5+J87lkZcTcMX6PEcvs2017wHKP76U8vOy7svYZ+r+AJ1XVe4bK59pN7JbRbaO2pX3IbkL7xTBWkpsD/0KrZbw2rXZt0HUXEfdC3kv78no87XLonMd3zwcPlN2OFnslWTdiXZfvnneeQFy/r6qfjig/lTYEyKh9d0oX37W4dA/AL1fVRSOWOYp2SfDWtPZaO9EuhX61qs4aMf/ngZd08w87ZkTZoiTZivZe349Wi7n50Czj3utRlzHn2hYNDiEx1/brM4sIZ+4YvsGY93lueImdaV+Q85lrh3Spy0lV9eMkvwJ2SLJljb6kslRfrKrd5/7p9uvf0pKgLyV5SI2+XL1vLb0DxnNHlK2rqn1HlK+otA5oz6W9d9vSanoHXZdL9m48kNY04Ym0y+/Qjr3rAW+rS/aI7nNOu9RnoqrOTvIpWhOW45J8jHbp7xtVNanRCI6rqr+OKP8lG17PQuYu4VfPefsuP8ll5z6PXxyeUFV/TfIVYMehSXP762/GnA9u0j3vDJywiBjm1rfFmPXNtcvdGSDJ5sAuwG+r6juLWP9yzHee+l2S79C+h3ei9b4etNhz8Iox+VtBVRW4+EC8E+2SwH8m+UVVDR4wW3XP/7LAKq8y38S0htqfp72vR9La2ZxNd1mV9uvoCmNXsEhV9askRwL3TLJzVf0wyba0X5PHVdXggT732m7H/A3a531ti/SHMeV/AaiqUdPnhjq4/Ihpvx2zvt90z1sMPY9rozNXvuU861qSro3hN2lJ7TG0hPws2uvZkvalPu69HpUwze2HTQbK5uJdzLAYc+/zIxaYbzHv82L253bdfBMf6qSqzgQ+leR8Wvu2N7BwwrpYO1TV+iRXpH0m/xPYJ8nPqup9E9rGgpI8FPgo7TL9/9EuN55LO1fsTvtxM3z8fAh4HfCUJPt3P4ye1k0bbs/a55w27jPxD7S2VI9mQwefC5J8lDZsybjP62KNO5b+wuI7R84ds9stYt7rDS0D7bXv0C3/o2Usv1hzHW5OX8S8c5/Hhc6Hg+be/6eMmDZosef9ufXds3sstL6lnLuWa9nn/TE/Wkedg1eMyd8qqKpzgSOSPJDWuPbgJDcd+MU6l5RsUVVn99jUS2hjOt2t2pARF0vyb7Tkb1IOpn0IHw/sTeu9timXrPWDDa/tDVX1zxPc/mq45pjya3XPfxh6vtaIeaHVwA7ON2g5v9qhNZrfgRG1T10PvFE1TEs1d4JaTG3x3Gt7cFWN6gi0FIP7c1RN7nz7c5K+0T3fJMkWY348LEtVXQAcndYL/0TgbUmOrKrlDFWzHPvRmqHctoaGCknydjZ0drlYVZ2f1vP2+cC9khxP+8H3jaEffNDjnFbdNbBR26drSpPk+rRalT1pHRe2p7XvnbavsKFjzIsXmHeuh/JXh5bfgREdawalDfI817P6q+Pmm8fduudvzDtXM/deLnQ+HLXM31TV95YS2AIxPLdGjI04wlLOXcs1eJ76wYjpq3WeWhaHellF3YfgHbRfbM8fmHR099z35HUj4KzhxK9zqZN5568s75fGx2m1io9NG1bk8bRfLsMD7B5Dq01YCyfmpbpLRgyZQqsZAZi7nPAjWludW2X0yPtzJ9oFe9UNme+9uVH3/LER08a910s1d1zed965LjnvJN7nuf26+/CEJDeifX5+PqFLvvMZfC9X5FxZrUfnq2iX7Ffz0u+NaO29hhO/y9F6z47zNrpOSrQfIJtw6Vo/mOzxcClV9cuq+m9ah4yf0D6rWy2w2Gr4KC3xuH2SsTVU3bTb0zpFfHRg0txoDE9OMi7ZAnghrWb2iFrieLFJdqLV0BeXPl+PMnfeutR5JckmjD5elvP+z11yH3XOW9L6ugqX44FrJhnV3GYp2x5nvvPUlrSa/QuAUeMwTp3J3+p7Be2AeOFAonAArSH1G5LcZHiBtPGtFnPQrweukeSWQ8s/iXaSHOVMYJskV1pk/MDFv8L/h/bL6vm0BrmHVdVpQ/OdBvw3cNskL01yqdrmJDsm2WEp218lN6YND3OxJA+mnQRPorU5otqYcP9Nu+Tw8qH5dwT2or2/S72sN997s7573n1oe7emDZEyCZ/qtvOgjBiPMcngr+pDaLV0z0pyv1ErSxsz8cqL2O67u+eXJLl4jL3ui+Y/aOetdy3qFfQzV1P9var63Qpu5y20S2p7ZvVuvbUeuHEGxtvrhsbYhzaMz0hV9RNak5IH0Dos/J7WmW3YpM5pc/Nvk+QOIyZtDlyV9sPzT4td30rpajlf0P37ga5d5SV0w6bMJV3Pq6o/Diz/JTYMM/Ppbhid4eWfTrv8fQ5LrOFPshtthIXNaO00h2tsR/ka7QfuXbvz36Bnc+n2ftCGWPk9rUnD7UfEcblcesy8M7vnS10yr6pv0c63D0vyxFFBJrlF1/xozlwN4dszNOZet/1rDxSN3fY83k87xp/T/SgdtB+tk9L7q+rCJaxz1XjZd5VV1SndZZXn0sb1+bdq4y49kfal94Mkh9PGers87WD8O1rbjJ0WWP0baUneV7pxnP5AG6LhLrRfl3uMWOZIWlu8w5N8idZ1/btV9alFvJyDab/+Xz3w/yjPpiVSLwf+qWsg/Ftau5Odu+0/ijasyFpyOPC67tLcd9kwzt8FtI47g51B9qa9T8/uGrh/gQ3j/F2V1ot2qa9vvvfmvbT2VG9MG7X/J7R9/ABarew/LOP1XkJV/SnJI2hj932gG9PqaFonop1ptyTbtJv3z0keRhvf79AkX6MNv3AecP3uddyQdilk3gb6VfW1JK+hfT6O79p0nUurgdyFdmnstX1f34DthxqRX4PW4WNXWm+9cbdi3H1M43NonY/eOGbaJVTVeUn2p7UtfDnts7DS3kBrb/idrvPEn2m9Om9GS/ofOM+yB9IuS14TeMuoDhcTPKfNuS7tMvkPaTVRv6R9uT6AdtntzYNJ1DRV1bu7mp/XAF9OG0T4WLj49m53o10NeV5d+u4e0IYo2ZR2HPwoyWdon+/Nu2V3oSUrD6+qcZ0lbjVwbF6B9l7dgfb+XkS79dy/LvL1VFeB8H+0MVPnxvn7G9pxcDjt8v/gMmcm2YPWq/voro34D7ptb0drA78Vl+yQeCTtnPaO7jN/Du1zdEA3/dG0Nu3vSrt93jdoCeb1gFt2++VObBgX8p20777HAT9JcgjtmLsObbihd9OaEQB8nXZeel7a/bzn2je+ZVxzj67t7vNonR6/3X3nnk6rHLgTrTnH/xu/Z6dsNboUz9oDRo/zNzD9mrQvs3O55LhEt6B1d/8F7Yv+LFrV9dsZGqaEMUO90E6GR9OGHfk97Yt7rm3Mpbqy004obwN+Rfv1fIlhAhgx1MvQ8j/p5jkT2Gye+TajfYl+jQ3j3p1M+8A/j8WP03UU44d6WT/fMmOmHcSlh37YnQ1D9dyJNrjp2d0+/RxwuzHr2pI2+OrcGHG/p50w7zVi3ou3Mc9rXei9uRmtU89p3bF0LC0Z33543nGvdTHx0E7WB9KS8z917/U3gBePmHdb2uDTx9NOpud0++OjtLZZm457vSPW9Y+0RO+PtIT7B7R2VFccMe+6UcfFAuufe83DjwtpwwO9E7jpPNua77F+aJn14/Z9N/2KtMbpFwG3HJq256j3cwmvc/1wPAPrPa47ds6gfVHfYqF9Sbs0dno3z80X2Hbvc9rAZ+tltC//U7p1/bpb5lEscviXUZ8BxnxeFhPXAtvaifb5nWsWch4tAX4bixvK5V60gYN/1b3eP9A+4+sYMaTU0LEy+Di3W8cRtKYFN1rmcbQrLdH7Y/c4gnZ+HHu8dPt2btzMC2jn0RNptZsPGTH/P9Muk17I6M/RVWlDjR1LO7ecTzsvHUpLmjcfsc7H0Hoq/6GL4ee0KzW3GZrvPrQkcG74pYuPkwVe471o3wu/6+I+iZb4bzli3vmO8bn3bs/lvD9LfaTbqKROdzniCyxvKA/pMi3JDWlfcF+tqo2xLa8082zzJ0laihfSxok7YKEZJa1NtvmTJM0ryXa0Nlc3pg1l8l3aJUlJGyGTP0nSQm5I69h1Hq0d6zNq9N1vJG0EbPMnSZI0Q2zzJ0mSNENM/iRJkmaIyZ8kSdIMscPHCN2tYHajjSI/9VsGSZIkzWMz2t2Uvlhj7koyyORvtN1o9yqVJEnaWDyYdueneZn8jfZLgE9+8pPc6EbD92uWJElaO0466SQe8pCHQJe/LMTkb7Q/AdzoRjfi5je/+bRjkSRJWoxFNVWzw4ckSdIMMfmTJEmaISZ/kiRJM8TkT5IkaYaY/EmSJM0Qkz9JkqQZYvInSZI0Q0z+JEmSZojJnyRJ0gwx+ZMkSZohJn+SJEkzxORPkiRphpj8SZIkzRCTP0mSpBmy6bQDkGbd9nsfOu0QAFi///2nHYIkaRVY8ydJkjRDTP4kSZJmiMmfJEnSDDH5kyRJmiEmf5IkSTPE5E+SJGmGmPxJkiTNEJM/SZKkGWLyJ0mSNENM/iRJkmaIyZ8kSdIMMfmTJEmaISZ/kiRJM8TkT5IkaYaY/EmSJM0Qkz9JkqQZYvInSZI0Q0z+JEmSZojJnyRJ0gwx+ZMkSZohJn+SJEkzxORPkiRphpj8SZIkzRCTP0mSpBli8idJkjRDTP4kSZJmiMmfJEnSDDH5kyRJmiEmf5IkSTPE5E+SJGmGbDrtACRtPLbf+9BphwDA+v3vP+0QJGmjZc2fJEnSDDH5kyRJmiGXieQvya2SHJrk5CTnJzkrydeTPHbasUmSJK0ll5U2f1sCvwQ+CJwCbA48Bnhfku2r6hXTDE6SJGmtuEwkf1V1FHDUUPGnk+wAPBUw+ZMkSeIyctl3HmcAf5l2EJIkSWvFsmv+krwMeGdVnTpi2rWBp1TVy/sEt4yYLkdLaK8OPAK4N/DsBZbZFthmqHjHFQlQkiRpyvrU/O0DXG/MtOt001fbgcCfgdOANwB7VdXbF1jmmcDxQ49DVjJISZKkaenT5i/zTLsKLQlbba8C3glsCzwQOCDJ5lX1H/MscyDwkaGyHTEBlCRJl0FLSv6S3BK41UDR/ZLsNDTblWg9bX/aM7Ylq6qTgZO7fw9LAvDqJAdX1eljljmNVlN4sW45SZKky5yl1vw9lA2Xcwt42Zj5zgeesNygJugY4OnADYGRyZ8kSdIsWWry91/Ap2mXfI+hJXjHD81zIfDTqjq/f3i93Q24CPjZtAORJElaC5aU/FXVr4FfAyS5G3BsVZ2zEoEtRZL/As6mJaS/Bbam9fb9B+C14y75SpIkzZpld/ioqi9OMpCevk6rhXw87W4f5wDfBf6pqt4/zcAkSZLWkl53+Ojunfto4Aa0jh6DqqpWZby8qnoP8J7V2JYkSdLGrM8gz/8PeDVwAq2W7cJJBSVJkqSV0afm76nAW6vqOZMKRpIkSSurzx0+rgV8YlKBSJIkaeX1Sf6OxXvgSpIkbVT6JH//DLwgya6TCkaSJEkrq0+bv/cAWwHHJPkNcObQ9Kqqv+mxfkmSJE1Yn+TvTOCMSQUiSZKklddnkOfdJxiHJEmSVkGfNn+SJEnayPRK/pJsk+TVSb6e5CdJbt6VPy3JrScToiRJkiZl2clfkh1od/bYCyjghsAVusm37MolSZK0hvSp+XsN8HvgxsBdgQxM+wpw5x7rliRJ0gro09v374FnVNWpSTYZmvZr4Do91i1JkqQV0Kfm74rAWWOmbQ5c1GPdkiRJWgF9kr8fAfcYM+2uwPE91i1JkqQV0Oey7zuA1yc5FfjvrmyzJHsAzwSe3Tc4SZIkTVafQZ4PTHIr4A3A67rir9A6fryjqg6eQHySJEmaoD41f1TVU5O8G7g/cE3a7d4+XVVfm0RwkiRJmqxeyR9AVR0NHD2BWCRJkrTCvL2bJEnSDFlSzV+SnwEPrarvJvk57c4e41RV7dgrOkmSJE3UUi/7fhE4e+Dv+ZI/SZIkrTFLSv6q6gkDf+858WgkSZK0omzzJ0mSNEOWnfwleUKSdWOmrUvyuGVHJUmSpBXRp+ZvL+B3Y6ad0U2XJEnSGtIn+bsR4+/fewJw4x7rliRJ0gro2+Zvi3nKew8gLUmSpMnqk6B9H/hH4OMjpj2qmy5Jmsf2ex867RAutn7/+087BEmroE/N3wHAHkkOTnKHJNftng8CHg68ZSIRSpIkaWKWXfNXVR9IshPwb8BjByZdBLyiqv67b3CSJEmarF7t8qrqZUneDdwT2AY4HfhcVf1iEsFJkiRpsnp3yqiq9cA7+ociSZKkleYdPiRJkmbIkpK/JH9Ncvvu74u6/8c9/rIyIUuSJGm5lnrZ9+XArwb+rsmGI0mSpJW01OTvE8BZAFW1buLRSJIkaUUttc3fd4BbAiT5fDfUiyRJkjYSS03+LgQ26/7eHbjaRKORJEnSilrqZd+fAS9Icq3u/92TXG/czFU16tZvkiRJmpKlJn/7Ae8FHkzr7LH/PPMWsMky45IkSdIKWFLyV1UfTnIkcFPgy8CzgBNWIjBJkiRN3pKSvyQPAr5YVV9NcjBweFX9fGVCkyRJ0qQttcPHJ2i1fgC7YYcPSZKkjcpSk7/zgSt3f28PXGGi0SxTkrsneXeSE5Ocm+SUJIck2XXasUmSJK0lS+3w8UPglUk+0f3/6CR3GTNvVdUblh/akjwD2Ap4E60N4jbAC4Cjk9y7qj6/SnFIkiStaUtN/vYGPgy8htabd6955i1gtZK/Z1XVaYMFSQ4HTgJeBJj8zZjt9z502iEAsH7/+087BEmSLmGpvX2PBLZOcl3gl8BDgeNWIrClGE78urJzkpwAXH8KIUmSJK1JS635A6CqTkmyL/DNqjp1wjFNRJItgNtgrZ8kSdLFlpX8AVTVvnN/J7kpsDVwXFWdO4nAJuCtwObAK+ebKcm2tDaCg3ZcqaAkSZKmaam9fS8hyeOS/IrWyeJLdMPAJPcNHmYAACAASURBVPmfJE+ZQHzLjWs/4DHA86vq2AVmfyZw/NDjkJWNUJIkaTqWnfwleQRwEPBt4NlABiZ/G3hkr8iWKck+wEuAF1fVAYtY5EBgl6HHg1cuQkmSpOlZ9mVf4N+A91TVk5JsQrvMOueHwHN6RbYMXeK3DlhXVa9azDJdZ5HhnsKTD06SJGkN6HPZd2fgQ2OmnUUbd2/VJHkpLfF7xWB7REmSJG3Qp+bvPGCLMdOuC/yux7qXJMkLgJcDhwOHJrnj4PSqOnq1YpEkSVrL+iR/XwWeneRjI6btCRzVY91L9cDu+T7dY5jXcSVJkuiX/L0c+ApwDPAB2h09HtaN/3dX4Pb9w1ucqtp9tbYlSZK0Meszzt+3ktyX1lv2dV3xi4CfAPerquMnEJ/WEG+ZJknSxq9PzR9V9QVg5yQ7AtcEzqiqH08kMkmSJE1cr+RvTlX9FPjpJNYlSZKkldP3Dh87JnlfklOTXJjklCQHdzWBkiRJWmOWXfOXZCfg68AVgc8DpwLXod3Z4wFJ7lxVJ04kSkmSJE1En8u+rwLOBHavql/NFSa5Hi0ZfCXw8H7hSZIkaZL6JH+7AXsNJn4AVfWrJC8H3twrMknSmrJWevyDvf6lPvq0+bsyreZvlDOAK/VYtyRJklZAn+TvR8Bjxkx7FGB7P0mSpDWmz2XfNwPvTLIFcDDwa+DawGOBBwFP7h+eJEmSJqnPHT7eneSawEuAucYXAc4HXlxV75lAfJK0LGulfZpt0yStNX3v8PHqJAcCdwK2orUB/HpV/WESwUmSJGmyet/ho0v0Dp9ALJIkSVphy+7wkeQJSdaNmbYuyeOWHZUkSZJWRJ/evnsBvxsz7YxuuiRJktaQPsnfjYDjx0w7Abhxj3VLkiRpBfRJ/gC2mKe8d3tCSZIkTVaf5O/7wD+OmfaobrokSZLWkD7J3wHAHkkOTnKHJNftng8CHg68ZSIRSpIkaWL6DPL8gSQ7Af9Gu6vHnIuAV1TVf/cNTpIkSZPVd5DnlyV5N3BPYBvgdOBzVfWLSQQnSZKkyZrEIM/rgXf0D0WSJEkrrW9vX0mSJG1ETP4kSZJmiMmfJEnSDDH5kyRJmiHehUOSdJmz/d6HTjuEi63f//7TDkG6hN7JXzfW327A1sC7quo3Sa4D/K6qzu+7fkmSJE3OspO/JJsA/wXsCQQo4DPAb4C3A98BXtY/REmSJE1KnzZ/LwYeDfwLsAstAZzzGeA+PdYtSZKkFdDnsu+ewH5V9fquFnDQz4EdeqxbkiRJK6BPzd91ga+PmXYBcNUe65YkSdIK6JP8nQbccMy0mwK/6rFuSZIkrYA+yd9hwIuTXHegrJJsAewFfKpXZJIkSZq4Psnfy2htBk8APkbr7fsq4HjgisB+vaOTJEnSRC07+auq3wK3Az4I7Ar8FfgbWk/fv62qsyYSoSRJkiZmWb19k1yRVvP3sap6+mRDkiRJ0kpZVs1fVV0APB/YfLLhSJIkaSX1afP3QxzLT5IkaaPSJ/nbD3hJkh0nFYwkSZJWVp87fDwBuDLwwyTfA35N6/E7p6rqwX2CkyRJ0mT1Sf5uCfwJOAXYqnsMqkstIUmSpKladvJXVdtPMI6Ztf3eh047BADW73//aYcgSZJWQZ82f5IkSdrI9LnsS5LLA48D/p522fcM4Ajg/VX15/7hSZIkaZKWXfPX3cP368A7gAcCNwQeBLwL+FqSq00kwsXHc9Ukr0nyuSSnJ6kk61YzBkmSpLWuz2XfVwI3Bf6hqq5aVTeuqqsCj+zKXzmJAJdgK+CpwBWAT67ytiVJkjYKfS77PgR4WVV9ZLCwqj6aZDvgn4Hn9AluiX4BXL2qKsnWwJNXcduSJEkbhT7J3zbA98ZM+y6wdY91L1lVObSMJEnSAvokf6cAdwGOHDHtzsCpPda9apJsS0tkB3nXEkmSdJnUJ/n7MPCiJH8EDq6qM5NsBTwWeBHw+kkEuAqeCewz7SAkSVrr1srYtOD4tH30Sf7WAbcG/gN4bZK/dOsL8Nlu+sbgQOAjQ2U7AodMIRZJkqQV1ecOHxcC90lyb+ButN62ZwJHVtX/TSi+FVdVpwGnDZYlmVI0kiRJK6vXIM8AVfVZWk2fJEmS1rg+gzzfMckjx0x7ZJI7LD8sSZIkrYQ+NX+vAr4K/M+IaTcDngLcs8f6lyzJfYHNgavOxZFkj+7vw6rqvNWMR5Ikaa3pk/zdEnjdmGnfoPWiXW1vA24w8P8jugfADsD61Q5IkiRpLemT/G0O/GXMtIvYUPu2aqpq+9XepiRJ0sakz719f07r5TvK3Wi3W5MkSdIa0if5+xDw/CRPGCxMsifwPOCDPdYtSZKkFdAn+duf1uHjXUnOTfKTJOcC7+rKXz2JACVJkjQ5fQZ5/lOSewKPBu5Duz/uMcBngA9W1V8nE6IkSZImpdcgz12C977uIUmSpDWuzyDPV0xytaGyRybZP8k9+ocmSZKkSevT5u99wJvn/kmyF60TyL8Cn01yv56xSZIkacL6JH+3Bw4f+H8v4P3AlsDHgRf2WLckSZJWQJ/kbxvgFIAkOwA3BN5SVWfTevzu0j88SZIkTVKf5O88YIvu778DzgG+1f1/AXCVHuuWJEnSCujT2/f7wLOS/IJ2H98vVFV107YDftM3OEmSLuu23/vQaYdwsfX733/aIWgV9En+9gM+DRwH/AkY7OF7f+DbPdYtSZKkFdBnkOfPJ9kZ2BU4rqp+NjD587SkUJIkSWtI30GefwH8YkT52/usV5IkSSujT4cPSZIkbWRM/iRJkmaIyZ8kSdIMMfmTJEmaIctK/pJcMclTu96+kiRJ2kgsK/mrqguANwPbTjYcSZIkraQ+l31/BlxrUoFIkiRp5fVJ/t4E7J3kapMKRpIkSSurzyDPNwe2BtYn+Tzwa6AGpldVPbdPcJIkSZqsPsnfswf+ftiI6QWY/EmSJK0hfe7t6zAxkiRJGxkTOEmSpBnSO/lLcu8kr07yjiTbdWW3S7JN//AkSZI0Scu+7JvkysAhwN+zoaPH24CTgRcCv+yeJUmStEb0qfl7JXBb4OHAFkAGpn0OuEePdUuSJGkF9Ont+wjgpVX1iSSbDE07Gdiux7olSZK0AvrU/G0D/GDMtIuAK/VYtyRJklZAn+TvFOAWY6bdEvh5j3VLkiRpBfRJ/j4OvDjJrQfKKskNgOcDH+kVmSRJkiauT/K3L3AqcAzwLVqP3/cAxwOnAfv3jk6SJEkTtezkr6r+CPwt8FLgHOCnwHnAq4G7VtX5E4lQkiRJE9Onty9dgrc/1vJJkiRtFJZd85fkLUluOslgJEmStLL6tPl7HHBCkiOSPDhJFlxCkiRJU9Un+bsO8BzgWsAngPVJ9k6y9UQikyRJ0sT16fBxblUdWFW70G7ldiywH/DLJAclue2kgpQkSdJk9Kn5u1hVfb6qHgbsAHwN+CfgG0m+keSBk9iGJEmS+ptI8pfkSkmeDHwKuBvwQ9o4gJsAn0zy0klsR5IkSf30Sv6S7Jjk9bRbvf0n8CvgXlW1S1W9vKpuC/w7rW2gJEmSpqzPUC+fAX4EPAl4L3DTqnpgVR0xNOunADuBSJIkrQF9BnnekXYP3/dU1TnzzHc87VKwJEmSpqxPb9+bVNVbFkj8qKo/VtUXl7udxUpylSRvTHJqkguSHJfkH1d6u5IkSRuTXrd3W2M+DtwO2Bv4MfBo4INJLldVH5hqZJIkSWtEr+QvyV2BvYCdgSsNTa6q2rHP+pcQx/2AewKPrqoPdsVfSHID4LVJPlxVf12NWCRJktayPh0+7gIcCWxBS/5OpPX63Q74C/ClSQS4SA8FzgE+MlT+HtqdSO6wirFIkiStWX1q/valJVfPAP4MvKSqvp3klsDhtMuwq2UX4IdV9Zeh8u8NTP/aqAWTbAtsM1S8KjWWkiRJqy1VtbwFk98CewKfpdX03aGqvtlNewbwxKq63YTiXCiWHwM/q6r7DJVfGzgVeFFVvXrMsuuAfUZNO/7447n5zW8+4WglSdJK237vQ6cdwsXW73//FV3/D37wA3bZZReAXarqBwvN36fm78rAOVV1UZILueRYficCN+ux7uWYL4udb9qBXPpy8Y7AIb0jkiRJWmP6JH8nA9fs/j4BuD/wme7/3YAze6x7qc4EthpRfo3u+axxC1bVacBpg2VJJheZJEnSGtLn9m5HAbt3f78DeGaSI5McBrwE+OCY5VbC94Gdkwwns7fono9fxVgkSZLWrD7J3z7AmwGq6j+BFwJbAtsCr6AlgKvlE8BVgIcPlT+e1ubvG6sYiyRJ0pq17Mu+VXUGcMbA/68HXj+JoJYRy2eS/B/wtiRXA04CHgXcB3isY/xJkiQ1l6U7fDwMeCXwclpbvxOBR1XVh6YalSRJ0hqypOQvycuWMHtV1X5LjGfZunsMP7d7SJIkaYSl1vytW8K8Baxa8idJkqSFLSn5q6o+HUQkSZI0ZSZzkiRJM8TkT5IkaYaY/EmSJM0Qkz9JkqQZYvInSZI0Q5aU/CXZK8m2KxWMJEmSVtZSa/7eAGwPkOSvSW4/8YgkSZK0Ypaa/P0R2LL7OxOORZIkSStsqXf4OA54c5Ijuv9fkOS3Y+atqvJWa5IkSWvIUpO/5wHvAZ5Ku33bI+aZt/A+u5IkSWvKki77VtVxVXXrqtqMdtn3jlV1uTGPTVYmZEmSJC1Xn6FengD8dFKBSJIkaeUt9bLvxarqYIAkVwXuBGwFnAEcXVV/nEx4kiRJmqRlJ38ASV4I7ANcmXYZuIDzkuxTVa+fQHySJEmaoGUnf0keB7wG+AxwEHAqcB3g8cBrk5xeVe+bRJCSJEmajD41f88HPlBVjx0q/0iS93fTTf4kSZLWkD4dPnYC3j9m2vuBnXusW5IkSSugT/J3PnCNMdOu0U2XJEnSGtIn+fsysC7JdQYLk1wLeBnwpT6BSZIkafL6tPl7EfA14KQkRwK/Bq4N3B34M/Cw/uFJkiRpkpZd81dVPwBuBxzSPT+he/4kcPuqOmEiEUqSJGlieo3zV1U/Bh41oVgkSZK0wvq0+ZMkSdJGxuRPkiRphpj8SZIkzRCTP0mSpBli8idJkjRDTP4kSZJmSK/kL8lOST6Y5NdJ/pTkNl35PknuNpkQJUmSNCnLTv6S3Ar4JrAbcBSwycDkqwBP7xWZJEmSJq5Pzd/+wPeAGwH/BGRg2jG0u31IkiRpDelzh487A4+tqvOSbDI07bfAtXqsW5IkSSugT81fgD+NmXZ14MIe65YkSdIK6JP8fQ946Jhp9wGO7bFuSZIkrYA+l33fBHwgybnA+7qy7ZLcHXgisEff4CRJkjRZy07+qurDSXYE1gF7dcUfA/4C7FNVn+ofniRJkiapT80fVfWqJO8F7g1cEzgD+GxV/WISwUmSJGmyeiV/AFX1K+BdE4hFkiRJK2zZyV+S7Raap6pOXu76JUmSNHl9av7WA7XAPMPj/0mSJGmK+iR/T+TSyd/WwIOA6wGv6LFuSZIkrYA+vX0PGjPpdUk+Alx/ueuWJEnSyugzyPN8DgKevELrliRJ0jKtVPK3KbDlCq37EpJcNclrknwuyelJKsm61di2JEnSxmaiyV+SyyfZFdgX+O4k1z2PrYCnAlcAPrlK25QkSdoo9Rnq5SLG9/b9HW3g59XwC+DqVVVJtsbLzZIkSWP16e37ci6d/F1AGwLmsKr6Y491L1pVLTTcjCRJkjp9evuum2AcU5NkW2CboeIdpxGLJEnSSut9e7fLgGcC+0w7CEmSpNWwpOQvycuWMHtV1X5LXP/uwBcWOfutq+q4pax/jAOBjwyV7QgcMoF1S5IkrSlLrflbt4R5C1hS8gf8CHjKIuedyH2Dq+o04LTBsiSTWLUkSdKas6Tkr6pWalzAufX/GnjnSm5DkiRplq1oMidJkqS15TLR4SPJfYHNgat2RTdLskf392FVdd50IpMkSVpbeiV/Se4K7AXsDFxpaHJV1WoNmfI24AYD/z+iewDsQBt7UJIkaeYt+7JvkrsARwJb0JK/E4FTgO2AvwBfmkSAi1FV21dVxjzWr1YckiRJa12fNn/7Au8B7tP9/5Kq+jvgNsBVgI/3jE2SJEkT1if52wX4BBtu8bYJQFV9jzbEy1LGBJQkSdIq6JP8XRk4p6ouAi4Eth6YdiJwsz6BSZIkafL6JH8nA9fs/j4BuP/AtN2AM3usW5IkSSugT2/fo4DdgY8C7wAOTLIzrRbwXsDr+gYnSZKkyVrqvX23qarTu3/3Aa4BUFX/meTKwGNobQBfAbxykoFKkiSpv6XW/J2S5H+BdwGHV9UZcxOq6vXA6ycZnCRJkiZrqW3+PgLcF/g08Mskr0iyWgM5S5IkqaclJX9V9Rjg2sCzaAM6vwj4cZIvJHlskiuuQIySJEmakCX39q2qs6vqP6vqDsDNgTcAOwHvBX6T5G1JbjfhOCVJkjQBfYZ6oap+WFUvBK4HPITWA/iJwNFJvtc/PEmSJE1Sr+RvTlX9tar+F3gacEBXfPNJrFuSJEmT02ecPwCSbAI8CHgC7T6/mwLfo/UIliRJ0hqy7OQvyc1pl3gfS7u129nAO4F3VdWxkwlPkiRJk7TUQZ6vBjyalvTt2hV/iVbL99GqumCy4UmSJGmSllrz9xvgCsCvgf2Bd1fVTycelSRJklbEUpO/w2m1fJ+pqotWIB5JkiStoCUlf1X1sJUKRJIkSStvIkO9SJIkaeNg8idJkjRDTP4kSZJmiMmfJEnSDDH5kyRJmiEmf5IkSTPE5E+SJGmGmPxJkiTNEJM/SZKkGWLyJ0mSNENM/iRJkmaIyZ8kSdIMMfmTJEmaISZ/kiRJM8TkT5IkaYaY/EmSJM0Qkz9JkqQZYvInSZI0Q0z+JEmSZojJnyRJ0gwx+ZMkSZohJn+SJEkzxORPkiRphpj8SZIkzRCTP0mSpBli8idJkjRDNvrkL8ndk7w7yYlJzk1ySpJDkuw67dgkSZLWmo0++QOeAWwPvAm4H/BcYFvg6CR3n2JckiRJa86m0w5gAp5VVacNFiQ5HDgJeBHw+alEJUmStAZt9DV/w4lfV3YOcAJw/dWPSJIkae3a6JO/UZJsAdwG+MG0Y5EkSVpLLguXfUd5K7A58MqFZkyyLbDNUPGOKxGUJEnStK2p5C/J7sAXFjn7ravquBHr2A94DPCcqjp2Eet5JrDPooOUJEnaiK2p5A/4EfCURc578nBBkn2AlwAvrqoDFrmeA4GPDJXtCByyyOUlSdIas37/+087hDVrTSV/VfVr4J3LWbZL/NYB66rqVUvY5mnAcG/h5YQgSZK05l0mOnwkeSkt8XtFVe075XAkSZLWrDVV87ccSV4AvBw4HDg0yR0Hp1fV0VMJTJIkaQ3a6JM/4IHd8326xzCv4UqSJHU2+uSvqnafdgySJEkbi8tEmz9JkiQtjsmfJEnSDDH5kyRJmiEmf5IkSTPE5E+SJGmGmPxJkiTNEJM/SZKkGWLyJ0mSNENM/iRJkmaIyZ8kSdIM2ehv77ZCNgM46aSTph2HJEnSvAbylc0WM3+qauWi2Ugl+SfgvdOOQ5IkaQkeXFX/u9BM1vyN9uPueQ/gxGkGsgg7AocADwZ+OuVYFmKsK8NYV4axrgxjXRkbS6wbS5ywccW6GXB94IuLmdnkb7RzuucTq+oHU41kAUnm/vypsU6Osa4MY10ZxroyjHXyNpY4YeOKtfOdxc5ohw9JkqQZYvInSZI0Q0z+JEmSZojJ32inA/t2z2udsa4MY10ZxroyjHVlGOvkbSxxwsYV65I41IskSdIMseZPkiRphpj8SZIkzRCTP0mSpBli8idJkjRDTP4GJLlKkjcmOTXJBUmOS/KP045rWJKrJnlNks8lOT1JJVk37bhGSXL3JO9OcmKSc5OckuSQJLtOO7ZhSW6V5NAkJyc5P8lZSb6e5LHTjm0hSZ7cHQfnLDz36kqyexfbqMcdpx3fKEnukuSwJL/rjoWfJHnptOMalOSgefbrmtq3SW6d5JPdufW87nzwsiRXnnZsw5LcPslnk/wxyTlJvpDkzmsgrkWf95PcJskRXfy/T/LxJDdca7F2n7N3Jjk2yYXdfNuvVpyLjTXJJkn+OcnhSX7VHcM/TLJ/ki1XM95JMfm7pI8Dj6d17b4v8E3gg0kePdWoLm0r4KnAFYBPTjmWhTwD2B54E3A/4LnAtsDRSe4+xbhG2RL4JfAiWqyPA9YD70vykinGNa8k1wX+Azh12rEs4EXAnYYex081ohG6z/sXgT/QjoH7Af8OZL7lpmA/Lr0/7wScAZxCO39NXZKbAV+jnQeeBzwA+BDwMuCD04vs0pLcDvgScCXgn7rHFYEjk9xpmrGxyPN+kp2Ao2j3en0k8ETgJsCXk2yz8mECi/+O+nvgHsDJtGNkGhYT65WAdcAvaMfw/YB3dMt9NcmVVj7MCasqH224m/sBBTxqqPxztBPpJtOOcSCmsGGYnq27uNdNO64xsW47ouwqwG+AI6Yd3yJfw9HAydOOY574PgX8L3AQcM604xkR3+7dMbrHtGNZRKzXpd3b+8Bpx7LM+Hfr9vV+045lIKZXdDHtOFT+9q786tOOcSCmw7tz05UHyq5KG+ftq1OObVHnfeB/univNlB2A+BPwL+vsVgvN/D3C7v5tl9r+xXYBNhqxLJ7dPM/dprHxnIe1vxt8FDaSf8jQ+XvAa4D3GHVIxqjOtOOYzGq6rQRZecAJwDXX/2IluUM4C/TDmKU7pL0bsAzpx3LZcSTgc1pNX0boyfRvozePe1ABvy5e/7DUPnvgYtoSclacWfgqKo6b66gqv5Iqw382yTXnlZgiznvJ9mUVrP6sao6e2DZXwBfoH3PrbjFfkdV1UWrEc8CMSwYa1X9tarOHDHpmO55Y/kuu5jJ3wa7AD+squEv+e8NTNcEJNkCuA3wg2nHMkqSyyXZNMk2SZ4J3Js1mAwk2RZ4I7B3Vf1q2vEswluT/CXJ2V2bqrtMO6AR7gqcBezUtfn9S5LTkvxnkqtNO7j5dJ+rPYAjq+rn045nwMG0RO9tSW7YtbF6APA04K1Vde50w7uEzYALR5TPld1iFWNZjh1plyi/N2La94AbJbni6oZ0mTbXdGlNfpfNx+Rvg61oJ/1hZw1M12S8lVa78sppBzLGgbTaitOANwB7VdXbpxvSSAcCPwLeNu1AFvAHWpvPpwF3o7X7vD5wVJJ7TzOwEa4LXJl2BeDDtPZIr6W1/TssyVpr9zfoUbQv/ndNO5BBVbWe1hZxF+CnwNm0pgoH046FteQE4I5JLv5u7GrT5q78rPXvgbn4xn2XBbj66oVz2dW1td4f+Bbw6SmHs2SbTjuANWa+qt+N4jLrWpdkP+AxwHOq6thpxzPGq4B30jqmPBA4IMnmVfUf0w1rgyQPp8V267XeBKCqvgN8Z6Doy0k+AXwfeA3w2akENtrlaA38962q/buyo5L8iVbL+vfAEdMKbgFPAs4EPjHtQAZ1vTc/BfyWVjN5Oi2Zegmt/e+TphXbCG+hJc8HJHkl7XjYh9ZmDtpl6o2B32UrKMk1gMNoyfQ/rIXL10tlzd8GZzL6V901uudRv6S0BEn2oZ3wX1xVB0w7nnGq6uSq+lZVHVZVzwD+C3j1KvaUm1eSq9BqT98CnJpky264gc266Vsm2XyaMS6kqn5P+7V8yzXWU26uXc9wQvqZ7vk2qxjLoiW5JXBb4P1VNeqy5TTtD1wNuHdVfayqvlRVr6X1mnxikt2mG94GVfVuYG9aL99f0Xqh3ozWmx5a57+1bO74HfddVrRL8FqmJFcH/o92leCeVfWzKYe0LCZ/G3wf2Lmr4h8018ZjzQ1JsTHpEr91tF5Ur5pyOEt1DK2WfNXGyVrA1sA1gRcAvxt4PIp2Of13wH9PLbrFm7uEupZqIka1lYINsa7VX/hztWfvnGoUo90KOGFE2765oWjWVHvqqvp32mfsFrSep39Lu1R6LrBWr1bM+SlwPqPbJt4COKmqLljdkC47usTvCGAHWuI37nyx5pn8bfAJ2iWIhw+VP542fto3Vj2iy4hucNx1wCuqat8ph7Mcd6N96a+VX3i/ocU0/PgscEH395odlxAuPok+AP5/e/cfa3Vdx3H8+RImMkdLsqClRDJsWc1F0GjLpVINkpIolRZul3KzEauWLXIppLVGBRXGsGzjkisSU8iBaFITS5SIFUhbBgjXVvzwamCICETv/vh8jh6/nHO90uV+z73f12P77t77uZ/vOe/v2bnnvO/78+OwucXejO7OXycV2j+cv27oxVi6RdIgYDqwMSJa8Z/U3cDbc8W6Xm3fvJZbrBQRRyLiLxHxpKQRwFXATyLicNmxdSUvWFwFTJU0pNaer+ES0l62dhLqEr/zgA/l6Sx9luf8ZRFxn6S1pBVprwF2kCopE0l7+BwvNcACSZNIVZ7aH/gFkj6Rv19Tv1VBmSRdB9xM2j/r3uKnDkREy7yZSrqNNBl9I2l+0tnAFaQX/u9GRGeJ4b0oJ0vriu2S2oDjEXHC78okaRlp+GwTaduc0aSq5TCgrbzIThQRD0haBczJk/43kIZT5wKrI+LhUgNsbAppSK8Vq36Q5kr+Clgr6fuk58B44HrSAov7uji3V0l6B6kAsIm0wvdC0jDwdqD0T3jp5uv+XFJVdbWkeaQ5rDeTHvcFrRRrnkpTG/avVSsnSeoEOiPioVaIlTQ68WvgXaTpCgML72WdEfFEb8TaY8reaLCVDlLlbyGwh/SHvwWYVnZcTWLtID0hGx0jy46vLs51XcQZZcdXiHUGaT+vTtJq3/05/j6xgSetu8nzV0kLPg6Q9kt8ilSBGFd2bE3iHUyap/b3/Dx4krQIaFDZsTWJ9wHSHqVDyo6lixhrlek9wPOkVerzabBxbslxnk/6dJdn8nvAdtInqZxZdmw5vm697gPvJlWpDpFW26+k7r4LcAAABZBJREFUsMl2K8TKSxvANzrWtUqs+Wj6PgYsLfu58WqP2q7WZmZmZlYBnvNnZmZmViFO/szMzMwqxMmfmZmZWYU4+TMzMzOrECd/ZmZmZhXi5M/MzMysQpz8mZmZmVWIkz8zMzOzCnHyZ2ZmZlYhTv7MrE+R1CYpJI0tO5aeIGmppI4evs2vS/LHN5lZQ07+zMzK9Q3gY2UHYWbVMbDsAMzM+jtJgyPicKPfRcQTvR2PmVWbK39m1u9IOkPSAkmbJT0r6V+SHpV0eaHfbyU9LkmFdknaIeneurbTJd2Q+x+R1CmpXdLrC+d2SFotaaqkP0t6AZjbRawnDPvmYe1Fkq6W9FdJz0vaImlyg/Mvy9d5RNIuSV9ucj+SNDP3PSxpv6S7JJ1X12davu9ZhXNvknRc0gebXYeZ9R2u/JlZfzQIGArMB/4JnA58AFghaUZE3J77LQTuASYAv6k7fxIwCvg8gKTTcr+LgO8AjwBvBm4C1kkaW6jsjQHeBnwT2AUcOolruAwYB8wBngO+AqyU9NaI2JnjmpDjehSYBgzI/YY1uL0fA23ALcBs0uMzB3hE0oURsS8i7pD0fmCBpA0RsUnSpcANwLciYu1JXIeZtRhFeE6wmfUdktqAdmBcRGzq5jkDAAE/AsZExJjcfhqwHdgaEVPq+q8BzgdGR0RImgb8Avh4RKyo6zcW+CMwMyJuzW0dwDnABRGxrRuxLQUujoiRdW0B7Mv3fzC3DQN2A1+LiHm5bQNwLjAqIl7IbUOADmBoRCi3jScliNdFxPfq7uccYBvww4iYndsG5b6vJSWgDwKPAxMi4vgrXY+ZtT4P+5pZvyTpCknrJT0H/Ac4BnyGVJEDICL+CywCJksakc8bBUwEFsdL/x1PBg4AqyQNrB3AZmAvcHHh7h/rTuL3Ch6sJX451n3AU6SKI5LOJFUGV9QSv9zvILCqcFuTgQB+Voh/L7ClPv6IOAJcCbwO+BMpaf6kEz+z/sPJn5n1O5KmAneShnynA+8lJUpLgDMK3ZcAh4HP5p8/l39eUtdnGKkSdpSURNYfw4GzC7e5pwcu45kGbUeAwfn7s0iv4Xsb9Cu2DSMlcfs4Mf7xFOKPiB3A70mP1c8joieux8xahOf8mVl/NJ001+6quupdbUjzZSLiWUk/Ba6RNB+YASyLiAN13Z4mJWMTm9zfwcLPvTGfZn++n+ENfldsezr3vYiUQBa9rE3SNaQh343ALEnLI+IP/3fEZtYSXPkzs/4ogKOFxG84cHmT/reQql93kSp8iwq/X00aBh0QEZsaHH/r+UvoWkQcIiVnUyW9WM3Mc/4+Uui+mlT5e1OT+LfWnf9O0uNxOylZfAxYLumsU3xJZtZLXPkzs77qUkkjG7SvISU7UyUtJiV05wI3koZjRxdPiIhtku4nrfJ9OCK2FLrcAXwKWCNpISnpOkZa2HEJcE9ErOyJi3qVbgTuB9ZKWkBa7TubtLp4aK1TRKyXdBvQnhep/C73eSPwPtKCl1vzPMI7SVXTmRFxVNKVpLl/7cAUzKzPc/JnZn3Vt5u0vyUi2iW9gTSP79PATmAeKVlrtufeclLyV6z6ERHHJX0U+AJwNXA9aRHJP4CHgK3Fc3pDRKyVNIW0pcxy0ly/xaR5gXMLfa/Nq4OvBWaSRn52A+tJySyk1dAjSCupD+XzduZh4F9K+mJE/ODUX5mZnUre6sXMDJB0N2nxw8iIOFZ2PGZmp4orf2ZWWXkByBjgPaTP1/2SEz8z6+9c+TOzyspzBncB/waWAbO8n52Z9XdO/szMzMwqxFu9mJmZmVWIkz8zMzOzCnHyZ2ZmZlYhTv7MzMzMKsTJn5mZmVmFOPkzMzMzqxAnf2ZmZmYV4uTPzMzMrEKc/JmZmZlViJM/MzMzswr5HzA7coUKDfuWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x480 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(dpi=120)\n",
    "plt.bar(range(1,13),LR.coef_[0])\n",
    "plt.title('Relative Importance of BERT layers in OOD detection')\n",
    "plt.xlabel('Layer index')\n",
    "plt.ylabel('Value of layer score coefficient')\n",
    "plt.xticks(range(13))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnEAAAGdCAYAAAB94yVcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAASdAAAEnQB3mYfeAAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfXgU9b3//9fsXW4IyQYkoGio3CSg4c6iwSIiFAKXSmkjoN8cQUTBoEF+kkM9R+0pInC1ihICNoInWkGsNxW8r0rqQam1aG9QoB4T4QgIEgPJJiYkm2wyvz9SVmIwhN3NzWSfj+viushn3vPZ9+wQ9+XM7IxhmqYpAAAAWIqtoxsAAADA2SPEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABbk6OgG2kN5+QnV15sd3QYAAMD3stsNxcVFt7o+LEJcfb2p+vqGjm4DAACgBWd3gpTTqQAAABZEiAMAALAgQhwAAIAFEeIAAAAsKCy+2AAAQLgyTVNeb41qaqrk89VJ4m4N7ctQZGSUoqO7y2azh3RmQhwAAF2UaZr65huPTpyokCTZbA7ZbEYHdxVe6uvrVVlZLq+3Rj169JZhhO79J8QBANBFeb01OnGiQk5nhOLizpHDwcd+ezNNUxUVpaqurlRNzQlFRXUL2dxcEwcAQBdVU1MlSQS4DmQYhrp3d0uSampOhHRuQhwAAF2Uz1cnm81BgOtgNptdNptdDQ31oZ03pLMBAIBOxOQauE7CMGwyzdA+PYoQBwAA0MZC+H0GP46vAgAQhpxOe0i/KRko0zRVVxfa04zhghAHAEAYMgxDu46Uq9YX2lN8Z8PlsGnEeXEBr//GG69q5cr79d//vVGDB1+k/Pz1evLJx/3LIyIiFBsbpwsv7K8f/egKXX31VEVHh+7boR2NEAcAQJiq9TVo/9eVHfb6/RNi2mTehx9eq27dYuTz1enYsRL97W8f6Te/Wavf/W6Tfv3r1Ro0KKlV85id/L7IhDgAANClJCcPkdvt9v88ceJkpafP1MKF83X33XfpuY2b5HK5vn8Cw5Dp7PxH7PhiAwAA6PIGDUrS7NlzVVxcrII/vi3T5/3eP53+ENy/EOIAAEBYuOKKKyVJuz7Z08GdhAYhDgAAhIU+fc6VJB07fryDOwkNQhwAAAgT1jhN2lqEOAAAEBaOHj0qSTqnZ88O7iQ0CHEAACAs/OlP70qSLhk+rIM7CQ1CHAAA6PKKigq1ceOTOrdPH00Yd2VHtxMS3CcOAAB0KZ999qm6dYtRfb1Px46V6K9//UhvvfWG4uPj9eDKlXI6nR3dYkgQ4gAACFMuh63NnprQ2tdvC9nZCxvnd7nUvXusBgwYqAULFuqaa6Yqxmk23guuCzBM0yJ3tAtCaWmV6us77tlwAAB0hGPHjkiSzjnnvGbLnE67DMNo75aaMU1TdXX17fJahiEZtZVnDHGGI0KmKyak9/xtaV+cZLfb1KNH658UwZE4AADCUHsFJ7QdvtgAAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQTyxAQCAMNTVHru1Z89uPfvs09q9e5fKy8sVGxunYcOG64YbblRKyrDm9f/8VM/+/kV9snevyisqFNu9u4alXKz/N/06DR02skntG2+8qpUr7/f/7HK5FBPTXf36/UCXXTZa1147TfHxPYLehrNFiAMAIAwZhqGIozs79GHwhiNC3j6pQc/z+98/q9zcRzRkyMVasOBO9elzroqLj2rLlhd0++23atGibF133fX++hdeelm5eY9pSHKybr/1FvXp3VvFX3+tF195VQvuytb/l5Wl666f3ex17rnnl0pM/IHq630qKyvVJ5/s0ubNT+l3v3ta99+/UpdeGvy2nA1CHAAAYcr0eeUr2d9hr+/o1T/oOT75ZJdycx/R6NFjtHLlQ3I4vo02P/5xmu65Z4nWrHlYgwYla/jwEfpk927l5j2myy+7VCuX/lIOu/3b+vFX6Z6l9ytn3ToNGjJMQ4eOaPJa/fsP0ODBF/l/vuqqH2vmzH/T7bffonvv/bmefXaLevToGfQ2tRbXxAEAAMt6+unfyjAM/fu//0eTACdJDodD2dl3yzAMPf30U5Kkjc9sliHp3+9c2CTASZLDblf2wiwZkr/+TPr06aOsrLt04kSVXn55Syg2qdUIcQAAwJLq6+v197//VcnJQ5SQ0Pu0Nb1791FS0mD9/e8fNdbv2qXkpCQl9Op1+vqEBCUnJetvf2usb43LLx8ju92uXbv+EfC2BIIQBwAALKm83KOamhqde+55Ldadd955qqmp8def16dPi/XnnttHNTU1qqgob1UfUVFRiotz6/jxklb3HgqEOAAA0KWZ5tmucPIvrf/2rnnWLxK8oELcX//6V82bN0+XXnqphg0bprS0ND366KNNavbu3as5c+Zo5MiRGjVqlLKysnTo0KHTzrdp0yZNmTJFKSkpmjBhgtatW6e6urpgWgQAAF1UXJxbkZGR+uqrIy3WffXVEUVGRvrrjxw92nL90aOKjIxUbGxsq/qorq5WRUW5evY8/SnathJwiHv11Vc1a9Ysde/eXb/+9a+1YcMGzZs3r0kS3bdvn2bNmqW6ujrl5ORo5cqV+uKLL5SRkaHS0tIm8+Xl5WnFihVKS0tTfn6+MjIy9Nhjj2nZsmWBbx0AAOiy7Ha7LrlklD777FN9/XXxaWu+/rpYhYX/q0suubSxfsQIfVZYqK9LTn/q8+uSEn1W+Jl++MPG+tb485//pPr6eo0ceUnA2xKIgEJccXGx/uu//kvXX3+9HnnkEU2YMEGjR4/WjBkzlJWV5a/Lzc2Vy+XS+vXrNW7cOKWlpWn9+vUqKytTfn6+v66srEx5eXmaOXOmFi9erNTUVN16663KysrSCy+8oM8//zz4LQUAAF3OjTfOkWmaevjhXzX7IkJ9fb1WrfqVTNPUrFlzJEmzM/5NpqRVuetOX5+7Vqbkrz+To0eP6tFHcxQTE6Np064LfoPOQkD3iXvhhRd04sQJzZs373trfD6ftm/frmnTpikmJsY/3rdvX6WmpqqgoEBLliyRJO3YsUNer1fp6elN5khPT9fq1atVUFCggQMHBtIqAADowoYNG6E771ys3NxHdPvtt+q662aqd+8+/pv9/vOfe3TnnYs1dOjwxvqhQ3Xngkzl5j2mBXdl67ppP1HvhF4q/rpEW155Rf/838+06I4sDR06vNm1dPv375PPV6/6+np5PKX6+ON/6I03XpXNZtfKlasUHx/frtseUIj76KOP5Ha7tX//ft1+++0qKipSXFycJk2apJ///OeKiYnRwYMHVVNTo+Tk5GbrJyUl6f3335fX61VERISKior846dKSEhQfHy8fzkAAAgdwxERkhvuBvP6oTB9+g0aPPhiPfvs01q3Lkfl5R7/Y7d+85v/bvbYrRk/naYhSUl69vcvat36DU0eu5W3+mENHTZSp/uawslHbzmdTv9jt/7t327Stdf+tN0DnBRgiCsuLlZ1dbUWLVqk2267TSNGjNDu3bu1du1aFRUV6ZlnnpHH45Ekud3uZuu73W6Zpqny8nIlJCTI4/HI5XIpOjq6WW1cXJx/LgAAEBqmaYbkkVeh6CMUUlKGavnyX7e+/qIhWv5f97Wq9uqrp+rqq6cG2lqbCSjEmaYpr9errKwszZ8/X5KUmpoqp9OplStX6oMPPlBkZKQktfhw3VOXdYaH8AIAEC5C8dB5dKyAvthw8ujaFVdc0WT8yiuvlNR4W5GTNWVlZc3W93g8MgzD/9Vdt9str9er6urqZrXl5eWnPZoHAAAQzgIKcae7zk369pCozWZTYmKiIiMjVVhY2KyusLBQ/fr1U0RE47nwk9fCfbe2pKREZWVlGjRoUCBtAgAAdFkBhbi0tDRJ0nvvvddk/OTPw4cPl8Ph0Pjx47Vt2zZVVlb6a44cOaKdO3dq0qRJ/rGxY8cqIiJCW7Y0fXDs1q1bZRiGJk6cGEibAAAAXZZ96dKlS892pcTERO3du1cvvviiJKmurk5/+MMftGbNGl155ZX+6+QGDRqkZ555Rh9++KF69uypoqIi/eIXv5BhGHrwwQcVFRUlSf7r5/Lz81VXVye73a4333xTubm5uu666zR9+vSgNrK6uq5DHocBAEBHOnHiG0lSdHT3Du6kczAMyaivlRpavh7QsDkkuyukr92afWGzGYqKav3rGmaA6aampkbr1q3Ta6+9ppKSEiUkJGjq1KnKysqSy/VtA3v27NGqVau0a9cu2e12jR49WnfffbcSExObzblx40Zt3rxZhw8fVq9evZSenq7MzEw5nc5AWvQrLa1SfX1DUHMAAGA1x441Po7qnHNafkB8uDAMyaitlOnztlzniJDpijn7Z662oKTkiAyj5X1ht9vUo0e3Vs8ZcIizEkIcACAcHTv2lerr65WQ0Je7QKjjQpxpmvr668NyOBzq2bPP99adbYgL+NmpAACgc4uMjJJp1quiolQNZziFiLZhmqa++cYj06yXwxHaU7QB3ScOAAB0ftHR3eX11qi6ulLV1ZWy2ewyDJvC+qBcg08yz3B2zqiSbBVBv5RpSg0NDTLNejmdEerePbS3TCPEAQDQRdlsdvXo0Vs1NSdUU3NCDQ31Ms8UYLowm80mW025GryVLddFxKghupcaGoJ7rwxDcjgccjii1b27WzZbaE+AEuIAAOjCDMNQVFQ3RUW1/lqrrsrlcsj15X75yve3WOfo3l+1fUaqttbXTp0FhmviAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsKKMTt3LlTycnJp/2za9euJrV79+7VnDlzNHLkSI0aNUpZWVk6dOjQaefdtGmTpkyZopSUFE2YMEHr1q1TXV1dIC0CAAB0aY5gVl68eLFSU1ObjA0aNMj/93379mnWrFkaMmSIcnJy5PV6lZubq4yMDL388svq0aOHvzYvL09r1qzR/PnzNWbMGO3evVs5OTkqLi7WAw88EEybAAAAXU5QIa5fv34aMWLE9y7Pzc2Vy+XS+vXrFRMTI0m6+OKLNXnyZOXn52vJkiWSpLKyMuXl5WnmzJlavHixJCk1NVU+n085OTm66aabNHDgwGBaBQAA6FLa7Jo4n8+n7du3Ky0tzR/gJKlv375KTU1VQUGBf2zHjh3yer1KT09vMkd6erpM02xSCwAAgCBD3LJly3TRRRfpkksu0S233KK//vWv/mUHDx5UTU2NkpOTm62XlJSkAwcOyOv1SpKKior846dKSEhQfHy8fzkAAAAaBXQ6tXv37po9e7ZSU1Pldrt14MAB5efna/bs2Vq/fr3Gjh0rj8cjSXK73c3Wd7vdMk1T5eXlSkhIkMfjkcvlUnR0dLPauLg4/1wAAABoFFCIu+iii3TRRRf5fx41apQmTZqkqVOn6qGHHtLYsWP9ywzD+N55Tl3WUh0AAACaCtk1cbGxsbrqqqv02Wefqaamxn8ErqysrFmtx+ORYRiKjY2V1Hhkzuv1qrq6ullteXn5aY/mAQAAhLOQfrHBNE1JjUfVEhMTFRkZqcLCwmZ1hYWF6tevnyIiIiR9ey3cd2tLSkpUVlbW5LYlAAAACGGIKy8v1/bt2zVkyBBFRETI4XBo/Pjx2rZtmyorK/11R44c0c6dOzVp0iT/2NixYxUREaEtW7Y0mXPr1q0yDEMTJ04MVZsAAABdQkDXxGVnZ+vcc89VSkqK4uPjdeDAAT3xxBM6fvy4fvWrX/nrFi5cqOnTpyszM1Pz5s1TbW2tcnNzFR8fr7lz5/rr3G63FixYoDVr1sjtdvtv9rt27VrNmDGDe8QBAAB8h2GePAd6FjZs2KA33nhDX375pU6cOKG4uDj98Ic/1Pz58zVs2LAmtXv27NGqVau0a9cu2e12jR49WnfffbcSExObzbtx40Zt3rxZhw8fVq9evZSenq7MzEw5nc7At1BSaWmV6usbgpoDAABYm8vlkOvL9+Qr2d9inaNXf9Wef6Vqa33t1Fkju92mHj26tbo+oBBnNYQ4AADQ1UJcmz2xAQAAAG2HEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsKCQhbgXXnhBycnJGjlyZLNle/fu1Zw5czRy5EiNGjVKWVlZOnTo0Gnn2bRpk6ZMmaKUlBRNmDBB69atU11dXajaBAAA6BJCEuKKi4v161//WgkJCc2W7du3T7NmzVJdXZ1ycnK0cuVKffHFF8rIyFBpaWmT2ry8PK1YsUJpaWnKz89XRkaGHnvsMS1btiwUbQIAAHQZjlBM8stf/lKjRo2S2+3WW2+91WRZbm6uXC6X1q9fr5iYGEnSxRdfrMmTJys/P19LliyRJJWVlSkvL08zZ87U4sWLJUmpqany+XzKycnRTTfdpIEDB4aiXQAAAMsL+kjcyy+/rA8//FBLly5ttszn82n79u1KS0vzBzhJ6tu3r1JTU1VQUOAf27Fjh7xer9LT05vMkZ6eLtM0m9QCAACEu6BC3PHjx7Vy5UplZ2erT58+zZYfPHhQNTU1Sk5ObrYsKSlJBw4ckNfrlSQVFRX5x0+VkJCg+Ph4/3IAAAAEGeLuv/9+XXjhhcrIyDjtco/HI0lyu93NlrndbpmmqfLycn+ty+VSdHR0s9q4uDj/XAAAAAjimri33npL77zzjl566SUZhtFibUvLT112pnkAAADQKKAjcVVVVVq2bJlmzZqlhIQEVVRUqKKiwn8rkIqKCp04ccJ/BK6srKzZHB6PR4ZhKDY2VlLjkTmv16vq6upmteXl5ac9mgcAABCuAjoSV1ZWpmPHjumJJ57QE0880Wz5pZdeqh//+MfKzc1VZGSkCgsLm9UUFhaqX79+ioiIkPTttXCFhYUaPny4v66kpERlZWUaNGhQIK0CAAB0SQGFuF69emnjxo3Nxjds2KCPPvpIjz/+uOLj4+VwODR+/Hht27ZNS5Ys8X9D9ciRI9q5c6fmzJnjX3fs2LGKiIjQli1bmoS4rVu3yjAMTZw4MZBWAQAAuqSAQlxERIRSU1ObjW/dulV2u73JsoULF2r69OnKzMzUvHnzVFtbq9zcXMXHx2vu3Ln+OrfbrQULFmjNmjVyu90aM2aMdu/erbVr12rGjBncIw4AAJyW02lv1XX1DodNNlvXeeJoSG7225IBAwZo06ZNWrVqlRYtWiS73a7Ro0fr0UcfVY8ePZrULliwQN26ddPmzZuVn5+vXr16af78+crMzGzrNgEAgEUZhqGIoztl+rwt1tnd58tsp57ag2GaZlfantMqLa1SfX1DR7cBAADagMvlkOvL9+Qr2d9y3YDLpcoS1RZ/3mKdo1d/1Z5/pWprfaFs84zsdpt69OjW6vquc0wRAAAgjBDiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAAAsixAEAAFgQIQ4AAMCCCHEAAAAWFFCI+/TTTzV//nxdddVVGjZsmC677DJdf/31evnll5vV7t27V3PmzNHIkSM1atQoZWVl6dChQ6edd9OmTZoyZYpSUlI0YcIErVu3TnV1dYG0CAAA0KU5AlmpoqJCffr00TXXXKPevXururpar776qn7+85/r8OHDuv322yVJ+/bt06xZszRkyBDl5OTI6/UqNzdXGRkZevnll9WjRw//nHl5eVqzZo3mz5+vMWPGaPfu3crJyVFxcbEeeOCB0GwtAABAF2GYpmmGarKZM2fq66+/1vbt2yVJixYt0s6dO1VQUKCYmBhJ0uHDhzV58mTddNNNWrJkiSSprKxM48aN009/+lMtW7bMP99jjz2mnJwcvfbaaxo4cGDAfZWWVqm+viHwDQMAAJ2Wy+WQ68v35CvZ33LdgMulyhLVFn/eYp2jV3/Vnn+lamt9oWzzjOx2m3r06Nbq+pBeExcfHy+73S5J8vl82r59u9LS0vwBTpL69u2r1NRUFRQU+Md27Nghr9er9PT0JvOlp6fLNM0mtQAAAAgyxDU0NMjn86m0tFSbN2/Wn/70J82bN0+SdPDgQdXU1Cg5ObnZeklJSTpw4IC8Xq8kqaioyD9+qoSEBMXHx/uXAwAAoFFA18SdtHTpUj333HOSJKfTqXvvvVc33HCDJMnj8UiS3G53s/XcbrdM01R5ebkSEhLk8XjkcrkUHR3drDYuLs4/FwAAABoFFeIyMzM1Y8YMlZaW6p133tEDDzyg6upq3XLLLf4awzC+d/1Tl7VUBwAAgKaCCnHnnXeezjvvPEnSuHHjJEmPPPKIfvazn/mPwJWVlTVbz+PxyDAMxcbGSmo8Muf1elVdXa2oqKgmteXl5UpJSQmmTQAAgC4npF9sGDZsmHw+nw4dOqTExERFRkaqsLCwWV1hYaH69euniIgISd9eC/fd2pKSEpWVlWnQoEGhbBMAAMDyQhridu7cKZvNpgsuuEAOh0Pjx4/Xtm3bVFlZ6a85cuSIdu7cqUmTJvnHxo4dq4iICG3ZsqXJfFu3bpVhGJo4cWIo2wQAALC8gE6n/uIXv1BMTIyGDh2qc845R2VlZXrzzTf1xhtv6JZbbvHfxHfhwoWaPn26MjMzNW/ePNXW1io3N1fx8fGaO3eufz63260FCxZozZo1crvd/pv9rl27VjNmzAjqHnEAAABdUUA3+33xxRe1ZcsW7du3T998842io6M1ePBgTZ8+XdOmTWtSu2fPHq1atUq7du2S3W7X6NGjdffddysxMbHZvBs3btTmzZt1+PBh9erVS+np6crMzJTT6Qx8C8XNfgEA6MrC9Wa/IX1iQ2dFiAMAoOsK1xAX0mviAAAA0D4IcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAAC3IEstIHH3ygV155Rf/4xz909OhRde/eXSkpKbrjjjuUkpLSpHbv3r166KGH9PHHH8tut2v06NG6++67dcEFFzSbd9OmTdq8ebO+/PJLJSQkKD09XbfddpucTmdgWwcAANBFBXQk7ne/+50OHz6s2bNna8OGDbr33ntVWlqq66+/Xh988IG/bt++fZo1a5bq6uqUk5OjlStX6osvvlBGRoZKS0ubzJmXl6cVK1YoLS1N+fn5ysjI0GOPPaZly5YFt4UAAABdUEBH4n75y1+qZ8+eTcbGjh2rtLQ0rV+/XpdffrkkKTc3Vy6XS+vXr1dMTIwk6eKLL9bkyZOVn5+vJUuWSJLKysqUl5enmTNnavHixZKk1NRU+Xw+5eTk6KabbtLAgQMD3kgAAICuJqAjcd8NcJLUrVs3DRgwQF999ZUkyefzafv27UpLS/MHOEnq27evUlNTVVBQ4B/bsWOHvF6v0tPTm8yZnp4u0zSb1AIAACCEX2z45ptv9M9//lODBg2SJB08eFA1NTVKTk5uVpuUlKQDBw7I6/VKkoqKivzjp0pISFB8fLx/OQAAABqFLMTdf//9qq6uVmZmpiTJ4/FIktxud7Nat9st0zRVXl7ur3W5XIqOjm5WGxcX558LAAAAjUIS4nJycvTqq6/qP//zP5t9O9UwjO9d79RlLdUBAACgqaBD3Lp165SXl6e77rpLN954o3/85BG4srKyZut4PB4ZhqHY2Fh/rdfrVXV1dbPa8vLy0x7NAwAACGdBhbh169Zp7dq1Wrhwof806kmJiYmKjIxUYWFhs/UKCwvVr18/RURESPr2Wrjv1paUlKisrMx/nR0AAAAaBRziHn30Ua1du1YLFixQVlZWs+UOh0Pjx4/Xtm3bVFlZ6R8/cuSIdu7cqUmTJvnHxo4dq4iICG3ZsqXJHFu3bpVhGJo4cWKgbQIAAHRJAd0n7oknnlBubq7Gjh2rq666Srt27WqyfMSIEZKkhQsXavr06crMzNS8efNUW1ur3NxcxcfHa+7cuf56t9utBQsWaM2aNXK73RozZox2796ttWvXasaMGdwjDgAA4DsM0zTNs11p1qxZ+vDDD5TYjukAABOWSURBVL93+Weffeb/+549e7Rq1Srt2rWryWO3EhMTm623ceNGbd68WYcPH1avXr2Unp6uzMzMoB+7VVpapfr6hqDmAAAAnZPL5ZDry/fkK9nfct2Ay6XKEtUWf95inaNXf9Wef6Vqa32hbPOM7HabevTo1ur6gEKc1RDiAADousI1xIXsPnEAAABoP4Q4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAnMJms8nhsMnptHd0Ky0ixAEAAHzHgbJqGYbR0W20iBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAAP9it9tksxmKjXLK4ejcMalzdwcAANCuDH3jrVeE0y67vXPfZoQQBwAA8B02Q/rbobJOfZsRQhwAAIAFEeIAAAAsiBAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAABZEiAMAALAgQhwAAIAFEeIAAADU+NxUu91Q531GQ1OEOAAAAEmSoeMn6jq6iVYjxAEAAFhQwCGusrJSDz74oObOnavRo0crOTlZa9euPW3t3r17NWfOHI0cOVKjRo1SVlaWDh06dNraTZs2acqUKUpJSdGECRO0bt061dVZJxUDAAC0h4BDnMfj0fPPP6/a2lpNnDjxe+v27dunWbNmqa6uTjk5OVq5cqW++OILZWRkqLS0tEltXl6eVqxYobS0NOXn5ysjI0OPPfaYli1bFmibAAAAXZIj0BX79u2rjz76SIZhqLS0VC+88MJp63Jzc+VyubR+/XrFxMRIki6++GJNnjxZ+fn5WrJkiSSprKxMeXl5mjlzphYvXixJSk1Nlc/nU05Ojm666SYNHDgw0HYBAICFOJ12GUbrvmLgcNhks4XfFWIBh7jWvLE+n0/bt2/XtGnT/AFOagyAqampKigo8Ie4HTt2yOv1Kj09vckc6enpWr16tQoKCghxAACECcMwFHF0p0yf94y1dvf5Mtuhp84m4BDXGgcPHlRNTY2Sk5ObLUtKStL7778vr9eriIgIFRUV+cdPlZCQoPj4eP9yAAAQHkyfV76S/Wess8X2boduOp82Pfbo8XgkSW63u9kyt9st0zRVXl7ur3W5XIqOjm5WGxcX558LAAAA7XSLkZZOvZ66rLXnvgEAAMJdm4a4k0fgysrKmi3zeDwyDEOxsbH+Wq/Xq+rq6ma15eXlpz2aBwAAEK7aNMQlJiYqMjJShYWFzZYVFhaqX79+ioiIkPTttXDfrS0pKVFZWZkGDRrUlq0CAABYSpuGOIfDofHjx2vbtm2qrKz0jx85ckQ7d+7UpEmT/GNjx45VRESEtmzZ0mSOrVu3yjCMFu9FBwAAEG6C+nbqu+++q+rqalVVVUmSPv/8c7355puSpHHjxikqKkoLFy7U9OnTlZmZqXnz5qm2tla5ubmKj4/X3Llz/XO53W4tWLBAa9askdvt1pgxY7R7926tXbtWM2bM4PYiAAAApzBM0wz41ioTJkzQ4cOHT7vsj3/8o84//3xJ0p49e7Rq1Srt2rVLdrtdo0eP1t13363ExMRm623cuFGbN2/W4cOH1atXL6WnpyszM1NOpzPQNlVaWqX6+oaA1wcAAO3L5XLI9eV7rbrFiGvA5VJliWqLPw+qzm63y+P1KSphgBouHKf/PVqhlITuqq31BbQNZ8tut6lHj26trg8qxFkFIQ4AAGvpiBDncjlUeqJOkf8KcUcravQDd5Tq6upVV1cf0HacjbMNceH3jAoAAIBWsNsM/ePL8k57CzRCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAACHt2u002myF1zvv6nhYhDgAAQIa+8bb9o7VCiRAHAABgQYQ4AAAACyLEAQAAWBAhDgAAwIIIcQAAIKzZ7TbZ7YaVvpgqiRAHAADCnqHjJ+o6uomzRogDAACwIEIcAACABRHiAAAALIgQBwAAwpoVv9QgEeIAAAAsydHRDQAAgPDhdNplGGc+7uVw2GSzcaypJYQ4AADQbgzDUMTRnTJ93hbr7O7zZbZTT1ZFiAMAAO3K9HnlK9nfYo0ttnc7dWNdHKcEAACwIEIcAACABRHiAAAALIgQBwAAYEGEOAAAEHbsdrvsdntHtxEUQhwAAAhbdrtNNpshKz6ygVuMAACAsGO3G43hTVKlt/5763rGRMjhsKm2tr06az1CHAAACEvHKhuTWaTTmqdVOZ0KAABgQYQ4AAAAC+J0KgAACBoPtm9/hDgAABA0Hmzf/ghxAAAgJKzyYPsmtxWxcKLsVCGuqqpKOTk5+sMf/qDy8nL1799f8+fP1zXXXNPRrQEA0KWczelPSfL5Gs5YZ53TpIa+aeG2IlbRqULcwoULtXv3bmVnZ+sHP/iBXnvtNS1evFgNDQ2aOnVqR7cHAECXcVanP6uOyVZXc+a6UDbYRux2m2xOm7yEuNB599139f777+vhhx/WtddeK0kaPXq0jhw5ogcffFBXX3215R+PAQBAoNriyJnRUKe61pz+rK+1xGnSM7EZhgxDKqmqVaSjdZmiW4RDdrtNTqdddXWdK/h1mhC3bds2RUdHa8qUKU3G09PTlZ2drY8//liXXHJJB3UHAAhWa0OIJJmm2ek+ME/qqO0I1yNnwbDbbTp54Vt9fWOora1vOdx+l82Q/naoTCkJ3UPfYJA6TYgrKirSgAED5HA0bSk5Odm/nBAHANbV2hBiOCLk7ZN6xvlCfWRKal3oanWYiu2juh6Dz9gjR84Cc2pAsxmGZDNkt9tUX98gu93m//LCF8dPKDE+6l8/W+WavdbpNCHO4/Ho/PPPbzYeFxfnXx4ou91QW97X2GnRx3UAQHuy220yXNGSveWPHsPuktNp9z/XsqX57MW7zvi6tugeMms8cjScOcTV9x7xr3DQ8uu2ajtc3eQ8tvuMr9va/mzRPWRGueU8f2jLda4YmVF11q+LiJFMnyIuGH7a5YbNUNmJWsVHu2RExOiEt0ZR5w+To8GUYTN0orYxjLu7N8gR41JljU9OI1L2aLeiLxgmp80mW0ODZEpOe+PfT44ZdqcMmyGXwyaH3VCUq/Hf45n+bQSrMa+0XqcJcZJa/L+V1h66Pp24uOiA1wUAhNCFl3Xu+Tr766KJhFP+HttrQJNlcd+pPfXniFbO7+7WWNnH3TlzRKc5ruh2u097tK28vFzSt0fkAAAA0IlCXFJSkvbt2yefz9dkvLCwUJI0aNCgjmgLAACgU+o0IW7ixIk6ceKE3n777SbjW7duVUJCgoYPP/05cQAAgHDUaa6JGzdunMaMGaOlS5eqsrJSiYmJev3117Vjxw499NBD3CMOAADgFIZpmp3mNjFVVVVavXq13nzzTXk8HvXv31+33XYbj90CAAD4jk4V4gAAANA6neaaOAAAALQeIQ4AAMCCCHEAAAAWRIgDAACwIEIcAACABRHizkJlZaUefPBBzZ07V6NHj1ZycrLWrl17VnMcP35c//Ef/6HU1FQNHz5c119/vT744IM26ji8VFVVacWKFbriiis0dOhQTZs2Ta+//nqr1//LX/6im2++WZdffrlGjhypqVOnauPGjaqvr2/Drru+YPeLJBUUFOjGG2/UJZdcohEjRuiaa67Rc88910Ydh49Q7JuTVq9ereTkZF177bUh7jL8BLNf3n77bS1evFiTJk3SsGHDNGHCBGVnZ+uLL75o26a7kGDe//b+jO80N/u1Ao/Ho+eff16DBw/WxIkT9cILL5zV+rW1tZozZ44qKip07733qmfPntq8ebNuvfVWPfnkk7rsMh6oHIyFCxdq9+7dys7O1g9+8AO99tprWrx4sRoaGjR16tQW1/3zn/+sW265RaNGjdIDDzyg6OhovfPOO1qxYoUOHjyo++67r522ousJZr9I0oYNG7R69WrdcMMNmj9/vpxOp/bv36+6urp26L5rC3bfnPTpp5/qiSee0DnnnNOG3YaPYPbL448/rnPOOUeZmZm64IIL9NVXX2n9+vX62c9+pueff55HWLZCoO9/h3zGm2i1hoYGs6GhwTRN0zx+/LiZlJRk5ubmtnr9p59+2kxKSjL//ve/+8fq6urMq6++2pw+fXrI+w0n27dvN5OSksxXX321yfjNN99sXnHFFabP52tx/ezsbDMlJcWsqqpqMj537lzzkksuCXm/4SLY/bJ7925z8ODB5oYNG9qyzbAU7L45qa6uzpw2bZr5wAMPmDfeeKN5zTXXtEW7YSPY/XLs2LFmY0ePHjUvvvhi85577glpr11RMO9/R3zGczr1LBiGIcMwAl6/oKBAF154oUaOHOkfczgc+slPfqJPPvlExcXFoWgzLG3btk3R0dGaMmVKk/H09HR9/fXX+vjjj1tc3+l0yul0KjIyssl49+7dFREREfJ+w0Ww+2Xz5s1yuVyaNWtWW7YZloLdNydt2LBB5eXluuuuu9qizbAT7H7p2bNns7HevXurd+/eOnr0aEh77YqCef874jOeENeOioqKlJyc3Gz85FhRUVF7t9RlFBUVacCAAXI4ml4h0Nr39oYbblBdXZ2WL1+u4uJiVVRU6KWXXlJBQYFuvfXWNuu7qwt2v3z00UcaMGCA3nrrLU2ePFlDhgzRlVdeqVWrVqm2trbN+g4Hwe4bSfr888+Vl5enpUuXqlu3bm3SZ7gJxX75rkOHDunIkSMaOHBgSHrsyoJ5/zviM55r4tqRx+NRXFxcs/GTYx6Pp71b6jI8Ho/OP//8ZuOtfW+HDx+up556SosWLdLmzZslSXa7XYsXL9bcuXND33CYCHa/FBcXq7S0VCtWrNCiRYs0YMAAffDBB3r88cf11Vdf6eGHH26TvsNBsPumoaFB99xzj9LS0jRu3Lg26TEcBbtfvsvn8+mee+5RdHS05syZE4oWu7Rg3v+O+IwP2xC3c+dOzZ49u1W1L730koYMGRKS123pdGwwp2q7kkD3TTDv7Z49e5SVlaVhw4Zp2bJlioqK0l/+8hfl5OTI6/XqjjvuaP0GdFEdsV9M01RVVZUeeeQRXXPNNZKk0aNHq7q6Wk899ZTuvPNO9evXr5Vb0HV1xL558skndeDAAeXl5bW+0TDTEfvlVKZp6t5779Xf/vY35ebm6txzz231uuEsmPe/vT/jwzbEXXjhhVq+fHmrakP1D9/tdp82iZeXl0vSaRN8OApk3wT73i5btkw9e/bUo48+KrvdLqkxLNhsNq1bt04/+clPdMEFF5zNZnQ5HbFf3G63SkpKdMUVVzQZv/LKK/XUU09p7969hDi1/745cuSIcnNzlZ2dLafTqYqKCkmNR30aGhpUUVEhl8vV7BrTcNMRvzMnnQxwr7zyin71q19p4sSJrew6vAXz/nfEZ3zYhriEhATNmDGjXV8zKSlJhYWFzcZPjvHV70aB7JukpCS99tpr8vl8Ta5laO17++mnn+raa6/1B7iThg4dqoaGBu3bty/sQ1xH7Jfk5GSVlJQ0GzdNU5Jks3FZr9T+++bQoUOqqanRihUrtGLFimbLL730Us2ePVv33nvvWfXU1XTE74z0bYDbsmWLVqxYoWnTpp1d42EsmPe/Iz7j+S9gO5o4caL279/f5NstPp9Pr7zyioYPH67evXt3YHfWNnHiRJ04cUJvv/12k/GtW7cqISFBw4cPb3H9hIQE7dmzp9mNfXft2iVJ6tOnT2gbDhPB7pe0tDRJ0nvvvddk/N1335XNZtPQoUND23AYCWbfDBkyRBs3bmz2Z/Dgwerbt682btyoG2+8sa03oUsK9nfGNE3dd9992rJli5YtW6brrruuLdvtcoJ5/zviMz5sj8QF6t1331V1dbWqqqokNX47680335QkjRs3TlFRUZKke+65Ry+99JK2bdumvn37SpKmT5+uZ555RosWLVJ2drZ69uypZ555Rv/3f/+nJ598smM2qIsYN26cxowZo6VLl6qyslKJiYl6/fXXtWPHDj300ENNjrCdbt/MmTNHy5cvV2Zmpq6//npFRUXpgw8+0JNPPqkf/ehHGjx4cEdtmqUFu1/S09P13HPP6f7771dZWZkGDhyoP//5z3rmmWeUkZHhr8PZC2bfxMbGKjU1tdmcsbGxqq+vP+0ytE6wvzPLly/X73//e1133XVKSkry/4+oJLlcLl100UXtvk1W0tr3v7N8xhPiztL999+vw4cP+39+8803/SHuj3/8o/9bLQ0NDaqvr/ef9pEaf4F++9vf6qGHHtLy5ctVXV2tIUOG6PHHH+dpDSGwdu1arV69Wrm5ufJ4POrfv3+TC+JPOt2+mTVrlnr37q3f/va3uu++++T1etW3b1/dcccdfKMrSMHsF6fTqSeeeEKPPPKI1q9fr/LycvXt21fZ2dm6+eab23tTupxg9g3aTjD75X/+538kSS+++KJefPHFJvV9+/bVO++80/YbYHGtef87y2e8YfJbCQAAYDlcEwcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsCBCHAAAgAUR4gAAACyIEAcAAGBBhDgAAAALIsQBAABYECEOAADAgghxAAAAFkSIAwAAsKD/H9J3PotI0bvSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This cell can be used to visualize the score distributions at a specific layer\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "l = 4 # Index of layer\n",
    "\n",
    "scores_ID, scores_OOD = [], []\n",
    "for k in range(len(y_test)):\n",
    "    if not y_test[k]:\n",
    "        scores_OOD.append(X_test[k,l])\n",
    "    else:\n",
    "        scores_ID.append(X_test[k,l])\n",
    "        \n",
    "plt.figure(dpi=120)\n",
    "sns.set_style('dark')\n",
    "sns.distplot(scores_ID, hist = True, kde = False, kde_kws = {'linewidth': 2}, label='ID')\n",
    "sns.distplot(scores_OOD, hist = True, kde = False, kde_kws = {'linewidth': 2}, label='OOD')\n",
    "plt.legend(loc='upper right')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "name": "pytorch-gpu.1-4.m49",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.1-4:m49"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}