Simple Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 데이터셋\n",
    "\n",
    "쇼핑 상품 분류 대회에서는 총 3개의 데이터셋을 제공합니다.\n",
    "\n",
    "    train\n",
    "    dev\n",
    "    test\n",
    "\n",
    "각 데이터셋은 hdf5 포맷으로 저장되어 있습니다. hdf5 파일을 사용하는 방법에 대해서는 대회 안내 페이지를 참고하세요.\n",
    "\n",
    "train은 총 8,134,818개의 샘플이 포함되어있고, dev는 507,783개, test는 1,526,523개의 샘플이 포함되어 있습니다. train은 실험하는 목적으로 정답 카테고리값과 함께 제공되나 dev와 test 샘플에는 정답 카테고리 값이 없습니다.\n",
    "\n",
    "dev 샘플에 대한 예측한 카테고리 정보는 대회 기간동안 예측 파일 제출을 통해 정확도를 리더보드에서 확인할 수 있습니다. test 샘플에 대한 예측 결과는 대회 종료 후에 공개되며 대회 종료전까지 여러번 제출이 가능하나, 마지막 제출한 결과만 사용되며 test 샘플에 대한 예측 결과가 대회 최종 순위를 결정하는데 사용됩니다.\n",
    "\n",
    "## 데이터셋 설명\n",
    "\n",
    "모든 샘플은 중카테고리까지의 분류는 있지만 소나 세카테고리까지의 분류는 없을 수 있습니다. 카테고리정보는 숫자로 치환되어 있으며 실제 카테고리명은 cate1.json에 있습니다. b, m, s, d는 각각 대/중/소/세 카테고리를 의미하며, train의 샘플 데이터중에 카테고리 정보가 -1인 것은 해당 카테고리 분류 결과가 없다는 뜻이며, dev나 test은 평가를 위해 카테고리 정보가 모두 -1로 제공됩니다.\n",
    "\n",
    "카테고리내 ID는 모두 1부터 시작합니다. (-1 제외)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\iam\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "C:\\Users\\iam\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n",
      "C:\\Users\\iam\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\gensim\\utils.py:1212: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# 필요한 Package import\n",
    "from catekitten.data import Loader, get_category_map, split_pos\n",
    "from catekitten.transform import PosTokenizer\n",
    "from catekitten.evaluate import arena_accuracy_score\n",
    "from catekitten import tf_glove\n",
    "\n",
    "from pprint import pprint\n",
    "from collections import defaultdict\n",
    "from eunjeon import Mecab\n",
    "\n",
    "from gensim.models import FastText, Doc2Vec\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import h5py\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Dataset object 생성\n",
    "dataset = Loader(\"data/prep/textonly.h5\")\n",
    "y_label = get_category_map(\"data/raw/cate1.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bcateid',\n",
      " 'dcateid',\n",
      " 'mcateid',\n",
      " 'price',\n",
      " 'scateid',\n",
      " 'brand',\n",
      " 'maker',\n",
      " 'model',\n",
      " 'pid',\n",
      " 'product',\n",
      " 'updttm']\n"
     ]
    }
   ],
   "source": [
    "# Dataset columns 확인\n",
    "pprint(dataset.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['가공식품/과자/초콜릿', '수납/정리/선반', '디카/캠코더/주변기기', '침구/커튼/카페트', '데스크탑/모니터/PC부품']\n",
      "['김치냉장고', '권투용품', '프라이팬/용품', '야구', 'CPU']\n",
      "['', '탱크 RC', '북엔드', '로비의자', '한복/생활한복']\n",
      "['', 'MP3 케이블/충전기', 'MP3 케이스/파우치', '전동칫솔', '춘천시']\n"
     ]
    }
   ],
   "source": [
    "# category id 확인\n",
    "pprint(list(y_label['bcateid'].values())[:5])\n",
    "pprint(list(y_label['mcateid'].values())[:5])\n",
    "pprint(list(y_label['scateid'].values())[:5])\n",
    "pprint(list(y_label['dcateid'].values())[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "57"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sorted(y_label['bcateid'].keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Price:\n",
      "0    16520\n",
      "1    20370\n",
      "2       -1\n",
      "3    16280\n",
      "4       -1\n",
      "Name: price, dtype: int32\n",
      "\n",
      "updttm:\n",
      "0    1519690229.0\n",
      "1    1524959419.0\n",
      "2    1524705794.0\n",
      "3    1524354312.0\n",
      "4    1524521783.0\n",
      "Name: updttm, dtype: object\n",
      "Brand:\n",
      "0    퍼즐라이프\n",
      "1     바보사랑\n",
      "2     크리비아\n",
      "3      잭앤질\n",
      "4         \n",
      "Name: brand, dtype: object\n",
      "\n",
      "Maker:\n",
      "0    상품상세설명 참조\n",
      "1    MORY|해당없음\n",
      "2             \n",
      "3       ㈜크리스패션\n",
      "4           기타\n",
      "Name: maker, dtype: object\n",
      "\n",
      "Model:\n",
      "0                           퍼즐라이프 직소퍼즐 바다거북의 여행\n",
      "1    아이폰6S/6S+ tree farm101 - 다이어리케이스|아이폰6S/6S+\n",
      "2                       크리비아 기모 3부 속바지 GLG4314P\n",
      "3     [잭앤질] 남성 솔리드 절개라인 포인트 포켓 팬츠 31133PT002_NA\n",
      "4                              SD코드프리혈당시험지[50매]\n",
      "Name: model, dtype: object\n",
      "\n",
      "Product:\n",
      "0                      직소퍼즐 - 1000조각 바다거북의 여행 (PL1275)\n",
      "1    [모리케이스]아이폰6S/6S+ tree farm101 - 다이어리케이스[바보사랑][...\n",
      "2                              크리비아 기모 3부 속바지 GLG4314P\n",
      "3        [하프클럽/잭앤질]남성 솔리드 절개라인 포인트 포켓 팬츠 31133PT002_NA\n",
      "4                          코드프리혈당시험지50매/코드프리시험지/최장유효기간\n",
      "Name: product, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nPrice:\")\n",
    "pprint(dataset[\"price\", :5])\n",
    "print(\"\\nupdttm:\")\n",
    "pprint(dataset[\"updttm\", :5])\n",
    "print(\"Brand:\")\n",
    "pprint(dataset[\"brand\", :5])\n",
    "print(\"\\nMaker:\")\n",
    "pprint(dataset[\"maker\", :5])\n",
    "print(\"\\nModel:\")\n",
    "pprint(dataset[\"model\", :5])\n",
    "print(\"\\nProduct:\")\n",
    "pprint(dataset[\"product\", :5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "brand\n",
      "maker\n",
      "model\n",
      "product\n"
     ]
    }
   ],
   "source": [
    "text_attrs = [\"brand\", \"maker\", \"model\", \"product\"]\n",
    "tagger = PosTokenizer()\n",
    "x = np.zeros([200000, 400])\n",
    "\n",
    "for i, text_attr in enumerate(text_attrs):\n",
    "    print(text_attr)\n",
    "    attr_text = tagger.transform(dataset[text_attr, :200000]) \n",
    "    attr_text = attr_text.map(split_pos)\n",
    "    documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(attr_text)]\n",
    "    model = Doc2Vec(documents, workers=4)\n",
    "    x[:,i*100:(i+1)*100] = np.vstack(attr_text.map(model.infer_vector))\n",
    "    model.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = dataset['bcateid', :200000]\n",
    "y = keras.utils.to_categorical(y, num_classes=57)\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=39)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 400, 100)          100000    \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 400, 64)           6464      \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 57)                3705      \n",
      "=================================================================\n",
      "Total params: 110,169\n",
      "Trainable params: 110,169\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\iam\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 50000 samples\n",
      "Epoch 1/100\n",
      " - 11s - loss: 4.0070 - categorical_accuracy: 0.0627 - val_loss: 3.9599 - val_categorical_accuracy: 0.1018\n",
      "Epoch 2/100\n",
      " - 7s - loss: 3.8731 - categorical_accuracy: 0.1006 - val_loss: 3.7630 - val_categorical_accuracy: 0.1018\n",
      "Epoch 3/100\n",
      " - 7s - loss: 3.6502 - categorical_accuracy: 0.1006 - val_loss: 3.5547 - val_categorical_accuracy: 0.1018\n",
      "Epoch 4/100\n",
      " - 7s - loss: 3.5087 - categorical_accuracy: 0.1006 - val_loss: 3.4773 - val_categorical_accuracy: 0.1018\n",
      "Epoch 5/100\n",
      " - 7s - loss: 3.4646 - categorical_accuracy: 0.1006 - val_loss: 3.4525 - val_categorical_accuracy: 0.1018\n",
      "Epoch 6/100\n",
      " - 7s - loss: 3.4482 - categorical_accuracy: 0.1006 - val_loss: 3.4407 - val_categorical_accuracy: 0.1018\n",
      "Epoch 7/100\n",
      " - 7s - loss: 3.4396 - categorical_accuracy: 0.1006 - val_loss: 3.4337 - val_categorical_accuracy: 0.1018\n",
      "Epoch 8/100\n",
      " - 7s - loss: 3.4344 - categorical_accuracy: 0.1006 - val_loss: 3.4291 - val_categorical_accuracy: 0.1018\n",
      "Epoch 9/100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-0314a14f08d4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[0mx_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         validation_data=(x_test, y_test), epochs=100, verbose=2, batch_size=1024)\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m   1637\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1638\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1639\u001b[1;33m           validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1640\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1641\u001b[0m   def evaluate(self,\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    213\u001b[0m           \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    214\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 215\u001b[1;33m         \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    216\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    217\u001b[0m           \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2984\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2985\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[1;32m-> 2986\u001b[1;33m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[0;32m   2987\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2988\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1440\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "with tf.device(\"/gpu:0\"):\n",
    "    pooled_tensors = []\n",
    "    model = Sequential()\n",
    "    model.add(layers.Embedding(1000, 100, input_length=400))\n",
    "    \n",
    "    for filter_size in self.filter_sizes:\n",
    "        x_i = model.add(layers.Conv1D(self.num_filters, filter_size, activation='elu', **self.conv_kwargs)\n",
    "        x_i = layers.GlobalMaxPooling1D\n",
    "        pooled_tensors.append(x_i)\n",
    "\n",
    "    x = pooled_tensors[0] if len(self.filter_sizes) == 1 else concatenate(pooled_tensors, axis=-1)\n",
    "\n",
    "#     model.add(layers.Dense(512, activation='relu', input_dim=400))\n",
    "#     model.add(layers.Dropout(0.5))\n",
    "#     model.add(layers.Dense(512, activation='relu'))\n",
    "#     model.add(layers.Dropout(0.5))\n",
    "#     model.add(layers.Dense(512, activation='relu', input_dim=400))\n",
    "#     model.add(layers.Dropout(0.5))\n",
    "#     model.add(layers.Dense(512, activation='relu'))\n",
    "#     model.add(layers.Dropout(0.5))\n",
    "    model.add(layers.Dense(57, activation='softmax'))\n",
    "    optimizer = tf.keras.optimizers.Nadam(1e-4)\n",
    "    model.compile(\n",
    "        loss='categorical_crossentropy',\n",
    "        optimizer=optimizer,\n",
    "        metrics=[keras.metrics.categorical_accuracy]\n",
    "    )\n",
    "    model.summary()\n",
    "    model.fit(\n",
    "        x_train,\n",
    "        y_train,\n",
    "        validation_data=(x_test, y_test), epochs=100, verbose=2, batch_size=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.6326895734024047, 0.54422]\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = np.argmax(model.predict(x_test), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.5736])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arena_accuracy_score(y_pred, np.argmax(y_test, axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}