dqn_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4221e1e3",
   "metadata": {},
   "source": [
    "# Training v2.0 with a Deep Q Network (DQN) #"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d628c23d",
   "metadata": {},
   "source": [
    "Import statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b9fe8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "rc('text', usetex=True)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "761fd1ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Flatten\n",
    "from tensorflow.keras.optimizers import Adam, SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f09c161",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rl.agents import DQNAgent\n",
    "from rl.policy import BoltzmannQPolicy, EpsGreedyQPolicy\n",
    "from rl.memory import SequentialMemory  # For experience replay!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c9f747a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gym_environment_ncml import *\n",
    "from learning import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3a0674c",
   "metadata": {},
   "source": [
    "Useful numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cf11480",
   "metadata": {},
   "outputs": [],
   "source": [
    "MILLION = 1000000\n",
    "HTHOUSAND = 100000\n",
    "THOUSAND = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edfb0b18",
   "metadata": {},
   "source": [
    "## 1. Create environment ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b9a3f4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = GridworldMultiAgentv2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "708f7943",
   "metadata": {},
   "outputs": [],
   "source": [
    "states = env.observation_space.shape[0]\n",
    "actions = env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eca1527b",
   "metadata": {},
   "outputs": [],
   "source": [
    "states, actions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0343c79a",
   "metadata": {},
   "source": [
    "## 2. Create a Deep Learning Model with Keras ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "941ff796",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(states, actions, [32, 32, 32], ['relu', 'relu', 'relu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f136bc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d0b28d2",
   "metadata": {},
   "source": [
    "## 3. Build Agent with Keras-RL ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a418e6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn = build_agent(model, actions, 0.01, EpsGreedyQPolicy(), 50000)\n",
    "dqn.compile(Adam(lr=1e-3), metrics=['mae'])\n",
    "# dqn.compile(Adam(lr=1e-2), metrics=['mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "073ef900",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = dqn.fit(env, nb_steps=5*MILLION, visualize=False, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "523042d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6025dba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,7))\n",
    "\n",
    "ax.plot(history.history['nb_steps'], history.history['episode_reward'])\n",
    "\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bf73776",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = dqn.test(env, nb_episodes=10, visualize=False)\n",
    "print(np.mean(scores.history['episode_reward']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5d1d24c",
   "metadata": {},
   "source": [
    "Save agent to memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1759baa",
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn.save_weights('agents/dqn_5b5_3030_adam_lr0.001_tmu0.01_ml50K_ns5M.h5f', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "910b175d",
   "metadata": {},
   "source": [
    "## 4. Reloading Agent from memory and test ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42713167",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'dqn20_5b5_3216_adam_lr0.001_tmu0.01_ml50K_ns5M_eps0.1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eb14792",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = GridworldMultiAgentv2(seed=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd5ad652",
   "metadata": {},
   "outputs": [],
   "source": [
    "states = env.observation_space.shape[0]\n",
    "actions = env.action_space.n\n",
    "model = build_model(states, actions, [32, 16], ['relu', 'relu'])\n",
    "print(model.summary())\n",
    "dqn = build_agent(model, actions, 0.01, EpsGreedyQPolicy(eps=0), 50000)\n",
    "dqn.compile(Adam(lr=1e-3), metrics=['mae'])\n",
    "\n",
    "# Load weights\n",
    "dqn.load_weights(get_agent_path(name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cc3b9c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "nb_episodes = 10*THOUSAND"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5aabed3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = dqn.test(env, nb_episodes=nb_episodes, visualize=False, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "594f77b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "rewards = np.array(scores.history['episode_reward'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d06402f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(get_test_path(name, nb_episodes), rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c40d006",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}