pytorch_ipynb/autoencoder/ae-conv-nneighbor-quickdraw-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "11xi8CRmVA1d"
   },
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2889,
     "status": "ok",
     "timestamp": 1525034072517,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "WuXDfh6UVA1g",
    "outputId": "80c92b95-76aa-444e-9aa3-9bcf5d000a6e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.6.8\n",
      "IPython 7.2.0\n",
      "\n",
      "torch 1.0.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Cii2luqnVA1s"
   },
   "source": [
    "- Runs on CPU (not recommended here) or GPU (if available)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EYAtjwgyVA1t"
   },
   "source": [
    "# Model Zoo -- Convolutional Autoencoder with Nearest-neighbor Interpolation (Trained on 10 categories of the Quickdraw dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ke9a_LDUVA1v"
   },
   "source": [
    "A convolutional autoencoder using nearest neighbor upscaling layers that compresses 768-pixel Quickdraw images down to a 7x7x8 (392 pixel) representation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1iZAQwueVA1x"
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "CO_0yUH6VA1z"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uhGdecraVA1-"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is based on Google's Quickdraw dataset (https://quickdraw.withgoogle.com). In particular we will be working with an arbitrary subset of 10 categories in png format:\n",
    "\n",
    "    label_dict = {\n",
    "             \"lollipop\": 0,\n",
    "             \"binoculars\": 1,\n",
    "             \"mouse\": 2,\n",
    "             \"basket\": 3,\n",
    "             \"penguin\": 4,\n",
    "             \"washing machine\": 5,\n",
    "             \"canoe\": 6,\n",
    "             \"eyeglasses\": 7,\n",
    "             \"beach\": 8,\n",
    "             \"screwdriver\": 9,\n",
    "    }\n",
    "    \n",
    "(The class labels 0-9 can be ignored in this notebook). \n",
    "\n",
    "For more details on obtaining and preparing the dataset, please see the\n",
    "\n",
    "- [custom-data-loader-quickdraw.ipynb](custom-data-loader-quickdraw.ipynb)\n",
    "\n",
    "notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(28, 28)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEMFJREFUeJzt3WuMVHWexvHnJzaCIEGW3rajsK1CNARcWBoUR404ikpMdIyB4YWwXoaJkWTHTIy3kCX6AmJ2ZiJx1TCLoTEzyEYH9YVhhzUaHbMIDUEuXlYXmQzY0E24yEXuv33RB9Nq1/80dU7VKfr//SSE7nrqdP1S+nCq+n/qHHN3AYjPOUUPAKAYlB+IFOUHIkX5gUhRfiBSlB+IFOUHIkX5gUhRfiBS51bzwYYOHepNTU3VfEggKtu2bdPu3butJ/fNVH4zu03Sc5L6SPoPd18Qun9TU5NaW1uzPCSAgObm5h7ft+yX/WbWR9K/S7pd0ihJM8xsVLk/D0B1ZXnPP1HSl+6+1d2PSXpV0p35jAWg0rKU/2JJf+vy/fbktu8xs9lm1mpmrR0dHRkeDkCeKv7bfndf5O7N7t5cX19f6YcD0ENZyr9D0rAu31+S3AbgLJCl/GsljTSzS82sr6SfS3orn7EAVFrZS33ufsLM5kj6L3Uu9b3s7ltym6wX+fjjj4P5Rx99FMz3798fzM85p/S/4aNGhRdg0paGeKvWe2Va53f3tyW9ndMsAKqIw3uBSFF+IFKUH4gU5QciRfmBSFF+IFJV/Tx/b5W2Tn/zzTcH84MHD+Y5Tq7Szr+QdpxAKE/bdvz48cF88ODBwRxh7PmBSFF+IFKUH4gU5QciRfmBSFF+IFIs9fXQ5s2bS2a33357cNtLLrkkmK9cuTKYNzQ0BHN3L5lt2LAhuG3a2ZTT8rVr1wbz119/vWQWmrsnRo4cGcxDS4kTJkwoe1tJamxsDOYjRowI5rWAPT8QKcoPRIryA5Gi/ECkKD8QKcoPRIryA5FinT+xdevWYD5lypSS2aBBg4Lbrlq1KpinHQeQxaRJkzLlWYU+rrx+/frgtlmPQQjly5YtC26b1auvvhrMp0+fXtHH7wn2/ECkKD8QKcoPRIryA5Gi/ECkKD8QKcoPRCrTOr+ZbZN0QNJJSSfcPfwh6AK1tbUF81tuuSWYnzp1qmRW5Dp+rRs4cGDJ7IYbbghum5Z/8803wXzevHkls+effz647bnnhqvx7bffBvP+/fsH81qQx0E+k919dw4/B0AV8bIfiFTW8rukP5vZOjObncdAAKoj68v+69x9h5n9vaRVZvaZu7/f9Q7JPwqzJWn48OEZHw5AXjLt+d19R/J3u6QVkiZ2c59F7t7s7s319fVZHg5Ajsouv5kNMLMLTn8taYqk0qe4BVBTsrzsb5C0wsxO/5w/unv4HNQAakbZ5Xf3rZL+McdZMtm7d28wv/XWW4P5nj17gvm7775bMks7fzy6l3be/paWlmD+xBNPBPPdu0uvQD/00EPBbe+///5gPm7cuGB+6NChYF4LWOoDIkX5gUhRfiBSlB+IFOUHIkX5gUj1mlN3T506NZiHLrEthT96KkmTJ08umSXHOpQ0ePDgYJ5VaPa6urpMP7tv377BfOzYscF8/PjxJbPFixcHt129enUwv/HGG4P5woULS2ZjxowJbnv48OFgnmbXrl2Ztq8G9vxApCg/ECnKD0SK8gORovxApCg/ECnKD0Sq16zz79y5M5innUXovvvuK/ux09aEjx49WvbPlqQTJ04E8wMHDmT6+SFpH5VesmRJMH/ppZdKZmn/TZYvXx7Mp02bFsyzOP/884P5gAEDgnl7e3ue41QEe34gUpQfiBTlByJF+YFIUX4gUpQfiBTlByLVa9b5006fnXY55wULFuQ5TjRmzpwZzF955ZWS2Zw5c4LbVnIdP6u0YxQ6OjqqNEn52PMDkaL8QKQoPxApyg9EivIDkaL8QKQoPxCp1HV+M3tZ0h2S2t19dHLbEEnLJTVJ2iZpmruHP/hdsLTPZxfpyJEjwfy1114L5tu3by+ZpZ3b/pprrgnmaZYuXRrM33vvvZLZjh07Mj12kRoaGoJ5b/k8/xJJt/3gtsclvePuIyW9k3wP4CySWn53f1/Snh/cfKekluTrFkl35TwXgAor9z1/g7u3JV/vlBR+DQSg5mT+hZ+7uyQvlZvZbDNrNbPWs+F4ZyAW5ZZ/l5k1SlLyd8nfbrj7IndvdvfmtA9DAKiecsv/lqRZydezJL2ZzzgAqiW1/Ga2TNL/SLrCzLab2QOSFki6xcy+kHRz8j2As0jqOr+7zygR/TTnWXqtZ599NpjPnz8/mO/bty/Pcb6npaUlmKd9Xj/NpZdeWjL76quvMv3sIqWt8+/atatKk5SPI/yASFF+IFKUH4gU5QciRfmBSFF+IFK95tTdRVqxYkUwf+yxxzL9/HvvvTeYv/jiiyWze+65J7jtww8/HMzTtk/7qHRoqe/DDz8MblvLhg4dGsw3btxYpUnKx54fiBTlByJF+YFIUX4gUpQfiBTlByJF+YFIsc6fg1WrVgXziy66KJinncL6nHPK/zf6qaeeCubXX399MF+5cmUwv/vuu4P5ZZddVjJbtmxZcNvjx48H87q6umBeSXv3hs9UP3DgwCpNUj72/ECkKD8QKcoPRIryA5Gi/ECkKD8QKcoPRIp1/hwcOnQomA8aNCiYZ1nHT3PttdcG87RjEN54441gnrbOP2bMmJLZsWPHgttu2bIlmI8dOzaYV9KaNWuC+ZQpU6o0SfnY8wORovxApCg/ECnKD0SK8gORovxApCg/EKnUdX4ze1nSHZLa3X10cts8Sb+Q1JHc7Ul3f7tSQ9a6pqamYL58+fJgfvLkyWDep0+fMx3pO2nHEITOqy9JHR0dwTxNc3Nz2du2trYG80qu87e1tQXztHMwXH311XmOUxE92fMvkXRbN7f/zt3HJn+iLT5wtkotv7u/L2lPFWYBUEVZ3vPPMbONZvaymV2Y20QAqqLc8r8o6XJJYyW1SfpNqTua2WwzazWz1qzvHwHkp6zyu/sudz/p7qck/V7SxMB9F7l7s7s319fXlzsngJyVVX4za+zy7c8kbc5nHADV0pOlvmWSbpQ01My2S/pXSTea2VhJLmmbpF9WcEYAFZBafnef0c3Niyswy1lrwoQJwfzo0aPB/JFHHgnmCxcuPOOZeuqzzz4L5rNmzcr084cNG1YySzuXwLp164L5gw8+WNZMPbF69epM20+cWPKdcM3gCD8gUpQfiBTlByJF+YFIUX4gUpQfiBSn7s7BHXfcEcznzp0bzJ955plgnrbsNH369JLZBRdcENw27VLT48ePD+ZZpP3stI/0VlLaqbn79esXzEePHp3nOBXBnh+IFOUHIkX5gUhRfiBSlB+IFOUHIkX5gUixzl8FTz/9dDAfPnx4MF+yZEkwf/TRR0tm7h7c9sorrwzmoWMIsko7rff8+fODedpHpc8777wznum0tHX+cePGBfO6urqyH7ta2PMDkaL8QKQoPxApyg9EivIDkaL8QKQoPxAp1vlrQNopqNPy0OWkv/766+C2aZforuR6ddrn+Y8dOxbMN23aFMyzXB78iiuuCOaVvDx4tbDnByJF+YFIUX4gUpQfiBTlByJF+YFIUX4gUqnr/GY2TNJSSQ2SXNIid3/OzIZIWi6pSdI2SdPcPXwSeFREY2NjWVnRrrrqqkzbf/LJJ8E8yzr/Cy+8UPa2Z4ue7PlPSPq1u4+SdI2kh81slKTHJb3j7iMlvZN8D+AskVp+d29z9/XJ1wckfSrpYkl3SmpJ7tYi6a5KDQkgf2f0nt/MmiSNk/SRpAZ3P31c6U51vi0AcJbocfnNbKCk1yX9yt2/6Zp554niuj1ZnJnNNrNWM2vt6OjINCyA/PSo/GZWp87i/8Hd/5TcvMvMGpO8UVJ7d9u6+yJ3b3b35vr6+jxmBpCD1PKbmUlaLOlTd/9tl+gtSbOSr2dJejP/8QBUSk8+0vsTSfdK2mRmG5LbnpS0QNJ/mtkDkv4qaVplRkRvde652T5RnnZacoSlPvvu/hdJViL+ab7jAKgWjvADIkX5gUhRfiBSlB+IFOUHIkX5gUj1mlN39+vXL5i3t3d7ACIKdOLEiUzb9+nTJ6dJ4sSeH4gU5QciRfmBSFF+IFKUH4gU5QciRfmBSPWadf4RI0YE8w8++CCY791b/lnHL7zwwrK3jdnx48czbd+3b9+cJokTe34gUpQfiBTlByJF+YFIUX4gUpQfiBTlByLVa9b5R48eHcz37dsXzIcMGZLnOGck7VwEaXlI1mMQ0h778ssvD+aTJk0qmWW9fHhdXV2m7WPHnh+IFOUHIkX5gUhRfiBSlB+IFOUHIkX5gUilrvOb2TBJSyU1SHJJi9z9OTObJ+kXkjqSuz7p7m9XatA0M2bMCOZp14I/cuRIMD969GjJ7PDhw8Ft06Qdg5B2HfpKzrZ///5g/vnnnwfzuXPnlsxOnTpV1kynsc6fTU8O8jkh6dfuvt7MLpC0zsxWJdnv3P3fKjcegEpJLb+7t0lqS74+YGafSrq40oMBqKwzes9vZk2Sxkn6KLlpjpltNLOXzazb40jNbLaZtZpZa0dHR3d3AVCAHpffzAZKel3Sr9z9G0kvSrpc0lh1vjL4TXfbufsid2929+b6+vocRgaQhx6V38zq1Fn8P7j7nyTJ3Xe5+0l3PyXp95ImVm5MAHlLLb+ZmaTFkj519992ub3rR7J+Jmlz/uMBqJSe/Lb/J5LulbTJzDYktz0paYaZjVXn8t82Sb+syIQ91L9//2A+c+bMKk2Crg4ePFgyW7NmTXDbLVu2BPPJkyeXNRM69eS3/X+RZN1Eha3pA8iOI/yASFF+IFKUH4gU5QciRfmBSFF+IFK95tTdqE0DBw4smd10003BbdNyZMOeH4gU5QciRfmBSFF+IFKUH4gU5QciRfmBSFnaaaFzfTCzDkl/7XLTUEm7qzbAmanV2Wp1LonZypXnbP/g7j06X15Vy/+jBzdrdffmwgYIqNXZanUuidnKVdRsvOwHIkX5gUgVXf5FBT9+SK3OVqtzScxWrkJmK/Q9P4DiFL3nB1CQQspvZreZ2edm9qWZPV7EDKWY2TYz22RmG8ysteBZXjazdjPb3OW2IWa2ysy+SP7u9jJpBc02z8x2JM/dBjObWtBsw8zsXTP7xMy2mNm/JLcX+twF5irkeav6y34z6yPpfyXdImm7pLWSZrj7J1UdpAQz2yap2d0LXxM2sxskHZS01N1HJ7c9K2mPuy9I/uG80N0fq5HZ5kk6WPSVm5MLyjR2vbK0pLsk/bMKfO4Cc01TAc9bEXv+iZK+dPet7n5M0quS7ixgjprn7u9L2vODm++U1JJ83aLO/3mqrsRsNcHd29x9ffL1AUmnryxd6HMXmKsQRZT/Ykl/6/L9dtXWJb9d0p/NbJ2ZzS56mG40JJdNl6SdkhqKHKYbqVdurqYfXFm6Zp67cq54nTd+4fdj17n7P0m6XdLDycvbmuSd79lqabmmR1durpZuriz9nSKfu3KveJ23Isq/Q9KwLt9fktxWE9x9R/J3u6QVqr2rD+86fZHU5O/2guf5Ti1dubm7K0urBp67WrridRHlXytppJldamZ9Jf1c0lsFzPEjZjYg+UWMzGyApCmqvasPvyVpVvL1LElvFjjL99TKlZtLXVlaBT93NXfFa3ev+h9JU9X5G///k/RUETOUmOsySR8nf7YUPZukZep8GXhcnb8beUDS30l6R9IXkv5b0pAamu0VSZskbVRn0RoLmu06db6k3yhpQ/JnatHPXWCuQp43jvADIsUv/IBIUX4gUpQfiBTlByJF+YFIUX4gUpQfiBTlByL1/6GuDeISpVyLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.read_csv('quickdraw_png_set1_train.csv', index_col=0)\n",
    "df.head()\n",
    "\n",
    "main_dir = 'quickdraw-png_set1/'\n",
    "\n",
    "img = Image.open(os.path.join(main_dir, df.index[99]))\n",
    "img = np.asarray(img, dtype=np.uint8)\n",
    "print(img.shape)\n",
    "plt.imshow(np.array(img), cmap='binary')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Z1zF9mPwVA2P"
   },
   "source": [
    "### Create a Custom Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "fV2cRUDx7mqz"
   },
   "outputs": [],
   "source": [
    "class QuickdrawDataset(Dataset):\n",
    "    \"\"\"Custom Dataset for loading Quickdraw images\"\"\"\n",
    "\n",
    "    def __init__(self, txt_path, img_dir, transform=None):\n",
    "    \n",
    "        df = pd.read_csv(txt_path, sep=\",\", index_col=0)\n",
    "        self.img_dir = img_dir\n",
    "        self.txt_path = txt_path\n",
    "        self.img_names = df.index.values\n",
    "        self.y = df['Label'].values\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        img = Image.open(os.path.join(self.img_dir,\n",
    "                                      self.img_names[index]))\n",
    "        \n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        \n",
    "        label = self.y[index]\n",
    "        return img, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.y.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that transforms.ToTensor()\n",
    "# already divides pixels by 255. internally\n",
    "\n",
    "\n",
    "BATCH_SIZE = 128\n",
    "\n",
    "custom_transform = transforms.Compose([#transforms.Lambda(lambda x: x/255.),\n",
    "                                       transforms.ToTensor()])\n",
    "\n",
    "train_dataset = QuickdrawDataset(txt_path='quickdraw_png_set1_train.csv',\n",
    "                                 img_dir='quickdraw-png_set1/',\n",
    "                                 transform=custom_transform)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset,\n",
    "                          batch_size=BATCH_SIZE,\n",
    "                          shuffle=True,\n",
    "                          num_workers=4) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | Batch index: 0 | Batch size: 128\n",
      "Epoch: 2 | Batch index: 0 | Batch size: 128\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "torch.manual_seed(0)\n",
    "\n",
    "num_epochs = 2\n",
    "for epoch in range(num_epochs):\n",
    "\n",
    "    for batch_idx, (x, y) in enumerate(train_loader):\n",
    "        \n",
    "        print('Epoch:', epoch+1, end='')\n",
    "        print(' | Batch index:', batch_idx, end='')\n",
    "        print(' | Batch size:', y.size()[0])\n",
    "        \n",
    "        x = x.to(device)\n",
    "        y = y.to(device)\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wHRUkZFFVA2T"
   },
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 330,
     "status": "ok",
     "timestamp": 1525034084237,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "KthquBjBVA2V",
    "outputId": "4014037d-f8b6-4dcc-e6ec-9db4e4cd38fd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda:3\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Device\n",
    "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "print('Device:', device)\n",
    "\n",
    "# Hyperparameters\n",
    "random_seed = 123\n",
    "learning_rate = 0.0005\n",
    "num_epochs = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pnSfNaJrVA2Z"
   },
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "aKDo_CM9-5eL"
   },
   "outputs": [],
   "source": [
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "\n",
    "class Autoencoder(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Autoencoder, self).__init__()\n",
    "        \n",
    "        # calculate same padding:\n",
    "        # (w - k + 2*p)/s + 1 = o\n",
    "        # => p = (s(o-1) - w + k)/2\n",
    "        \n",
    "        ### ENCODER\n",
    "        \n",
    "        # 28x28x1 => 28x28x4\n",
    "        self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
    "                                      out_channels=4,\n",
    "                                      kernel_size=(3, 3),\n",
    "                                      stride=(1, 1),\n",
    "                                      # (1(28-1) - 28 + 3) / 2 = 1\n",
    "                                      padding=1)\n",
    "        # 28x28x4 => 14x14x4                              \n",
    "        self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
    "                                         stride=(2, 2),\n",
    "                                         # (2(14-1) - 28 + 2) / 2 = 0\n",
    "                                         padding=0)                                       \n",
    "        # 14x14x4 => 14x14x8\n",
    "        self.conv_2 = torch.nn.Conv2d(in_channels=4,\n",
    "                                      out_channels=8,\n",
    "                                      kernel_size=(3, 3),\n",
    "                                      stride=(1, 1),\n",
    "                                      # (1(14-1) - 14 + 3) / 2 = 1\n",
    "                                      padding=1)                 \n",
    "        # 14x14x8 => 7x7x8                             \n",
    "        self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
    "                                         stride=(2, 2),\n",
    "                                         # (2(7-1) - 14 + 2) / 2 = 0\n",
    "                                         padding=0)\n",
    "        \n",
    "        ### DECODER\n",
    "                                         \n",
    "        # 7x7x8 => 14x14x8                          \n",
    "        \n",
    "        ## interpolation\n",
    "        \n",
    "        # 14x14x8 => 14x14x8\n",
    "        self.conv_3 = torch.nn.Conv2d(in_channels=8,\n",
    "                                      out_channels=4,\n",
    "                                      kernel_size=(3, 3),\n",
    "                                      stride=(1, 1),\n",
    "                                      # (1(14-1) - 14 + 3) / 2 = 1\n",
    "                                      padding=1)\n",
    "        # 14x14x4 => 28x28x4                            \n",
    "\n",
    "        ## interpolation\n",
    "        \n",
    "        # 28x28x4 => 28x28x1\n",
    "        self.conv_4 = torch.nn.Conv2d(in_channels=4,\n",
    "                                      out_channels=1,\n",
    "                                      kernel_size=(3, 3),\n",
    "                                      stride=(1, 1),\n",
    "                                      # (1(28-1) - 28 + 3) / 2 = 1\n",
    "                                      padding=1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        ### ENCODER\n",
    "        x = self.conv_1(x)\n",
    "        x = F.leaky_relu(x)\n",
    "        x = self.pool_1(x)\n",
    "        x = self.conv_2(x)\n",
    "        x = F.leaky_relu(x)\n",
    "        x = self.pool_2(x)\n",
    "        \n",
    "        ### DECODER\n",
    "        x = F.interpolate(x, scale_factor=2, mode='nearest')\n",
    "        x = self.conv_3(x)\n",
    "        x = F.leaky_relu(x)\n",
    "        x = F.interpolate(x, scale_factor=2, mode='nearest')\n",
    "        x = self.conv_4(x)\n",
    "        x = F.leaky_relu(x)\n",
    "        x = torch.sigmoid(x)\n",
    "        return x\n",
    "\n",
    "    \n",
    "torch.manual_seed(random_seed)\n",
    "model = Autoencoder()\n",
    "model = model.to(device)\n",
    "    \n",
    "\n",
    "##########################\n",
    "### COST AND OPTIMIZER\n",
    "##########################\n",
    "\n",
    "cost_fn = torch.nn.BCELoss() # torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PfAT3P8_VA2e"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1224
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10453399,
     "status": "ok",
     "timestamp": 1525044538453,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "rbZ8ploO_JW2",
    "outputId": "42e24455-31a2-425b-fea4-599e7cc144d3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/050 | Batch 0000/8290 | Cost: 0.2212\n",
      "Epoch: 001/050 | Batch 0500/8290 | Cost: 0.0968\n",
      "Epoch: 001/050 | Batch 1000/8290 | Cost: 0.0681\n",
      "Epoch: 001/050 | Batch 1500/8290 | Cost: 0.0658\n",
      "Epoch: 001/050 | Batch 2000/8290 | Cost: 0.0531\n",
      "Epoch: 001/050 | Batch 2500/8290 | Cost: 0.0411\n",
      "Epoch: 001/050 | Batch 3000/8290 | Cost: 0.0417\n",
      "Epoch: 001/050 | Batch 3500/8290 | Cost: 0.0363\n",
      "Epoch: 001/050 | Batch 4000/8290 | Cost: 0.0338\n",
      "Epoch: 001/050 | Batch 4500/8290 | Cost: 0.0352\n",
      "Epoch: 001/050 | Batch 5000/8290 | Cost: 0.0363\n",
      "Epoch: 001/050 | Batch 5500/8290 | Cost: 0.0334\n",
      "Epoch: 001/050 | Batch 6000/8290 | Cost: 0.0346\n",
      "Epoch: 001/050 | Batch 6500/8290 | Cost: 0.0299\n",
      "Epoch: 001/050 | Batch 7000/8290 | Cost: 0.0320\n",
      "Epoch: 001/050 | Batch 7500/8290 | Cost: 0.0288\n",
      "Epoch: 001/050 | Batch 8000/8290 | Cost: 0.0288\n",
      "Time elapsed: 1.08 min\n",
      "Epoch: 002/050 | Batch 0000/8290 | Cost: 0.0308\n",
      "Epoch: 002/050 | Batch 0500/8290 | Cost: 0.0291\n",
      "Epoch: 002/050 | Batch 1000/8290 | Cost: 0.0292\n",
      "Epoch: 002/050 | Batch 1500/8290 | Cost: 0.0291\n",
      "Epoch: 002/050 | Batch 2000/8290 | Cost: 0.0259\n",
      "Epoch: 002/050 | Batch 2500/8290 | Cost: 0.0274\n",
      "Epoch: 002/050 | Batch 3000/8290 | Cost: 0.0257\n",
      "Epoch: 002/050 | Batch 3500/8290 | Cost: 0.0262\n",
      "Epoch: 002/050 | Batch 4000/8290 | Cost: 0.0261\n",
      "Epoch: 002/050 | Batch 4500/8290 | Cost: 0.0281\n",
      "Epoch: 002/050 | Batch 5000/8290 | Cost: 0.0238\n",
      "Epoch: 002/050 | Batch 5500/8290 | Cost: 0.0236\n",
      "Epoch: 002/050 | Batch 6000/8290 | Cost: 0.0234\n",
      "Epoch: 002/050 | Batch 6500/8290 | Cost: 0.0219\n",
      "Epoch: 002/050 | Batch 7000/8290 | Cost: 0.0225\n",
      "Epoch: 002/050 | Batch 7500/8290 | Cost: 0.0237\n",
      "Epoch: 002/050 | Batch 8000/8290 | Cost: 0.0237\n",
      "Time elapsed: 2.01 min\n",
      "Epoch: 003/050 | Batch 0000/8290 | Cost: 0.0263\n",
      "Epoch: 003/050 | Batch 0500/8290 | Cost: 0.0231\n",
      "Epoch: 003/050 | Batch 1000/8290 | Cost: 0.0231\n",
      "Epoch: 003/050 | Batch 1500/8290 | Cost: 0.0238\n",
      "Epoch: 003/050 | Batch 2000/8290 | Cost: 0.0235\n",
      "Epoch: 003/050 | Batch 2500/8290 | Cost: 0.0218\n",
      "Epoch: 003/050 | Batch 3000/8290 | Cost: 0.0214\n",
      "Epoch: 003/050 | Batch 3500/8290 | Cost: 0.0223\n",
      "Epoch: 003/050 | Batch 4000/8290 | Cost: 0.0230\n",
      "Epoch: 003/050 | Batch 4500/8290 | Cost: 0.0231\n",
      "Epoch: 003/050 | Batch 5000/8290 | Cost: 0.0222\n",
      "Epoch: 003/050 | Batch 5500/8290 | Cost: 0.0240\n",
      "Epoch: 003/050 | Batch 6000/8290 | Cost: 0.0224\n",
      "Epoch: 003/050 | Batch 6500/8290 | Cost: 0.0221\n",
      "Epoch: 003/050 | Batch 7000/8290 | Cost: 0.0225\n",
      "Epoch: 003/050 | Batch 7500/8290 | Cost: 0.0223\n",
      "Epoch: 003/050 | Batch 8000/8290 | Cost: 0.0210\n",
      "Time elapsed: 2.95 min\n",
      "Epoch: 004/050 | Batch 0000/8290 | Cost: 0.0215\n",
      "Epoch: 004/050 | Batch 0500/8290 | Cost: 0.0225\n",
      "Epoch: 004/050 | Batch 1000/8290 | Cost: 0.0235\n",
      "Epoch: 004/050 | Batch 1500/8290 | Cost: 0.0230\n",
      "Epoch: 004/050 | Batch 2000/8290 | Cost: 0.0218\n",
      "Epoch: 004/050 | Batch 2500/8290 | Cost: 0.0228\n",
      "Epoch: 004/050 | Batch 3000/8290 | Cost: 0.0223\n",
      "Epoch: 004/050 | Batch 3500/8290 | Cost: 0.0221\n",
      "Epoch: 004/050 | Batch 4000/8290 | Cost: 0.0225\n",
      "Epoch: 004/050 | Batch 4500/8290 | Cost: 0.0212\n",
      "Epoch: 004/050 | Batch 5000/8290 | Cost: 0.0219\n",
      "Epoch: 004/050 | Batch 5500/8290 | Cost: 0.0216\n",
      "Epoch: 004/050 | Batch 6000/8290 | Cost: 0.0229\n",
      "Epoch: 004/050 | Batch 6500/8290 | Cost: 0.0221\n",
      "Epoch: 004/050 | Batch 7000/8290 | Cost: 0.0237\n",
      "Epoch: 004/050 | Batch 7500/8290 | Cost: 0.0195\n",
      "Epoch: 004/050 | Batch 8000/8290 | Cost: 0.0195\n",
      "Time elapsed: 3.88 min\n",
      "Epoch: 005/050 | Batch 0000/8290 | Cost: 0.0219\n",
      "Epoch: 005/050 | Batch 0500/8290 | Cost: 0.0214\n",
      "Epoch: 005/050 | Batch 1000/8290 | Cost: 0.0217\n",
      "Epoch: 005/050 | Batch 1500/8290 | Cost: 0.0210\n",
      "Epoch: 005/050 | Batch 2000/8290 | Cost: 0.0220\n",
      "Epoch: 005/050 | Batch 2500/8290 | Cost: 0.0215\n",
      "Epoch: 005/050 | Batch 3000/8290 | Cost: 0.0213\n",
      "Epoch: 005/050 | Batch 3500/8290 | Cost: 0.0214\n",
      "Epoch: 005/050 | Batch 4000/8290 | Cost: 0.0232\n",
      "Epoch: 005/050 | Batch 4500/8290 | Cost: 0.0216\n",
      "Epoch: 005/050 | Batch 5000/8290 | Cost: 0.0214\n",
      "Epoch: 005/050 | Batch 5500/8290 | Cost: 0.0228\n",
      "Epoch: 005/050 | Batch 6000/8290 | Cost: 0.0211\n",
      "Epoch: 005/050 | Batch 6500/8290 | Cost: 0.0221\n",
      "Epoch: 005/050 | Batch 7000/8290 | Cost: 0.0216\n",
      "Epoch: 005/050 | Batch 7500/8290 | Cost: 0.0216\n",
      "Epoch: 005/050 | Batch 8000/8290 | Cost: 0.0208\n",
      "Time elapsed: 4.82 min\n",
      "Epoch: 006/050 | Batch 0000/8290 | Cost: 0.0227\n",
      "Epoch: 006/050 | Batch 0500/8290 | Cost: 0.0226\n",
      "Epoch: 006/050 | Batch 1000/8290 | Cost: 0.0214\n",
      "Epoch: 006/050 | Batch 1500/8290 | Cost: 0.0206\n",
      "Epoch: 006/050 | Batch 2000/8290 | Cost: 0.0209\n",
      "Epoch: 006/050 | Batch 2500/8290 | Cost: 0.0208\n",
      "Epoch: 006/050 | Batch 3000/8290 | Cost: 0.0215\n",
      "Epoch: 006/050 | Batch 3500/8290 | Cost: 0.0226\n",
      "Epoch: 006/050 | Batch 4000/8290 | Cost: 0.0203\n",
      "Epoch: 006/050 | Batch 4500/8290 | Cost: 0.0195\n",
      "Epoch: 006/050 | Batch 5000/8290 | Cost: 0.0215\n",
      "Epoch: 006/050 | Batch 5500/8290 | Cost: 0.0204\n",
      "Epoch: 006/050 | Batch 6000/8290 | Cost: 0.0196\n",
      "Epoch: 006/050 | Batch 6500/8290 | Cost: 0.0209\n",
      "Epoch: 006/050 | Batch 7000/8290 | Cost: 0.0197\n",
      "Epoch: 006/050 | Batch 7500/8290 | Cost: 0.0191\n",
      "Epoch: 006/050 | Batch 8000/8290 | Cost: 0.0218\n",
      "Time elapsed: 5.76 min\n",
      "Epoch: 007/050 | Batch 0000/8290 | Cost: 0.0198\n",
      "Epoch: 007/050 | Batch 0500/8290 | Cost: 0.0211\n",
      "Epoch: 007/050 | Batch 1000/8290 | Cost: 0.0218\n",
      "Epoch: 007/050 | Batch 1500/8290 | Cost: 0.0216\n",
      "Epoch: 007/050 | Batch 2000/8290 | Cost: 0.0218\n",
      "Epoch: 007/050 | Batch 2500/8290 | Cost: 0.0203\n",
      "Epoch: 007/050 | Batch 3000/8290 | Cost: 0.0196\n",
      "Epoch: 007/050 | Batch 3500/8290 | Cost: 0.0203\n",
      "Epoch: 007/050 | Batch 4000/8290 | Cost: 0.0202\n",
      "Epoch: 007/050 | Batch 4500/8290 | Cost: 0.0219\n",
      "Epoch: 007/050 | Batch 5000/8290 | Cost: 0.0190\n",
      "Epoch: 007/050 | Batch 5500/8290 | Cost: 0.0204\n",
      "Epoch: 007/050 | Batch 6000/8290 | Cost: 0.0206\n",
      "Epoch: 007/050 | Batch 6500/8290 | Cost: 0.0201\n",
      "Epoch: 007/050 | Batch 7000/8290 | Cost: 0.0210\n",
      "Epoch: 007/050 | Batch 7500/8290 | Cost: 0.0213\n",
      "Epoch: 007/050 | Batch 8000/8290 | Cost: 0.0215\n",
      "Time elapsed: 6.69 min\n",
      "Epoch: 008/050 | Batch 0000/8290 | Cost: 0.0218\n",
      "Epoch: 008/050 | Batch 0500/8290 | Cost: 0.0229\n",
      "Epoch: 008/050 | Batch 1000/8290 | Cost: 0.0194\n",
      "Epoch: 008/050 | Batch 1500/8290 | Cost: 0.0220\n",
      "Epoch: 008/050 | Batch 2000/8290 | Cost: 0.0208\n",
      "Epoch: 008/050 | Batch 2500/8290 | Cost: 0.0209\n",
      "Epoch: 008/050 | Batch 3000/8290 | Cost: 0.0197\n",
      "Epoch: 008/050 | Batch 3500/8290 | Cost: 0.0218\n",
      "Epoch: 008/050 | Batch 4000/8290 | Cost: 0.0214\n",
      "Epoch: 008/050 | Batch 4500/8290 | Cost: 0.0209\n",
      "Epoch: 008/050 | Batch 5000/8290 | Cost: 0.0202\n",
      "Epoch: 008/050 | Batch 5500/8290 | Cost: 0.0202\n",
      "Epoch: 008/050 | Batch 6000/8290 | Cost: 0.0215\n",
      "Epoch: 008/050 | Batch 6500/8290 | Cost: 0.0213\n",
      "Epoch: 008/050 | Batch 7000/8290 | Cost: 0.0215\n",
      "Epoch: 008/050 | Batch 7500/8290 | Cost: 0.0201\n",
      "Epoch: 008/050 | Batch 8000/8290 | Cost: 0.0215\n",
      "Time elapsed: 7.62 min\n",
      "Epoch: 009/050 | Batch 0000/8290 | Cost: 0.0212\n",
      "Epoch: 009/050 | Batch 0500/8290 | Cost: 0.0211\n",
      "Epoch: 009/050 | Batch 1000/8290 | Cost: 0.0208\n",
      "Epoch: 009/050 | Batch 1500/8290 | Cost: 0.0213\n",
      "Epoch: 009/050 | Batch 2000/8290 | Cost: 0.0196\n",
      "Epoch: 009/050 | Batch 2500/8290 | Cost: 0.0199\n",
      "Epoch: 009/050 | Batch 3000/8290 | Cost: 0.0211\n",
      "Epoch: 009/050 | Batch 3500/8290 | Cost: 0.0218\n",
      "Epoch: 009/050 | Batch 4000/8290 | Cost: 0.0201\n",
      "Epoch: 009/050 | Batch 4500/8290 | Cost: 0.0208\n",
      "Epoch: 009/050 | Batch 5000/8290 | Cost: 0.0202\n",
      "Epoch: 009/050 | Batch 5500/8290 | Cost: 0.0202\n",
      "Epoch: 009/050 | Batch 6000/8290 | Cost: 0.0214\n",
      "Epoch: 009/050 | Batch 6500/8290 | Cost: 0.0197\n",
      "Epoch: 009/050 | Batch 7000/8290 | Cost: 0.0205\n",
      "Epoch: 009/050 | Batch 7500/8290 | Cost: 0.0196\n",
      "Epoch: 009/050 | Batch 8000/8290 | Cost: 0.0199\n",
      "Time elapsed: 8.55 min\n",
      "Epoch: 010/050 | Batch 0000/8290 | Cost: 0.0200\n",
      "Epoch: 010/050 | Batch 0500/8290 | Cost: 0.0210\n",
      "Epoch: 010/050 | Batch 1000/8290 | Cost: 0.0203\n",
      "Epoch: 010/050 | Batch 1500/8290 | Cost: 0.0198\n",
      "Epoch: 010/050 | Batch 2000/8290 | Cost: 0.0186\n",
      "Epoch: 010/050 | Batch 2500/8290 | Cost: 0.0205\n",
      "Epoch: 010/050 | Batch 3000/8290 | Cost: 0.0201\n",
      "Epoch: 010/050 | Batch 3500/8290 | Cost: 0.0197\n",
      "Epoch: 010/050 | Batch 4000/8290 | Cost: 0.0211\n",
      "Epoch: 010/050 | Batch 4500/8290 | Cost: 0.0182\n",
      "Epoch: 010/050 | Batch 5000/8290 | Cost: 0.0208\n",
      "Epoch: 010/050 | Batch 5500/8290 | Cost: 0.0182\n",
      "Epoch: 010/050 | Batch 6000/8290 | Cost: 0.0209\n",
      "Epoch: 010/050 | Batch 6500/8290 | Cost: 0.0229\n",
      "Epoch: 010/050 | Batch 7000/8290 | Cost: 0.0180\n",
      "Epoch: 010/050 | Batch 7500/8290 | Cost: 0.0211\n",
      "Epoch: 010/050 | Batch 8000/8290 | Cost: 0.0194\n",
      "Time elapsed: 9.48 min\n",
      "Epoch: 011/050 | Batch 0000/8290 | Cost: 0.0203\n",
      "Epoch: 011/050 | Batch 0500/8290 | Cost: 0.0197\n",
      "Epoch: 011/050 | Batch 1000/8290 | Cost: 0.0212\n",
      "Epoch: 011/050 | Batch 1500/8290 | Cost: 0.0207\n",
      "Epoch: 011/050 | Batch 2000/8290 | Cost: 0.0203\n",
      "Epoch: 011/050 | Batch 2500/8290 | Cost: 0.0194\n",
      "Epoch: 011/050 | Batch 3000/8290 | Cost: 0.0191\n",
      "Epoch: 011/050 | Batch 3500/8290 | Cost: 0.0202\n",
      "Epoch: 011/050 | Batch 4000/8290 | Cost: 0.0206\n",
      "Epoch: 011/050 | Batch 4500/8290 | Cost: 0.0213\n",
      "Epoch: 011/050 | Batch 5000/8290 | Cost: 0.0205\n",
      "Epoch: 011/050 | Batch 5500/8290 | Cost: 0.0221\n",
      "Epoch: 011/050 | Batch 6000/8290 | Cost: 0.0193\n",
      "Epoch: 011/050 | Batch 6500/8290 | Cost: 0.0203\n",
      "Epoch: 011/050 | Batch 7000/8290 | Cost: 0.0205\n",
      "Epoch: 011/050 | Batch 7500/8290 | Cost: 0.0202\n",
      "Epoch: 011/050 | Batch 8000/8290 | Cost: 0.0206\n",
      "Time elapsed: 10.40 min\n",
      "Epoch: 012/050 | Batch 0000/8290 | Cost: 0.0201\n",
      "Epoch: 012/050 | Batch 0500/8290 | Cost: 0.0190\n",
      "Epoch: 012/050 | Batch 1000/8290 | Cost: 0.0190\n",
      "Epoch: 012/050 | Batch 1500/8290 | Cost: 0.0200\n",
      "Epoch: 012/050 | Batch 2000/8290 | Cost: 0.0216\n",
      "Epoch: 012/050 | Batch 2500/8290 | Cost: 0.0199\n",
      "Epoch: 012/050 | Batch 3000/8290 | Cost: 0.0206\n",
      "Epoch: 012/050 | Batch 3500/8290 | Cost: 0.0199\n",
      "Epoch: 012/050 | Batch 4000/8290 | Cost: 0.0202\n",
      "Epoch: 012/050 | Batch 4500/8290 | Cost: 0.0208\n",
      "Epoch: 012/050 | Batch 5000/8290 | Cost: 0.0201\n",
      "Epoch: 012/050 | Batch 5500/8290 | Cost: 0.0206\n",
      "Epoch: 012/050 | Batch 6000/8290 | Cost: 0.0192\n",
      "Epoch: 012/050 | Batch 6500/8290 | Cost: 0.0210\n",
      "Epoch: 012/050 | Batch 7000/8290 | Cost: 0.0228\n",
      "Epoch: 012/050 | Batch 7500/8290 | Cost: 0.0200\n",
      "Epoch: 012/050 | Batch 8000/8290 | Cost: 0.0191\n",
      "Time elapsed: 11.33 min\n",
      "Epoch: 013/050 | Batch 0000/8290 | Cost: 0.0196\n",
      "Epoch: 013/050 | Batch 0500/8290 | Cost: 0.0193\n",
      "Epoch: 013/050 | Batch 1000/8290 | Cost: 0.0188\n",
      "Epoch: 013/050 | Batch 1500/8290 | Cost: 0.0187\n",
      "Epoch: 013/050 | Batch 2000/8290 | Cost: 0.0205\n",
      "Epoch: 013/050 | Batch 2500/8290 | Cost: 0.0195\n",
      "Epoch: 013/050 | Batch 3000/8290 | Cost: 0.0201\n",
      "Epoch: 013/050 | Batch 3500/8290 | Cost: 0.0216\n",
      "Epoch: 013/050 | Batch 4000/8290 | Cost: 0.0205\n",
      "Epoch: 013/050 | Batch 4500/8290 | Cost: 0.0194\n",
      "Epoch: 013/050 | Batch 5000/8290 | Cost: 0.0202\n",
      "Epoch: 013/050 | Batch 5500/8290 | Cost: 0.0213\n",
      "Epoch: 013/050 | Batch 6000/8290 | Cost: 0.0186\n",
      "Epoch: 013/050 | Batch 6500/8290 | Cost: 0.0195\n",
      "Epoch: 013/050 | Batch 7000/8290 | Cost: 0.0181\n",
      "Epoch: 013/050 | Batch 7500/8290 | Cost: 0.0189\n",
      "Epoch: 013/050 | Batch 8000/8290 | Cost: 0.0190\n",
      "Time elapsed: 12.27 min\n",
      "Epoch: 014/050 | Batch 0000/8290 | Cost: 0.0192\n",
      "Epoch: 014/050 | Batch 0500/8290 | Cost: 0.0198\n",
      "Epoch: 014/050 | Batch 1000/8290 | Cost: 0.0209\n",
      "Epoch: 014/050 | Batch 1500/8290 | Cost: 0.0188\n",
      "Epoch: 014/050 | Batch 2000/8290 | Cost: 0.0200\n",
      "Epoch: 014/050 | Batch 2500/8290 | Cost: 0.0190\n",
      "Epoch: 014/050 | Batch 3000/8290 | Cost: 0.0200\n",
      "Epoch: 014/050 | Batch 3500/8290 | Cost: 0.0203\n",
      "Epoch: 014/050 | Batch 4000/8290 | Cost: 0.0205\n",
      "Epoch: 014/050 | Batch 4500/8290 | Cost: 0.0184\n",
      "Epoch: 014/050 | Batch 5000/8290 | Cost: 0.0194\n",
      "Epoch: 014/050 | Batch 5500/8290 | Cost: 0.0193\n",
      "Epoch: 014/050 | Batch 6000/8290 | Cost: 0.0195\n",
      "Epoch: 014/050 | Batch 6500/8290 | Cost: 0.0195\n",
      "Epoch: 014/050 | Batch 7000/8290 | Cost: 0.0214\n",
      "Epoch: 014/050 | Batch 7500/8290 | Cost: 0.0189\n",
      "Epoch: 014/050 | Batch 8000/8290 | Cost: 0.0201\n",
      "Time elapsed: 13.20 min\n",
      "Epoch: 015/050 | Batch 0000/8290 | Cost: 0.0200\n",
      "Epoch: 015/050 | Batch 0500/8290 | Cost: 0.0198\n",
      "Epoch: 015/050 | Batch 1000/8290 | Cost: 0.0181\n",
      "Epoch: 015/050 | Batch 1500/8290 | Cost: 0.0196\n",
      "Epoch: 015/050 | Batch 2000/8290 | Cost: 0.0204\n",
      "Epoch: 015/050 | Batch 2500/8290 | Cost: 0.0186\n",
      "Epoch: 015/050 | Batch 3000/8290 | Cost: 0.0201\n",
      "Epoch: 015/050 | Batch 3500/8290 | Cost: 0.0201\n",
      "Epoch: 015/050 | Batch 4000/8290 | Cost: 0.0193\n",
      "Epoch: 015/050 | Batch 4500/8290 | Cost: 0.0197\n",
      "Epoch: 015/050 | Batch 5000/8290 | Cost: 0.0181\n",
      "Epoch: 015/050 | Batch 5500/8290 | Cost: 0.0194\n",
      "Epoch: 015/050 | Batch 6000/8290 | Cost: 0.0190\n",
      "Epoch: 015/050 | Batch 6500/8290 | Cost: 0.0202\n",
      "Epoch: 015/050 | Batch 7000/8290 | Cost: 0.0188\n",
      "Epoch: 015/050 | Batch 7500/8290 | Cost: 0.0208\n",
      "Epoch: 015/050 | Batch 8000/8290 | Cost: 0.0185\n",
      "Time elapsed: 14.14 min\n",
      "Epoch: 016/050 | Batch 0000/8290 | Cost: 0.0186\n",
      "Epoch: 016/050 | Batch 0500/8290 | Cost: 0.0194\n",
      "Epoch: 016/050 | Batch 1000/8290 | Cost: 0.0217\n",
      "Epoch: 016/050 | Batch 1500/8290 | Cost: 0.0194\n",
      "Epoch: 016/050 | Batch 2000/8290 | Cost: 0.0180\n",
      "Epoch: 016/050 | Batch 2500/8290 | Cost: 0.0194\n",
      "Epoch: 016/050 | Batch 3000/8290 | Cost: 0.0197\n",
      "Epoch: 016/050 | Batch 3500/8290 | Cost: 0.0183\n",
      "Epoch: 016/050 | Batch 4000/8290 | Cost: 0.0198\n",
      "Epoch: 016/050 | Batch 4500/8290 | Cost: 0.0193\n",
      "Epoch: 016/050 | Batch 5000/8290 | Cost: 0.0198\n",
      "Epoch: 016/050 | Batch 5500/8290 | Cost: 0.0177\n",
      "Epoch: 016/050 | Batch 6000/8290 | Cost: 0.0195\n",
      "Epoch: 016/050 | Batch 6500/8290 | Cost: 0.0203\n",
      "Epoch: 016/050 | Batch 7000/8290 | Cost: 0.0181\n",
      "Epoch: 016/050 | Batch 7500/8290 | Cost: 0.0196\n",
      "Epoch: 016/050 | Batch 8000/8290 | Cost: 0.0186\n",
      "Time elapsed: 15.08 min\n",
      "Epoch: 017/050 | Batch 0000/8290 | Cost: 0.0179\n",
      "Epoch: 017/050 | Batch 0500/8290 | Cost: 0.0200\n",
      "Epoch: 017/050 | Batch 1000/8290 | Cost: 0.0202\n",
      "Epoch: 017/050 | Batch 1500/8290 | Cost: 0.0198\n",
      "Epoch: 017/050 | Batch 2000/8290 | Cost: 0.0199\n",
      "Epoch: 017/050 | Batch 2500/8290 | Cost: 0.0201\n",
      "Epoch: 017/050 | Batch 3000/8290 | Cost: 0.0195\n",
      "Epoch: 017/050 | Batch 3500/8290 | Cost: 0.0193\n",
      "Epoch: 017/050 | Batch 4000/8290 | Cost: 0.0191\n",
      "Epoch: 017/050 | Batch 4500/8290 | Cost: 0.0205\n",
      "Epoch: 017/050 | Batch 5000/8290 | Cost: 0.0200\n",
      "Epoch: 017/050 | Batch 5500/8290 | Cost: 0.0200\n",
      "Epoch: 017/050 | Batch 6000/8290 | Cost: 0.0202\n",
      "Epoch: 017/050 | Batch 6500/8290 | Cost: 0.0193\n",
      "Epoch: 017/050 | Batch 7000/8290 | Cost: 0.0192\n",
      "Epoch: 017/050 | Batch 7500/8290 | Cost: 0.0207\n",
      "Epoch: 017/050 | Batch 8000/8290 | Cost: 0.0183\n",
      "Time elapsed: 16.02 min\n",
      "Epoch: 018/050 | Batch 0000/8290 | Cost: 0.0199\n",
      "Epoch: 018/050 | Batch 0500/8290 | Cost: 0.0202\n",
      "Epoch: 018/050 | Batch 1000/8290 | Cost: 0.0203\n",
      "Epoch: 018/050 | Batch 1500/8290 | Cost: 0.0193\n",
      "Epoch: 018/050 | Batch 2000/8290 | Cost: 0.0190\n",
      "Epoch: 018/050 | Batch 2500/8290 | Cost: 0.0205\n",
      "Epoch: 018/050 | Batch 3000/8290 | Cost: 0.0210\n",
      "Epoch: 018/050 | Batch 3500/8290 | Cost: 0.0184\n",
      "Epoch: 018/050 | Batch 4000/8290 | Cost: 0.0197\n",
      "Epoch: 018/050 | Batch 4500/8290 | Cost: 0.0206\n",
      "Epoch: 018/050 | Batch 5000/8290 | Cost: 0.0173\n",
      "Epoch: 018/050 | Batch 5500/8290 | Cost: 0.0177\n",
      "Epoch: 018/050 | Batch 6000/8290 | Cost: 0.0198\n",
      "Epoch: 018/050 | Batch 6500/8290 | Cost: 0.0200\n",
      "Epoch: 018/050 | Batch 7000/8290 | Cost: 0.0203\n",
      "Epoch: 018/050 | Batch 7500/8290 | Cost: 0.0199\n",
      "Epoch: 018/050 | Batch 8000/8290 | Cost: 0.0193\n",
      "Time elapsed: 16.96 min\n",
      "Epoch: 019/050 | Batch 0000/8290 | Cost: 0.0193\n",
      "Epoch: 019/050 | Batch 0500/8290 | Cost: 0.0193\n",
      "Epoch: 019/050 | Batch 1000/8290 | Cost: 0.0202\n",
      "Epoch: 019/050 | Batch 1500/8290 | Cost: 0.0210\n",
      "Epoch: 019/050 | Batch 2000/8290 | Cost: 0.0193\n",
      "Epoch: 019/050 | Batch 2500/8290 | Cost: 0.0193\n",
      "Epoch: 019/050 | Batch 3000/8290 | Cost: 0.0197\n",
      "Epoch: 019/050 | Batch 3500/8290 | Cost: 0.0197\n",
      "Epoch: 019/050 | Batch 4000/8290 | Cost: 0.0208\n",
      "Epoch: 019/050 | Batch 4500/8290 | Cost: 0.0199\n",
      "Epoch: 019/050 | Batch 5000/8290 | Cost: 0.0191\n",
      "Epoch: 019/050 | Batch 5500/8290 | Cost: 0.0204\n",
      "Epoch: 019/050 | Batch 6000/8290 | Cost: 0.0208\n",
      "Epoch: 019/050 | Batch 6500/8290 | Cost: 0.0207\n",
      "Epoch: 019/050 | Batch 7000/8290 | Cost: 0.0184\n",
      "Epoch: 019/050 | Batch 7500/8290 | Cost: 0.0182\n",
      "Epoch: 019/050 | Batch 8000/8290 | Cost: 0.0203\n",
      "Time elapsed: 17.91 min\n",
      "Epoch: 020/050 | Batch 0000/8290 | Cost: 0.0189\n",
      "Epoch: 020/050 | Batch 0500/8290 | Cost: 0.0190\n",
      "Epoch: 020/050 | Batch 1000/8290 | Cost: 0.0204\n",
      "Epoch: 020/050 | Batch 1500/8290 | Cost: 0.0174\n",
      "Epoch: 020/050 | Batch 2000/8290 | Cost: 0.0205\n",
      "Epoch: 020/050 | Batch 2500/8290 | Cost: 0.0191\n",
      "Epoch: 020/050 | Batch 3000/8290 | Cost: 0.0187\n",
      "Epoch: 020/050 | Batch 3500/8290 | Cost: 0.0192\n",
      "Epoch: 020/050 | Batch 4000/8290 | Cost: 0.0207\n",
      "Epoch: 020/050 | Batch 4500/8290 | Cost: 0.0183\n",
      "Epoch: 020/050 | Batch 5000/8290 | Cost: 0.0191\n",
      "Epoch: 020/050 | Batch 5500/8290 | Cost: 0.0202\n",
      "Epoch: 020/050 | Batch 6000/8290 | Cost: 0.0208\n",
      "Epoch: 020/050 | Batch 6500/8290 | Cost: 0.0211\n",
      "Epoch: 020/050 | Batch 7000/8290 | Cost: 0.0202\n",
      "Epoch: 020/050 | Batch 7500/8290 | Cost: 0.0199\n",
      "Epoch: 020/050 | Batch 8000/8290 | Cost: 0.0184\n",
      "Time elapsed: 18.85 min\n",
      "Epoch: 021/050 | Batch 0000/8290 | Cost: 0.0188\n",
      "Epoch: 021/050 | Batch 0500/8290 | Cost: 0.0208\n",
      "Epoch: 021/050 | Batch 1000/8290 | Cost: 0.0190\n",
      "Epoch: 021/050 | Batch 1500/8290 | Cost: 0.0204\n",
      "Epoch: 021/050 | Batch 2000/8290 | Cost: 0.0203\n",
      "Epoch: 021/050 | Batch 2500/8290 | Cost: 0.0188\n",
      "Epoch: 021/050 | Batch 3000/8290 | Cost: 0.0205\n",
      "Epoch: 021/050 | Batch 3500/8290 | Cost: 0.0188\n",
      "Epoch: 021/050 | Batch 4000/8290 | Cost: 0.0195\n",
      "Epoch: 021/050 | Batch 4500/8290 | Cost: 0.0199\n",
      "Epoch: 021/050 | Batch 5000/8290 | Cost: 0.0198\n",
      "Epoch: 021/050 | Batch 5500/8290 | Cost: 0.0208\n",
      "Epoch: 021/050 | Batch 6000/8290 | Cost: 0.0195\n",
      "Epoch: 021/050 | Batch 6500/8290 | Cost: 0.0194\n",
      "Epoch: 021/050 | Batch 7000/8290 | Cost: 0.0187\n",
      "Epoch: 021/050 | Batch 7500/8290 | Cost: 0.0175\n",
      "Epoch: 021/050 | Batch 8000/8290 | Cost: 0.0195\n",
      "Time elapsed: 19.79 min\n",
      "Epoch: 022/050 | Batch 0000/8290 | Cost: 0.0184\n",
      "Epoch: 022/050 | Batch 0500/8290 | Cost: 0.0184\n",
      "Epoch: 022/050 | Batch 1000/8290 | Cost: 0.0206\n",
      "Epoch: 022/050 | Batch 1500/8290 | Cost: 0.0195\n",
      "Epoch: 022/050 | Batch 2000/8290 | Cost: 0.0192\n",
      "Epoch: 022/050 | Batch 2500/8290 | Cost: 0.0184\n",
      "Epoch: 022/050 | Batch 3000/8290 | Cost: 0.0208\n",
      "Epoch: 022/050 | Batch 3500/8290 | Cost: 0.0194\n",
      "Epoch: 022/050 | Batch 4000/8290 | Cost: 0.0206\n",
      "Epoch: 022/050 | Batch 4500/8290 | Cost: 0.0184\n",
      "Epoch: 022/050 | Batch 5000/8290 | Cost: 0.0201\n",
      "Epoch: 022/050 | Batch 5500/8290 | Cost: 0.0196\n",
      "Epoch: 022/050 | Batch 6000/8290 | Cost: 0.0192\n",
      "Epoch: 022/050 | Batch 6500/8290 | Cost: 0.0196\n",
      "Epoch: 022/050 | Batch 7000/8290 | Cost: 0.0198\n",
      "Epoch: 022/050 | Batch 7500/8290 | Cost: 0.0183\n",
      "Epoch: 022/050 | Batch 8000/8290 | Cost: 0.0199\n",
      "Time elapsed: 20.72 min\n",
      "Epoch: 023/050 | Batch 0000/8290 | Cost: 0.0206\n",
      "Epoch: 023/050 | Batch 0500/8290 | Cost: 0.0210\n",
      "Epoch: 023/050 | Batch 1000/8290 | Cost: 0.0193\n",
      "Epoch: 023/050 | Batch 1500/8290 | Cost: 0.0186\n",
      "Epoch: 023/050 | Batch 2000/8290 | Cost: 0.0180\n",
      "Epoch: 023/050 | Batch 2500/8290 | Cost: 0.0201\n",
      "Epoch: 023/050 | Batch 3000/8290 | Cost: 0.0192\n",
      "Epoch: 023/050 | Batch 3500/8290 | Cost: 0.0187\n",
      "Epoch: 023/050 | Batch 4000/8290 | Cost: 0.0195\n",
      "Epoch: 023/050 | Batch 4500/8290 | Cost: 0.0174\n",
      "Epoch: 023/050 | Batch 5000/8290 | Cost: 0.0199\n",
      "Epoch: 023/050 | Batch 5500/8290 | Cost: 0.0193\n",
      "Epoch: 023/050 | Batch 6000/8290 | Cost: 0.0183\n",
      "Epoch: 023/050 | Batch 6500/8290 | Cost: 0.0193\n",
      "Epoch: 023/050 | Batch 7000/8290 | Cost: 0.0182\n",
      "Epoch: 023/050 | Batch 7500/8290 | Cost: 0.0186\n",
      "Epoch: 023/050 | Batch 8000/8290 | Cost: 0.0208\n",
      "Time elapsed: 21.65 min\n",
      "Epoch: 024/050 | Batch 0000/8290 | Cost: 0.0189\n",
      "Epoch: 024/050 | Batch 0500/8290 | Cost: 0.0189\n",
      "Epoch: 024/050 | Batch 1000/8290 | Cost: 0.0191\n",
      "Epoch: 024/050 | Batch 1500/8290 | Cost: 0.0193\n",
      "Epoch: 024/050 | Batch 2000/8290 | Cost: 0.0192\n",
      "Epoch: 024/050 | Batch 2500/8290 | Cost: 0.0193\n",
      "Epoch: 024/050 | Batch 3000/8290 | Cost: 0.0201\n",
      "Epoch: 024/050 | Batch 3500/8290 | Cost: 0.0193\n",
      "Epoch: 024/050 | Batch 4000/8290 | Cost: 0.0201\n",
      "Epoch: 024/050 | Batch 4500/8290 | Cost: 0.0205\n",
      "Epoch: 024/050 | Batch 5000/8290 | Cost: 0.0193\n",
      "Epoch: 024/050 | Batch 5500/8290 | Cost: 0.0186\n",
      "Epoch: 024/050 | Batch 6000/8290 | Cost: 0.0187\n",
      "Epoch: 024/050 | Batch 6500/8290 | Cost: 0.0186\n",
      "Epoch: 024/050 | Batch 7000/8290 | Cost: 0.0191\n",
      "Epoch: 024/050 | Batch 7500/8290 | Cost: 0.0202\n",
      "Epoch: 024/050 | Batch 8000/8290 | Cost: 0.0172\n",
      "Time elapsed: 22.58 min\n",
      "Epoch: 025/050 | Batch 0000/8290 | Cost: 0.0195\n",
      "Epoch: 025/050 | Batch 0500/8290 | Cost: 0.0203\n",
      "Epoch: 025/050 | Batch 1000/8290 | Cost: 0.0179\n",
      "Epoch: 025/050 | Batch 1500/8290 | Cost: 0.0182\n",
      "Epoch: 025/050 | Batch 2000/8290 | Cost: 0.0195\n",
      "Epoch: 025/050 | Batch 2500/8290 | Cost: 0.0181\n",
      "Epoch: 025/050 | Batch 3000/8290 | Cost: 0.0191\n",
      "Epoch: 025/050 | Batch 3500/8290 | Cost: 0.0181\n",
      "Epoch: 025/050 | Batch 4000/8290 | Cost: 0.0189\n",
      "Epoch: 025/050 | Batch 4500/8290 | Cost: 0.0199\n",
      "Epoch: 025/050 | Batch 5000/8290 | Cost: 0.0211\n",
      "Epoch: 025/050 | Batch 5500/8290 | Cost: 0.0176\n",
      "Epoch: 025/050 | Batch 6000/8290 | Cost: 0.0179\n",
      "Epoch: 025/050 | Batch 6500/8290 | Cost: 0.0186\n",
      "Epoch: 025/050 | Batch 7000/8290 | Cost: 0.0182\n",
      "Epoch: 025/050 | Batch 7500/8290 | Cost: 0.0174\n",
      "Epoch: 025/050 | Batch 8000/8290 | Cost: 0.0187\n",
      "Time elapsed: 23.52 min\n",
      "Epoch: 026/050 | Batch 0000/8290 | Cost: 0.0196\n",
      "Epoch: 026/050 | Batch 0500/8290 | Cost: 0.0174\n",
      "Epoch: 026/050 | Batch 1000/8290 | Cost: 0.0183\n",
      "Epoch: 026/050 | Batch 1500/8290 | Cost: 0.0193\n",
      "Epoch: 026/050 | Batch 2000/8290 | Cost: 0.0183\n",
      "Epoch: 026/050 | Batch 2500/8290 | Cost: 0.0173\n",
      "Epoch: 026/050 | Batch 3000/8290 | Cost: 0.0190\n",
      "Epoch: 026/050 | Batch 3500/8290 | Cost: 0.0181\n",
      "Epoch: 026/050 | Batch 4000/8290 | Cost: 0.0190\n",
      "Epoch: 026/050 | Batch 4500/8290 | Cost: 0.0193\n",
      "Epoch: 026/050 | Batch 5000/8290 | Cost: 0.0194\n",
      "Epoch: 026/050 | Batch 5500/8290 | Cost: 0.0196\n",
      "Epoch: 026/050 | Batch 6000/8290 | Cost: 0.0193\n",
      "Epoch: 026/050 | Batch 6500/8290 | Cost: 0.0187\n",
      "Epoch: 026/050 | Batch 7000/8290 | Cost: 0.0175\n",
      "Epoch: 026/050 | Batch 7500/8290 | Cost: 0.0176\n",
      "Epoch: 026/050 | Batch 8000/8290 | Cost: 0.0177\n",
      "Time elapsed: 24.45 min\n",
      "Epoch: 027/050 | Batch 0000/8290 | Cost: 0.0190\n",
      "Epoch: 027/050 | Batch 0500/8290 | Cost: 0.0180\n",
      "Epoch: 027/050 | Batch 1000/8290 | Cost: 0.0183\n",
      "Epoch: 027/050 | Batch 1500/8290 | Cost: 0.0182\n",
      "Epoch: 027/050 | Batch 2000/8290 | Cost: 0.0194\n",
      "Epoch: 027/050 | Batch 2500/8290 | Cost: 0.0192\n",
      "Epoch: 027/050 | Batch 3000/8290 | Cost: 0.0196\n",
      "Epoch: 027/050 | Batch 3500/8290 | Cost: 0.0183\n",
      "Epoch: 027/050 | Batch 4000/8290 | Cost: 0.0191\n",
      "Epoch: 027/050 | Batch 4500/8290 | Cost: 0.0177\n",
      "Epoch: 027/050 | Batch 5000/8290 | Cost: 0.0178\n",
      "Epoch: 027/050 | Batch 5500/8290 | Cost: 0.0198\n",
      "Epoch: 027/050 | Batch 6000/8290 | Cost: 0.0205\n",
      "Epoch: 027/050 | Batch 6500/8290 | Cost: 0.0192\n",
      "Epoch: 027/050 | Batch 7000/8290 | Cost: 0.0190\n",
      "Epoch: 027/050 | Batch 7500/8290 | Cost: 0.0194\n",
      "Epoch: 027/050 | Batch 8000/8290 | Cost: 0.0176\n",
      "Time elapsed: 25.39 min\n",
      "Epoch: 028/050 | Batch 0000/8290 | Cost: 0.0173\n",
      "Epoch: 028/050 | Batch 0500/8290 | Cost: 0.0178\n",
      "Epoch: 028/050 | Batch 1000/8290 | Cost: 0.0179\n",
      "Epoch: 028/050 | Batch 1500/8290 | Cost: 0.0169\n",
      "Epoch: 028/050 | Batch 2000/8290 | Cost: 0.0173\n",
      "Epoch: 028/050 | Batch 2500/8290 | Cost: 0.0184\n",
      "Epoch: 028/050 | Batch 3000/8290 | Cost: 0.0185\n",
      "Epoch: 028/050 | Batch 3500/8290 | Cost: 0.0183\n",
      "Epoch: 028/050 | Batch 4000/8290 | Cost: 0.0183\n",
      "Epoch: 028/050 | Batch 4500/8290 | Cost: 0.0174\n",
      "Epoch: 028/050 | Batch 5000/8290 | Cost: 0.0191\n",
      "Epoch: 028/050 | Batch 5500/8290 | Cost: 0.0187\n",
      "Epoch: 028/050 | Batch 6000/8290 | Cost: 0.0171\n",
      "Epoch: 028/050 | Batch 6500/8290 | Cost: 0.0179\n",
      "Epoch: 028/050 | Batch 7000/8290 | Cost: 0.0200\n",
      "Epoch: 028/050 | Batch 7500/8290 | Cost: 0.0189\n",
      "Epoch: 028/050 | Batch 8000/8290 | Cost: 0.0198\n",
      "Time elapsed: 26.33 min\n",
      "Epoch: 029/050 | Batch 0000/8290 | Cost: 0.0195\n",
      "Epoch: 029/050 | Batch 0500/8290 | Cost: 0.0179\n",
      "Epoch: 029/050 | Batch 1000/8290 | Cost: 0.0185\n",
      "Epoch: 029/050 | Batch 1500/8290 | Cost: 0.0184\n",
      "Epoch: 029/050 | Batch 2000/8290 | Cost: 0.0191\n",
      "Epoch: 029/050 | Batch 2500/8290 | Cost: 0.0190\n",
      "Epoch: 029/050 | Batch 3000/8290 | Cost: 0.0183\n",
      "Epoch: 029/050 | Batch 3500/8290 | Cost: 0.0188\n",
      "Epoch: 029/050 | Batch 4000/8290 | Cost: 0.0183\n",
      "Epoch: 029/050 | Batch 4500/8290 | Cost: 0.0181\n",
      "Epoch: 029/050 | Batch 5000/8290 | Cost: 0.0195\n",
      "Epoch: 029/050 | Batch 5500/8290 | Cost: 0.0176\n",
      "Epoch: 029/050 | Batch 6000/8290 | Cost: 0.0183\n",
      "Epoch: 029/050 | Batch 6500/8290 | Cost: 0.0169\n",
      "Epoch: 029/050 | Batch 7000/8290 | Cost: 0.0183\n",
      "Epoch: 029/050 | Batch 7500/8290 | Cost: 0.0181\n",
      "Epoch: 029/050 | Batch 8000/8290 | Cost: 0.0192\n",
      "Time elapsed: 27.27 min\n",
      "Epoch: 030/050 | Batch 0000/8290 | Cost: 0.0165\n",
      "Epoch: 030/050 | Batch 0500/8290 | Cost: 0.0184\n",
      "Epoch: 030/050 | Batch 1000/8290 | Cost: 0.0184\n",
      "Epoch: 030/050 | Batch 1500/8290 | Cost: 0.0187\n",
      "Epoch: 030/050 | Batch 2000/8290 | Cost: 0.0178\n",
      "Epoch: 030/050 | Batch 2500/8290 | Cost: 0.0185\n",
      "Epoch: 030/050 | Batch 3000/8290 | Cost: 0.0199\n",
      "Epoch: 030/050 | Batch 3500/8290 | Cost: 0.0181\n",
      "Epoch: 030/050 | Batch 4000/8290 | Cost: 0.0192\n",
      "Epoch: 030/050 | Batch 4500/8290 | Cost: 0.0199\n",
      "Epoch: 030/050 | Batch 5000/8290 | Cost: 0.0198\n",
      "Epoch: 030/050 | Batch 5500/8290 | Cost: 0.0184\n",
      "Epoch: 030/050 | Batch 6000/8290 | Cost: 0.0170\n",
      "Epoch: 030/050 | Batch 6500/8290 | Cost: 0.0184\n",
      "Epoch: 030/050 | Batch 7000/8290 | Cost: 0.0182\n",
      "Epoch: 030/050 | Batch 7500/8290 | Cost: 0.0184\n",
      "Epoch: 030/050 | Batch 8000/8290 | Cost: 0.0175\n",
      "Time elapsed: 28.21 min\n",
      "Epoch: 031/050 | Batch 0000/8290 | Cost: 0.0193\n",
      "Epoch: 031/050 | Batch 0500/8290 | Cost: 0.0169\n",
      "Epoch: 031/050 | Batch 1000/8290 | Cost: 0.0184\n",
      "Epoch: 031/050 | Batch 1500/8290 | Cost: 0.0187\n",
      "Epoch: 031/050 | Batch 2000/8290 | Cost: 0.0184\n",
      "Epoch: 031/050 | Batch 2500/8290 | Cost: 0.0158\n",
      "Epoch: 031/050 | Batch 3000/8290 | Cost: 0.0178\n",
      "Epoch: 031/050 | Batch 3500/8290 | Cost: 0.0203\n",
      "Epoch: 031/050 | Batch 4000/8290 | Cost: 0.0188\n",
      "Epoch: 031/050 | Batch 4500/8290 | Cost: 0.0180\n",
      "Epoch: 031/050 | Batch 5000/8290 | Cost: 0.0179\n",
      "Epoch: 031/050 | Batch 5500/8290 | Cost: 0.0192\n",
      "Epoch: 031/050 | Batch 6000/8290 | Cost: 0.0191\n",
      "Epoch: 031/050 | Batch 6500/8290 | Cost: 0.0182\n",
      "Epoch: 031/050 | Batch 7000/8290 | Cost: 0.0199\n",
      "Epoch: 031/050 | Batch 7500/8290 | Cost: 0.0181\n",
      "Epoch: 031/050 | Batch 8000/8290 | Cost: 0.0184\n",
      "Time elapsed: 29.14 min\n",
      "Epoch: 032/050 | Batch 0000/8290 | Cost: 0.0186\n",
      "Epoch: 032/050 | Batch 0500/8290 | Cost: 0.0187\n",
      "Epoch: 032/050 | Batch 1000/8290 | Cost: 0.0193\n",
      "Epoch: 032/050 | Batch 1500/8290 | Cost: 0.0189\n",
      "Epoch: 032/050 | Batch 2000/8290 | Cost: 0.0189\n",
      "Epoch: 032/050 | Batch 2500/8290 | Cost: 0.0200\n",
      "Epoch: 032/050 | Batch 3000/8290 | Cost: 0.0188\n",
      "Epoch: 032/050 | Batch 3500/8290 | Cost: 0.0195\n",
      "Epoch: 032/050 | Batch 4000/8290 | Cost: 0.0190\n",
      "Epoch: 032/050 | Batch 4500/8290 | Cost: 0.0182\n",
      "Epoch: 032/050 | Batch 5000/8290 | Cost: 0.0155\n",
      "Epoch: 032/050 | Batch 5500/8290 | Cost: 0.0218\n",
      "Epoch: 032/050 | Batch 6000/8290 | Cost: 0.0190\n",
      "Epoch: 032/050 | Batch 6500/8290 | Cost: 0.0186\n",
      "Epoch: 032/050 | Batch 7000/8290 | Cost: 0.0176\n",
      "Epoch: 032/050 | Batch 7500/8290 | Cost: 0.0197\n",
      "Epoch: 032/050 | Batch 8000/8290 | Cost: 0.0190\n",
      "Time elapsed: 30.07 min\n",
      "Epoch: 033/050 | Batch 0000/8290 | Cost: 0.0188\n",
      "Epoch: 033/050 | Batch 0500/8290 | Cost: 0.0182\n",
      "Epoch: 033/050 | Batch 1000/8290 | Cost: 0.0189\n",
      "Epoch: 033/050 | Batch 1500/8290 | Cost: 0.0172\n",
      "Epoch: 033/050 | Batch 2000/8290 | Cost: 0.0174\n",
      "Epoch: 033/050 | Batch 2500/8290 | Cost: 0.0203\n",
      "Epoch: 033/050 | Batch 3000/8290 | Cost: 0.0205\n",
      "Epoch: 033/050 | Batch 3500/8290 | Cost: 0.0194\n",
      "Epoch: 033/050 | Batch 4000/8290 | Cost: 0.0177\n",
      "Epoch: 033/050 | Batch 4500/8290 | Cost: 0.0192\n",
      "Epoch: 033/050 | Batch 5000/8290 | Cost: 0.0192\n",
      "Epoch: 033/050 | Batch 5500/8290 | Cost: 0.0196\n",
      "Epoch: 033/050 | Batch 6000/8290 | Cost: 0.0180\n",
      "Epoch: 033/050 | Batch 6500/8290 | Cost: 0.0196\n",
      "Epoch: 033/050 | Batch 7000/8290 | Cost: 0.0186\n",
      "Epoch: 033/050 | Batch 7500/8290 | Cost: 0.0180\n",
      "Epoch: 033/050 | Batch 8000/8290 | Cost: 0.0168\n",
      "Time elapsed: 31.01 min\n",
      "Epoch: 034/050 | Batch 0000/8290 | Cost: 0.0188\n",
      "Epoch: 034/050 | Batch 0500/8290 | Cost: 0.0181\n",
      "Epoch: 034/050 | Batch 1000/8290 | Cost: 0.0186\n",
      "Epoch: 034/050 | Batch 1500/8290 | Cost: 0.0177\n",
      "Epoch: 034/050 | Batch 2000/8290 | Cost: 0.0168\n",
      "Epoch: 034/050 | Batch 2500/8290 | Cost: 0.0187\n",
      "Epoch: 034/050 | Batch 3000/8290 | Cost: 0.0193\n",
      "Epoch: 034/050 | Batch 3500/8290 | Cost: 0.0191\n",
      "Epoch: 034/050 | Batch 4000/8290 | Cost: 0.0175\n",
      "Epoch: 034/050 | Batch 4500/8290 | Cost: 0.0169\n",
      "Epoch: 034/050 | Batch 5000/8290 | Cost: 0.0172\n",
      "Epoch: 034/050 | Batch 5500/8290 | Cost: 0.0202\n",
      "Epoch: 034/050 | Batch 6000/8290 | Cost: 0.0190\n",
      "Epoch: 034/050 | Batch 6500/8290 | Cost: 0.0175\n",
      "Epoch: 034/050 | Batch 7000/8290 | Cost: 0.0178\n",
      "Epoch: 034/050 | Batch 7500/8290 | Cost: 0.0184\n",
      "Epoch: 034/050 | Batch 8000/8290 | Cost: 0.0176\n",
      "Time elapsed: 31.94 min\n",
      "Epoch: 035/050 | Batch 0000/8290 | Cost: 0.0174\n",
      "Epoch: 035/050 | Batch 0500/8290 | Cost: 0.0176\n",
      "Epoch: 035/050 | Batch 1000/8290 | Cost: 0.0184\n",
      "Epoch: 035/050 | Batch 1500/8290 | Cost: 0.0180\n",
      "Epoch: 035/050 | Batch 2000/8290 | Cost: 0.0208\n",
      "Epoch: 035/050 | Batch 2500/8290 | Cost: 0.0198\n",
      "Epoch: 035/050 | Batch 3000/8290 | Cost: 0.0177\n",
      "Epoch: 035/050 | Batch 3500/8290 | Cost: 0.0174\n",
      "Epoch: 035/050 | Batch 4000/8290 | Cost: 0.0171\n",
      "Epoch: 035/050 | Batch 4500/8290 | Cost: 0.0187\n",
      "Epoch: 035/050 | Batch 5000/8290 | Cost: 0.0175\n",
      "Epoch: 035/050 | Batch 5500/8290 | Cost: 0.0189\n",
      "Epoch: 035/050 | Batch 6000/8290 | Cost: 0.0191\n",
      "Epoch: 035/050 | Batch 6500/8290 | Cost: 0.0177\n",
      "Epoch: 035/050 | Batch 7000/8290 | Cost: 0.0158\n",
      "Epoch: 035/050 | Batch 7500/8290 | Cost: 0.0175\n",
      "Epoch: 035/050 | Batch 8000/8290 | Cost: 0.0190\n",
      "Time elapsed: 32.87 min\n",
      "Epoch: 036/050 | Batch 0000/8290 | Cost: 0.0170\n",
      "Epoch: 036/050 | Batch 0500/8290 | Cost: 0.0188\n",
      "Epoch: 036/050 | Batch 1000/8290 | Cost: 0.0172\n",
      "Epoch: 036/050 | Batch 1500/8290 | Cost: 0.0193\n",
      "Epoch: 036/050 | Batch 2000/8290 | Cost: 0.0168\n",
      "Epoch: 036/050 | Batch 2500/8290 | Cost: 0.0194\n",
      "Epoch: 036/050 | Batch 3000/8290 | Cost: 0.0190\n",
      "Epoch: 036/050 | Batch 3500/8290 | Cost: 0.0189\n",
      "Epoch: 036/050 | Batch 4000/8290 | Cost: 0.0191\n",
      "Epoch: 036/050 | Batch 4500/8290 | Cost: 0.0180\n",
      "Epoch: 036/050 | Batch 5000/8290 | Cost: 0.0202\n",
      "Epoch: 036/050 | Batch 5500/8290 | Cost: 0.0179\n",
      "Epoch: 036/050 | Batch 6000/8290 | Cost: 0.0200\n",
      "Epoch: 036/050 | Batch 6500/8290 | Cost: 0.0175\n",
      "Epoch: 036/050 | Batch 7000/8290 | Cost: 0.0194\n",
      "Epoch: 036/050 | Batch 7500/8290 | Cost: 0.0190\n",
      "Epoch: 036/050 | Batch 8000/8290 | Cost: 0.0189\n",
      "Time elapsed: 33.80 min\n",
      "Epoch: 037/050 | Batch 0000/8290 | Cost: 0.0180\n",
      "Epoch: 037/050 | Batch 0500/8290 | Cost: 0.0181\n",
      "Epoch: 037/050 | Batch 1000/8290 | Cost: 0.0194\n",
      "Epoch: 037/050 | Batch 1500/8290 | Cost: 0.0185\n",
      "Epoch: 037/050 | Batch 2000/8290 | Cost: 0.0190\n",
      "Epoch: 037/050 | Batch 2500/8290 | Cost: 0.0195\n",
      "Epoch: 037/050 | Batch 3000/8290 | Cost: 0.0197\n",
      "Epoch: 037/050 | Batch 3500/8290 | Cost: 0.0174\n",
      "Epoch: 037/050 | Batch 4000/8290 | Cost: 0.0196\n",
      "Epoch: 037/050 | Batch 4500/8290 | Cost: 0.0182\n",
      "Epoch: 037/050 | Batch 5000/8290 | Cost: 0.0174\n",
      "Epoch: 037/050 | Batch 5500/8290 | Cost: 0.0175\n",
      "Epoch: 037/050 | Batch 6000/8290 | Cost: 0.0167\n",
      "Epoch: 037/050 | Batch 6500/8290 | Cost: 0.0161\n",
      "Epoch: 037/050 | Batch 7000/8290 | Cost: 0.0162\n",
      "Epoch: 037/050 | Batch 7500/8290 | Cost: 0.0179\n",
      "Epoch: 037/050 | Batch 8000/8290 | Cost: 0.0196\n",
      "Time elapsed: 34.73 min\n",
      "Epoch: 038/050 | Batch 0000/8290 | Cost: 0.0174\n",
      "Epoch: 038/050 | Batch 0500/8290 | Cost: 0.0184\n",
      "Epoch: 038/050 | Batch 1000/8290 | Cost: 0.0178\n",
      "Epoch: 038/050 | Batch 1500/8290 | Cost: 0.0190\n",
      "Epoch: 038/050 | Batch 2000/8290 | Cost: 0.0182\n",
      "Epoch: 038/050 | Batch 2500/8290 | Cost: 0.0168\n",
      "Epoch: 038/050 | Batch 3000/8290 | Cost: 0.0185\n",
      "Epoch: 038/050 | Batch 3500/8290 | Cost: 0.0178\n",
      "Epoch: 038/050 | Batch 4000/8290 | Cost: 0.0194\n",
      "Epoch: 038/050 | Batch 4500/8290 | Cost: 0.0178\n",
      "Epoch: 038/050 | Batch 5000/8290 | Cost: 0.0193\n",
      "Epoch: 038/050 | Batch 5500/8290 | Cost: 0.0192\n",
      "Epoch: 038/050 | Batch 6000/8290 | Cost: 0.0182\n",
      "Epoch: 038/050 | Batch 6500/8290 | Cost: 0.0182\n",
      "Epoch: 038/050 | Batch 7000/8290 | Cost: 0.0184\n",
      "Epoch: 038/050 | Batch 7500/8290 | Cost: 0.0189\n",
      "Epoch: 038/050 | Batch 8000/8290 | Cost: 0.0176\n",
      "Time elapsed: 35.66 min\n",
      "Epoch: 039/050 | Batch 0000/8290 | Cost: 0.0190\n",
      "Epoch: 039/050 | Batch 0500/8290 | Cost: 0.0171\n",
      "Epoch: 039/050 | Batch 1000/8290 | Cost: 0.0185\n",
      "Epoch: 039/050 | Batch 1500/8290 | Cost: 0.0180\n",
      "Epoch: 039/050 | Batch 2000/8290 | Cost: 0.0193\n",
      "Epoch: 039/050 | Batch 2500/8290 | Cost: 0.0196\n",
      "Epoch: 039/050 | Batch 3000/8290 | Cost: 0.0176\n",
      "Epoch: 039/050 | Batch 3500/8290 | Cost: 0.0185\n",
      "Epoch: 039/050 | Batch 4000/8290 | Cost: 0.0176\n",
      "Epoch: 039/050 | Batch 4500/8290 | Cost: 0.0181\n",
      "Epoch: 039/050 | Batch 5000/8290 | Cost: 0.0189\n",
      "Epoch: 039/050 | Batch 5500/8290 | Cost: 0.0176\n",
      "Epoch: 039/050 | Batch 6000/8290 | Cost: 0.0183\n",
      "Epoch: 039/050 | Batch 6500/8290 | Cost: 0.0191\n",
      "Epoch: 039/050 | Batch 7000/8290 | Cost: 0.0177\n",
      "Epoch: 039/050 | Batch 7500/8290 | Cost: 0.0179\n",
      "Epoch: 039/050 | Batch 8000/8290 | Cost: 0.0168\n",
      "Time elapsed: 36.60 min\n",
      "Epoch: 040/050 | Batch 0000/8290 | Cost: 0.0181\n",
      "Epoch: 040/050 | Batch 0500/8290 | Cost: 0.0184\n",
      "Epoch: 040/050 | Batch 1000/8290 | Cost: 0.0177\n",
      "Epoch: 040/050 | Batch 1500/8290 | Cost: 0.0194\n",
      "Epoch: 040/050 | Batch 2000/8290 | Cost: 0.0176\n",
      "Epoch: 040/050 | Batch 2500/8290 | Cost: 0.0197\n",
      "Epoch: 040/050 | Batch 3000/8290 | Cost: 0.0184\n",
      "Epoch: 040/050 | Batch 3500/8290 | Cost: 0.0170\n",
      "Epoch: 040/050 | Batch 4000/8290 | Cost: 0.0182\n",
      "Epoch: 040/050 | Batch 4500/8290 | Cost: 0.0175\n",
      "Epoch: 040/050 | Batch 5000/8290 | Cost: 0.0194\n",
      "Epoch: 040/050 | Batch 5500/8290 | Cost: 0.0166\n",
      "Epoch: 040/050 | Batch 6000/8290 | Cost: 0.0177\n",
      "Epoch: 040/050 | Batch 6500/8290 | Cost: 0.0177\n",
      "Epoch: 040/050 | Batch 7000/8290 | Cost: 0.0179\n",
      "Epoch: 040/050 | Batch 7500/8290 | Cost: 0.0208\n",
      "Epoch: 040/050 | Batch 8000/8290 | Cost: 0.0184\n",
      "Time elapsed: 37.53 min\n",
      "Epoch: 041/050 | Batch 0000/8290 | Cost: 0.0196\n",
      "Epoch: 041/050 | Batch 0500/8290 | Cost: 0.0179\n",
      "Epoch: 041/050 | Batch 1000/8290 | Cost: 0.0183\n",
      "Epoch: 041/050 | Batch 1500/8290 | Cost: 0.0169\n",
      "Epoch: 041/050 | Batch 2000/8290 | Cost: 0.0189\n",
      "Epoch: 041/050 | Batch 2500/8290 | Cost: 0.0187\n",
      "Epoch: 041/050 | Batch 3000/8290 | Cost: 0.0177\n",
      "Epoch: 041/050 | Batch 3500/8290 | Cost: 0.0182\n",
      "Epoch: 041/050 | Batch 4000/8290 | Cost: 0.0173\n",
      "Epoch: 041/050 | Batch 4500/8290 | Cost: 0.0185\n",
      "Epoch: 041/050 | Batch 5000/8290 | Cost: 0.0168\n",
      "Epoch: 041/050 | Batch 5500/8290 | Cost: 0.0196\n",
      "Epoch: 041/050 | Batch 6000/8290 | Cost: 0.0179\n",
      "Epoch: 041/050 | Batch 6500/8290 | Cost: 0.0171\n",
      "Epoch: 041/050 | Batch 7000/8290 | Cost: 0.0181\n",
      "Epoch: 041/050 | Batch 7500/8290 | Cost: 0.0184\n",
      "Epoch: 041/050 | Batch 8000/8290 | Cost: 0.0187\n",
      "Time elapsed: 38.47 min\n",
      "Epoch: 042/050 | Batch 0000/8290 | Cost: 0.0185\n",
      "Epoch: 042/050 | Batch 0500/8290 | Cost: 0.0175\n",
      "Epoch: 042/050 | Batch 1000/8290 | Cost: 0.0188\n",
      "Epoch: 042/050 | Batch 1500/8290 | Cost: 0.0196\n",
      "Epoch: 042/050 | Batch 2000/8290 | Cost: 0.0185\n",
      "Epoch: 042/050 | Batch 2500/8290 | Cost: 0.0189\n",
      "Epoch: 042/050 | Batch 3000/8290 | Cost: 0.0193\n",
      "Epoch: 042/050 | Batch 3500/8290 | Cost: 0.0185\n",
      "Epoch: 042/050 | Batch 4000/8290 | Cost: 0.0179\n",
      "Epoch: 042/050 | Batch 4500/8290 | Cost: 0.0185\n",
      "Epoch: 042/050 | Batch 5000/8290 | Cost: 0.0170\n",
      "Epoch: 042/050 | Batch 5500/8290 | Cost: 0.0187\n",
      "Epoch: 042/050 | Batch 6000/8290 | Cost: 0.0184\n",
      "Epoch: 042/050 | Batch 6500/8290 | Cost: 0.0185\n",
      "Epoch: 042/050 | Batch 7000/8290 | Cost: 0.0156\n",
      "Epoch: 042/050 | Batch 7500/8290 | Cost: 0.0183\n",
      "Epoch: 042/050 | Batch 8000/8290 | Cost: 0.0186\n",
      "Time elapsed: 39.41 min\n",
      "Epoch: 043/050 | Batch 0000/8290 | Cost: 0.0179\n",
      "Epoch: 043/050 | Batch 0500/8290 | Cost: 0.0180\n",
      "Epoch: 043/050 | Batch 1000/8290 | Cost: 0.0189\n",
      "Epoch: 043/050 | Batch 1500/8290 | Cost: 0.0182\n",
      "Epoch: 043/050 | Batch 2000/8290 | Cost: 0.0167\n",
      "Epoch: 043/050 | Batch 2500/8290 | Cost: 0.0178\n",
      "Epoch: 043/050 | Batch 3000/8290 | Cost: 0.0180\n",
      "Epoch: 043/050 | Batch 3500/8290 | Cost: 0.0175\n",
      "Epoch: 043/050 | Batch 4000/8290 | Cost: 0.0177\n",
      "Epoch: 043/050 | Batch 4500/8290 | Cost: 0.0180\n",
      "Epoch: 043/050 | Batch 5000/8290 | Cost: 0.0174\n",
      "Epoch: 043/050 | Batch 5500/8290 | Cost: 0.0200\n",
      "Epoch: 043/050 | Batch 6000/8290 | Cost: 0.0168\n",
      "Epoch: 043/050 | Batch 6500/8290 | Cost: 0.0186\n",
      "Epoch: 043/050 | Batch 7000/8290 | Cost: 0.0186\n",
      "Epoch: 043/050 | Batch 7500/8290 | Cost: 0.0202\n",
      "Epoch: 043/050 | Batch 8000/8290 | Cost: 0.0189\n",
      "Time elapsed: 40.35 min\n",
      "Epoch: 044/050 | Batch 0000/8290 | Cost: 0.0166\n",
      "Epoch: 044/050 | Batch 0500/8290 | Cost: 0.0181\n",
      "Epoch: 044/050 | Batch 1000/8290 | Cost: 0.0185\n",
      "Epoch: 044/050 | Batch 1500/8290 | Cost: 0.0186\n",
      "Epoch: 044/050 | Batch 2000/8290 | Cost: 0.0168\n",
      "Epoch: 044/050 | Batch 2500/8290 | Cost: 0.0166\n",
      "Epoch: 044/050 | Batch 3000/8290 | Cost: 0.0178\n",
      "Epoch: 044/050 | Batch 3500/8290 | Cost: 0.0176\n",
      "Epoch: 044/050 | Batch 4000/8290 | Cost: 0.0180\n",
      "Epoch: 044/050 | Batch 4500/8290 | Cost: 0.0175\n",
      "Epoch: 044/050 | Batch 5000/8290 | Cost: 0.0186\n",
      "Epoch: 044/050 | Batch 5500/8290 | Cost: 0.0162\n",
      "Epoch: 044/050 | Batch 6000/8290 | Cost: 0.0181\n",
      "Epoch: 044/050 | Batch 6500/8290 | Cost: 0.0183\n",
      "Epoch: 044/050 | Batch 7000/8290 | Cost: 0.0190\n",
      "Epoch: 044/050 | Batch 7500/8290 | Cost: 0.0175\n",
      "Epoch: 044/050 | Batch 8000/8290 | Cost: 0.0174\n",
      "Time elapsed: 41.30 min\n",
      "Epoch: 045/050 | Batch 0000/8290 | Cost: 0.0166\n",
      "Epoch: 045/050 | Batch 0500/8290 | Cost: 0.0186\n",
      "Epoch: 045/050 | Batch 1000/8290 | Cost: 0.0186\n",
      "Epoch: 045/050 | Batch 1500/8290 | Cost: 0.0176\n",
      "Epoch: 045/050 | Batch 2000/8290 | Cost: 0.0182\n",
      "Epoch: 045/050 | Batch 2500/8290 | Cost: 0.0188\n",
      "Epoch: 045/050 | Batch 3000/8290 | Cost: 0.0171\n",
      "Epoch: 045/050 | Batch 3500/8290 | Cost: 0.0171\n",
      "Epoch: 045/050 | Batch 4000/8290 | Cost: 0.0187\n",
      "Epoch: 045/050 | Batch 4500/8290 | Cost: 0.0172\n",
      "Epoch: 045/050 | Batch 5000/8290 | Cost: 0.0181\n",
      "Epoch: 045/050 | Batch 5500/8290 | Cost: 0.0183\n",
      "Epoch: 045/050 | Batch 6000/8290 | Cost: 0.0183\n",
      "Epoch: 045/050 | Batch 6500/8290 | Cost: 0.0190\n",
      "Epoch: 045/050 | Batch 7000/8290 | Cost: 0.0177\n",
      "Epoch: 045/050 | Batch 7500/8290 | Cost: 0.0164\n",
      "Epoch: 045/050 | Batch 8000/8290 | Cost: 0.0183\n",
      "Time elapsed: 42.24 min\n",
      "Epoch: 046/050 | Batch 0000/8290 | Cost: 0.0171\n",
      "Epoch: 046/050 | Batch 0500/8290 | Cost: 0.0174\n",
      "Epoch: 046/050 | Batch 1000/8290 | Cost: 0.0176\n",
      "Epoch: 046/050 | Batch 1500/8290 | Cost: 0.0176\n",
      "Epoch: 046/050 | Batch 2000/8290 | Cost: 0.0174\n",
      "Epoch: 046/050 | Batch 2500/8290 | Cost: 0.0194\n",
      "Epoch: 046/050 | Batch 3000/8290 | Cost: 0.0175\n",
      "Epoch: 046/050 | Batch 3500/8290 | Cost: 0.0184\n",
      "Epoch: 046/050 | Batch 4000/8290 | Cost: 0.0187\n",
      "Epoch: 046/050 | Batch 4500/8290 | Cost: 0.0177\n",
      "Epoch: 046/050 | Batch 5000/8290 | Cost: 0.0184\n",
      "Epoch: 046/050 | Batch 5500/8290 | Cost: 0.0190\n",
      "Epoch: 046/050 | Batch 6000/8290 | Cost: 0.0171\n",
      "Epoch: 046/050 | Batch 6500/8290 | Cost: 0.0184\n",
      "Epoch: 046/050 | Batch 7000/8290 | Cost: 0.0167\n",
      "Epoch: 046/050 | Batch 7500/8290 | Cost: 0.0185\n",
      "Epoch: 046/050 | Batch 8000/8290 | Cost: 0.0188\n",
      "Time elapsed: 43.18 min\n",
      "Epoch: 047/050 | Batch 0000/8290 | Cost: 0.0184\n",
      "Epoch: 047/050 | Batch 0500/8290 | Cost: 0.0167\n",
      "Epoch: 047/050 | Batch 1000/8290 | Cost: 0.0169\n",
      "Epoch: 047/050 | Batch 1500/8290 | Cost: 0.0182\n",
      "Epoch: 047/050 | Batch 2000/8290 | Cost: 0.0191\n",
      "Epoch: 047/050 | Batch 2500/8290 | Cost: 0.0172\n",
      "Epoch: 047/050 | Batch 3000/8290 | Cost: 0.0169\n",
      "Epoch: 047/050 | Batch 3500/8290 | Cost: 0.0183\n",
      "Epoch: 047/050 | Batch 4000/8290 | Cost: 0.0173\n",
      "Epoch: 047/050 | Batch 4500/8290 | Cost: 0.0163\n",
      "Epoch: 047/050 | Batch 5000/8290 | Cost: 0.0165\n",
      "Epoch: 047/050 | Batch 5500/8290 | Cost: 0.0184\n",
      "Epoch: 047/050 | Batch 6000/8290 | Cost: 0.0163\n",
      "Epoch: 047/050 | Batch 6500/8290 | Cost: 0.0181\n",
      "Epoch: 047/050 | Batch 7000/8290 | Cost: 0.0184\n",
      "Epoch: 047/050 | Batch 7500/8290 | Cost: 0.0181\n",
      "Epoch: 047/050 | Batch 8000/8290 | Cost: 0.0180\n",
      "Time elapsed: 44.11 min\n",
      "Epoch: 048/050 | Batch 0000/8290 | Cost: 0.0173\n",
      "Epoch: 048/050 | Batch 0500/8290 | Cost: 0.0166\n",
      "Epoch: 048/050 | Batch 1000/8290 | Cost: 0.0177\n",
      "Epoch: 048/050 | Batch 1500/8290 | Cost: 0.0174\n",
      "Epoch: 048/050 | Batch 2000/8290 | Cost: 0.0188\n",
      "Epoch: 048/050 | Batch 2500/8290 | Cost: 0.0192\n",
      "Epoch: 048/050 | Batch 3000/8290 | Cost: 0.0170\n",
      "Epoch: 048/050 | Batch 3500/8290 | Cost: 0.0170\n",
      "Epoch: 048/050 | Batch 4000/8290 | Cost: 0.0194\n",
      "Epoch: 048/050 | Batch 4500/8290 | Cost: 0.0185\n",
      "Epoch: 048/050 | Batch 5000/8290 | Cost: 0.0180\n",
      "Epoch: 048/050 | Batch 5500/8290 | Cost: 0.0187\n",
      "Epoch: 048/050 | Batch 6000/8290 | Cost: 0.0168\n",
      "Epoch: 048/050 | Batch 6500/8290 | Cost: 0.0199\n",
      "Epoch: 048/050 | Batch 7000/8290 | Cost: 0.0174\n",
      "Epoch: 048/050 | Batch 7500/8290 | Cost: 0.0189\n",
      "Epoch: 048/050 | Batch 8000/8290 | Cost: 0.0175\n",
      "Time elapsed: 45.03 min\n",
      "Epoch: 049/050 | Batch 0000/8290 | Cost: 0.0185\n",
      "Epoch: 049/050 | Batch 0500/8290 | Cost: 0.0185\n",
      "Epoch: 049/050 | Batch 1000/8290 | Cost: 0.0180\n",
      "Epoch: 049/050 | Batch 1500/8290 | Cost: 0.0173\n",
      "Epoch: 049/050 | Batch 2000/8290 | Cost: 0.0163\n",
      "Epoch: 049/050 | Batch 2500/8290 | Cost: 0.0191\n",
      "Epoch: 049/050 | Batch 3000/8290 | Cost: 0.0177\n",
      "Epoch: 049/050 | Batch 3500/8290 | Cost: 0.0161\n",
      "Epoch: 049/050 | Batch 4000/8290 | Cost: 0.0180\n",
      "Epoch: 049/050 | Batch 4500/8290 | Cost: 0.0179\n",
      "Epoch: 049/050 | Batch 5000/8290 | Cost: 0.0173\n",
      "Epoch: 049/050 | Batch 5500/8290 | Cost: 0.0190\n",
      "Epoch: 049/050 | Batch 6000/8290 | Cost: 0.0165\n",
      "Epoch: 049/050 | Batch 6500/8290 | Cost: 0.0186\n",
      "Epoch: 049/050 | Batch 7000/8290 | Cost: 0.0161\n",
      "Epoch: 049/050 | Batch 7500/8290 | Cost: 0.0173\n",
      "Epoch: 049/050 | Batch 8000/8290 | Cost: 0.0178\n",
      "Time elapsed: 45.96 min\n",
      "Epoch: 050/050 | Batch 0000/8290 | Cost: 0.0171\n",
      "Epoch: 050/050 | Batch 0500/8290 | Cost: 0.0168\n",
      "Epoch: 050/050 | Batch 1000/8290 | Cost: 0.0178\n",
      "Epoch: 050/050 | Batch 1500/8290 | Cost: 0.0169\n",
      "Epoch: 050/050 | Batch 2000/8290 | Cost: 0.0172\n",
      "Epoch: 050/050 | Batch 2500/8290 | Cost: 0.0169\n",
      "Epoch: 050/050 | Batch 3000/8290 | Cost: 0.0168\n",
      "Epoch: 050/050 | Batch 3500/8290 | Cost: 0.0155\n",
      "Epoch: 050/050 | Batch 4000/8290 | Cost: 0.0180\n",
      "Epoch: 050/050 | Batch 4500/8290 | Cost: 0.0187\n",
      "Epoch: 050/050 | Batch 5000/8290 | Cost: 0.0189\n",
      "Epoch: 050/050 | Batch 5500/8290 | Cost: 0.0182\n",
      "Epoch: 050/050 | Batch 6000/8290 | Cost: 0.0193\n",
      "Epoch: 050/050 | Batch 6500/8290 | Cost: 0.0189\n",
      "Epoch: 050/050 | Batch 7000/8290 | Cost: 0.0176\n",
      "Epoch: 050/050 | Batch 7500/8290 | Cost: 0.0174\n",
      "Epoch: 050/050 | Batch 8000/8290 | Cost: 0.0180\n",
      "Time elapsed: 46.89 min\n",
      "Total Training Time: 46.89 min\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### TRAINING\n",
    "##########################\n",
    "\n",
    "epoch_start = 1\n",
    "\n",
    "\n",
    "torch.manual_seed(random_seed)\n",
    "model = Autoencoder()\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "\n",
    "################## Load previous\n",
    "# the code saves the autoencoder\n",
    "# after each epoch so that in case\n",
    "# the training process gets interrupted,\n",
    "# we will not have to start training it\n",
    "# from scratch\n",
    "files = os.listdir()\n",
    "\n",
    "start_time = time.time()\n",
    "for epoch in range(epoch_start, num_epochs+1):\n",
    "    \n",
    "    \n",
    "    for batch_idx, (x, y) in enumerate(train_loader):\n",
    "\n",
    "        # don't need labels, only the images (features)\n",
    "        features = x.to(device)\n",
    "        \n",
    "        ### FORWARD AND BACK PROP\n",
    "        decoded = model(features)\n",
    "        cost = F.mse_loss(decoded, features)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        ### LOGGING\n",
    "        if not batch_idx % 500:\n",
    "            print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
    "                   %(epoch, num_epochs, batch_idx, \n",
    "                     len(train_loader), cost))\n",
    "\n",
    "        \n",
    "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
    "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n",
    "        \n",
    "# Save model\n",
    "if os.path.isfile('autoencoder_quickdraw-1_i_%d_%s.pt' % (epoch-1, device)):\n",
    "    os.remove('autoencoder_quickdraw-1_i_%d_%s.pt' % (epoch-1, device))\n",
    "torch.save(model.state_dict(), 'autoencoder_quickdraw-1_i_%d_%s.pt' % (epoch, device))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OBO9L5FnVA2h"
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 323
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3782,
     "status": "ok",
     "timestamp": 1525044542253,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "UpJLf9FnVqSw",
    "outputId": "121c6c55-6171-4b1b-c6ea-5a199abb4bc5"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9wAAAEwCAYAAABFSXnNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VGWW//En7EtIQhIQZF8EkcUNRTZBcBdEx4XRxm4Btd3QBrG1FbXVccUNWwcVV3REHQaRhnZwEHD3Jbs4CiiLmMhOgBCWEMjvn1+/xnPOQ24l1FNZ6vP+73s9t+omqVu3HspzbkpxcbEDAAAAAADxVa28DwAAAAAAgKqIBTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABMCCGwAAAACAAFhwAwAAAAAQAAtuAAAAAAACqJHg5+Om38klpbwPoIrh/EkunD/xxfmTXDh/4odzJ7lw7sQX509y8Z4/fMMNAAAAAEAALLgBAAAAAAiABTcAAAAAAAEkuocbQAJt3rzZbJsyZYrImZmZpqZNmzYit2vXTuSmTZvG4egAAEi8adOmmW3r1q0T+cwzzxS5W7duIQ8JQBXGN9wAAAAAAATAghsAAAAAgABYcAMAAAAAEAALbgAAAAAAAkgpLk7o/di5+Xty8d78HWUWef4UFRWJ3LVrV1OzYsWKIz6QOnXqmG3t27cXuW3btqZGb/PV6IFt+nH1f3fOudq1ax/+YCsvzp/44vqTXDh/4qfSnzuPPfaYyHfeeWepH2Ps2LFm2/jx48t8TBUY5058VfrzB6XiPX/4hhsAAAAAgABYcAMAAAAAEAALbgAAAAAAAqCHGyHRBxRfkefP4sWLRT755JNNzdtvvy1yr169TM2aNWtKzKtXry71Pr79tm/fbmrKonnz5iI3btzY1NSsWVPkBg0amJrq1auLnJaWJnK1avbfKNPT00VOSbEv+yZNmoh8++23m5r69evrTZw/8cX1J7lw/sRPpTp38vPzzbbMzEyR//Vf/9XU3H///SI/8MADIk+ePNnss2nTJpEbNWoU83FWYJw78VWpzh8cMXq4AQAAAABIFBbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABBAjfI+AADxs2TJksiavn37iqwHjjnn3EMPPSTyt99+K/Jpp51m9rniiitEHjJkSOSx7Ny502zTg9ViGcamt/mGse3Zs0fk/fv3m5rdu3eLvGXLlsh99OMWFBSYms2bN4t83nnnmZoePXqYbaic9NAm34A+AGGsWrXKbCsqKhI5NTXV1LRr167Uz3XttdeKPH369FI/BoDyl5uba7bNnDlT5PPPP9/UtGjRIqbH5xtuAAAAAAACYMENAAAAAEAALLgBAAAAAAiAHm6gCtm7d29kTVpaWmTNjBkzRK5RQ75VvPLKK2af559/XmTd/+yccxkZGSKnp6ebmpNOOqnEXNG99dZbZttVV10l8lFHHZWow0FgkyZNMttuvvlmkZctWybyscceG/SYKjPda+ucc/fee6/Il19+uak54YQTgh0TKpeffvopssbXw10WTZs2jcvjACHoz2Hz5s0zNXPmzCkxO+dcTk6OyL6+5bZt24rcpk0bkfWMHOec+/LLL0Vu2bKlqdGfR/XnyFj45hv9+c9/Fvnjjz82NcXFxSK/8847pmbo0KExHQPfcAMAAAAAEAALbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIgKFpQJLbt2+f2bZp0yaRx48fL7JvINpLL70kclkGW1QFGzdujKxhaFrltXbtWpHHjBljagoLC0V+7rnnSsz4P7fccovZNnHiRJEbN25sahiahn+qU6dOZM3BgwfNttmzZ4v8+eefi/zggw+afT788EOR9ZAl55xLSUmJPB6gtHxDvsaOHSuyHtjpe33q99MBAwaYmt///vci//zzz6bm66+/FvmTTz4R+cCBA2Yfzfe4mZmZInfr1s3UPPnkkyLrwb5TpkyJfO727dubbf/+7/8u8llnnRX5OIfDN9wAAAAAAATAghsAAAAAgABYcAMAAAAAEAA93ECS++WXXyJrWrZsKXJRUZGp2bFjh8i+Hrnq1auX8ugqH93/7pztZ69bt26iDgdH4NChQ2bb8OHDRa5Vq5apufjii0WePHmyyA8//LDZJy0trSyHWOlNmDBBZN2v7dO8efNQh4MqwPf60H3UU6dONTVPPfWUyGeffbbIf/vb38w+uud0yZIlpuakk046/MECMZozZ47IgwcPNjXHHHOMyE888YTIZ555ptmna9euIvtmDuhe8JtvvtnUrFixwmz7rdatW5tt+pzS80+cc+7SSy8VeeXKlabG93NFueCCC0T2vSfEMg8iVnzDDQAAAABAACy4AQAAAAAIgAU3AAAAAAAB0MMNJLkNGzZE1jz66KMix9LXonu6nXMuKysrcr+9e/eK/NVXX4ms743qnO37yc7ONjV6W8eOHU3NKaecEnl8UXy/zyZNmhzx4yLxfPfL1vcW9d3fs3PnziK///77Ir/++utmH9/9p6uamTNnmm36Pua+c0Xf297XCwj802uvvWa26fsP++YzRKlRI/oj86xZs8w2ergRD0OHDhXZNz9EX7N69uwp8tatW80+8+fPF1nPMnDOvq599/PW+vbtK7KvR1rfA/zll182Nfq59u3bF/nc2ogRI8w2fa/u0PiGGwAAAACAAFhwAwAAAAAQAAtuAAAAAAACYMENAAAAAEAAKbE0vsdRQp8Mzu3evVvkNWvWmBo94Kd69erxevqUeD0QnHMxnD96YMaoUaNMzc6dO0VOSbF/pgceeEDkRYsWibx06VKzjx52lpuba2ry8/NFHj16tKn5xz/+IfL+/ftF9g0KqVmzpsgFBQWmJhbnnXeeyG+//bbIGRkZkY9x1llnmW1FRUUiz5s3L5bD4fyJr8jz58CBAyI3a9bM1PTq1Uvk6dOnRz5xnz59Imt8wwArO/0+oYfoOOfc8ccfL7IeouOcHZyYk5NjajzXLc6f+KnQn930eXv00UebGj0syndu+15Xv3XyySebbYsXLxa5d+/epqYSntucO/EVl/Nn4MCBIs+dOzceDxuM/lzWqlUrU6PPS9+w3bI499xzRb7rrrtMzauvviryf//3f5uajz/+WOTjjjsulqf3nj98ww0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAdQo7wOIsm7dOrPtvffeE3n27NmmZvv27SLr/lJfn81FF10k8uDBg2M9zIQ4dOiQyE8//bSp0TeWX7hwoci6l9Q557Kzs0X+y1/+YmrGjBkT83GicmnQoIHZNn78+FI/ju61/uCDD0zNNddcI7JvXsDYsWNFHjBggMg9e/Y0+9StW1dkfb4759yWLVtEnjVrlqnRr/M//OEPIvt+Jm3Xrl1mW9OmTSP3Q/n7n//5H5H1a8Y552688cZSP67uvXvsscdMjX7N6td0ZbBixQqR9UyERo0amX0mTpwo8qmnnmpq9O88jnNGUAXo2TS6L9QnNTW11M/j60HVPdzff/99qR8XiIXuJ/71119NzYwZM0R+9NFHRf7555/NPq1btxbZN8tArx369+9vai655BKRn332WZF//PFHs09Z1K5d22y74YYbRNa/q9NPPz3ycY899lizrUWLFqU8usPjG24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAVS4oWlLliwR2dfoXlBQILJvyEqHDh1E1kNWZs6cafbRN0F/+eWXTc3IkSPNthD0z+icc1dccYXIf//7302NHmRwzz33iOwbCjBlyhSRb7vtNlOjB2L5BqshueXm5oo8YsQIU6PP1f/4j/8wNfEYMOYbONWyZUuR9ZAN5+xgkFtuuUXkRYsWmX1OPvlkkQsLC01NzZo1D3+wqDDefvttkRs3bmxq9BC/WPTq1Utk/X7qnHMLFiwQOZYhL+XJNwBH/270637OnDlmnyeeeCLyuW699dZSHh2Sie/zUhTfZ6EosQxQysvLM9v0+e4b+gT8lm/wnx6K3Lx5c1Oj11B6iOvZZ59t9lm1alXk8VSrJr+f3bBhg6kZNWpUiY9Rp04ds23fvn0i169f39TUq1dPZN8Q5wceeEBkPWTaJz09XeTp06ebGt9Q4bLiG24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACKBce7jz8/PNtsGDB4vs66GbP3++yGW5Mbnv/+8fNGiQyKNHj46sycjIMDXbt28XWff06P/unO3x+eSTT0yN7tnWPefOOTd8+HCzLcpll10msq/39u677xZ52LBhpiaeN4hHxVZcXGy2XXPNNSL7+tT0vICjjjoqvgd2hK666iqR9TyD2bNnm310D/eBAwdMDT3cFc+ePXvMtg8++EBk38yOGjVKf9ns3bt35GN8/vnnIle0Hu6ffvpJ5DPOOMPU6Fkp8+bNEzklJcXs88orr4h80003mRo9fwH4Ld+5HKUsPdy+/tJYHDx4sEz7IXksXLhQZL3WcM65TZs2HfHzfPTRR0f8GM45t3LlylLvo/u1fXzzGPr06SOyb4aUnv2j15e+/nG9purYsWPk8R0JvuEGAAAAACAAFtwAAAAAAATAghsAAAAAgABYcAMAAAAAEEC5Dk3z3WQ8NzdX5Dlz5piaeAzn0jdxd865CRMmiNyhQwdTo4e3FBYWHvGxxEoPXirLgDQfPchm3Lhxpub1118XeebMmabmhhtuiMvxoOLTw8+cswOS3nzzTVNT0YakaXoIYmZmpsibN2+OfAyGplUOvvew3bt3i3zFFVfE5blSU1NF7tq1q6nRQ9PK05o1a8y2AQMGiOwbPKqHfbZr107kW2+91eyjr8W+gThASXyDlqKUZWhaWdWqVSthz4XK6U9/+pPIdevWNTV6zeQbJvnnP/9ZZN8aqiyys7NF/t3vfmdqzj//fJHvvPNOkZcvX272KSoqinxuPazWN3xzw4YNIushaZ999pnZ54QTToh87njiG24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACKBce7i7detmtulet5EjR5oa3aPg62X29T9EOeaYY0Ru3rx55ONef/31pqZhw4Yi6z5Q/d+dc6527doiX3311aYmUX2g3333XWRNp06dEnAkqKjeeOMNs61Lly4iDxs2LFGHEze6L3XPnj0ix3IO+uY61KhRrm+18MjJyYmsOf7444M8d9++fc22yZMni+zrkfbNHomHdevWiaz7tZ2zswnmz59vavQ1VMvKyjLb9u3bJ/K7775ram6++eYSHxfJbe/evaXeJ5GfYXw9p0heCxcuNNu++OILkV944QVTM2TIkMjHnjhxosh6FpVvjo6+Hl1yySWm5tJLLxW5evXqpmbGjBkiL1mypOSDLaPi4mKzTfdjT5s2TeQ2bdoEOZbS4BtuAAAAAAACYMENAAAAAEAALLgBAAAAAAigXBsLff1xc+fOFfmOO+4wNTfddJPI999/v6kZNWqUyKeccorIvh66evXqiTxw4EBTo+81PGbMGFMTD76+dP27OO+880yN/t3onzM9Pd3so+9f57sPt773+emnn25qUHVt2bJFZH2eOufcfffdl6jDCUb3VuXn54vcq1evyMfw3VeS+7BWPHq2hk9eXp7Z1rRp0yN+7t69e5ttzz77rMi++3Lr9+/169ebGt2Prd/ffaZMmSKy7qt2zl77OnbsGPm42pVXXmm26fcNPccFiNKkSZNS75PI+3Dr+9pHzTpA1bZ48eLImrJ+xm7fvr3Iu3btEjnk++vjjz8ucqNGjUT2zbKJ5fqk9/N9xtK96xWhZ1vjG24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAZTr0DQfPdzMN5zpm2++Efnhhx82Nffcc0+Jz5ORkWG26UE2y5YtMzVdunQp8XHjxTeMrWbNmiI/9dRTpmbw4MEip6SkiNy1a1ezz86dO0vMzjn397//XeRq1fi3mmSiByb5hlZcdNFFiTqcYD766COR9bCOAQMGRD7GgQMHzDaGplU82dnZkTWrV6822+IxNK1Pnz6RNf369Tvi53HODsnxDURr2LChyL7rbqdOnUr93KtWrRL5mmuuidwnlnMM+K3OnTtH1uj3YN8A2Sj79+8v9T7OObd27VqRGZqW3C688EKzTQ881gPInHPuueeeE7l+/fqRzxVqSNqvv/5qtn3xxRelfhx9HhYUFJiaQ4cOifzMM8+YmtNOO63Uz51orJoAAAAAAAiABTcAAAAAAAGw4AYAAAAAIIAK18Mdi1NPPVXk//qv/zI1kyZNEln3MXz55Zdmn4ULF4qs+5+dc+4vf/lLzMd5JHw3iB89erTIuufDOdvf/vXXX4usbw7vnHOFhYUif/bZZ6YmUb3rqJh8ff1aPHpby5vu4dYzJWLp+9Pnk3P+8xnl64wzzjDbGjduLPJjjz1mao466iiRfT1yUefC0Ucfbba98cYbIvt62Vq0aFHisTjn3P333y/yhAkTRN6yZYvZp169eiJv3rzZ1Gh79uwx2/RckYceekjktLQ0s88777wjcsuWLSOfG/itBg0aiNy6dWtTk5ubK/L06dNNTdQckry8vMhj6du3r9nWv3//yP2QPJo0aWK2jRo1SuSnn37a1MycOVPknj17mpqsrCyR9WvW9xouS43v+lQWHTp0EHngwIGm5rrrrhO5TZs2cXnuROMbbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIgAU3AAAAAAABVIlJPtWrVzfbrr/++hL3GTZsWKjDSZhatWqZbX369Ckx/+EPfzD7VKsm/91FD10A9u3bF1lT2QaDffLJJ2abHhioh0DFoqioyGzznasoX/Xr1zfb9FBMPajSOed++uknkU8++WRTo68/a9euLfExnHPu2WefLfFYnHPunHPOEdk3WGfr1q0iT506VeTbb7/d7KMHto0YMcLUnHvuuSLPnz/f1Oj3CX29efLJJ80+XG8Qb77Pf88995zIl19+uanRr2k9jE2fWz5jxowx23j/RxT9WcO3RnnhhRdEXrRokalZuXKlyA0bNiwxO+dcs2bNRM7IyDA1mZmZIvsGYD7zzDNm228tWbLEbPMN/qyq+IYbAAAAAIAAWHADAAAAABAAC24AAAAAAAJIKS4uTuTzJfTJUO5SyvsAqpjI80f3qY0aNcrU7Ny5U2RfL472+eefi9y3b19T89prr4nsmxdw6NAhkX3zF7Zt2yZyWXo8d+3aZba9/PLLIt93332mpkWLFiLrnqPatWtHPnfdunXNtrFjx4r84IMPRj6O4/yJt8jzR/cg9+zZ09QsXbr0iA8kOzvbbIulP/Tmm28WWZ/vzjnXsWNHkfPy8iL3+fDDD0XW57JzzrVu3VrkSy+91NQMHTpU5O7du5uaBOL8iZ9K/9mtoKBAZN9rU/e/6s/Hvvf2E044QeQvv/yyrIdYkXDuxFelP3+0F1980WzTsxPmzp0r8hlnnBH0mCoQ7/nDN9wAAAAAAATAghsAAAAAgABYcAMAAAAAEAALbgAAAAAAAqhR3gcAIH42bdokcp06dUxN/fr1S/24vXr1Etk3/EIPdJo4caKp2bBhg8iDBg0yNZMmTRL5qaeeKvFYnHNu8uTJIr/++uumRg9SGzJkiKl56aWXRI5lSJpWVFRkttWsWbPUj4PEmzNnjsg///xzmR5H/71TU1NF9g1I0+eqHuDmnH/gmaYHP2mXX3652abfE8aNG2dq7r33XpF5TaMy0a9xPdDJOefGjx8v8tNPPy3y8ccfb/b54IMP4nB0QOUya9Yss61r164iJ9GQtJjwDTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABJBSXJzQ+7FXuZu/o0Tem7+jzCLPnwsvvFDknJwcU7N48eIjPpDt27ebbcOGDRP5ww8/POLniZXuf73qqqtMzS233CJyly5d4vLc+j20WjX775iPPPKIyHfeeWcsD835E1+R58/q1atFfvnll01NvXr1RN6xY4epmTJlish6doHvNaJ7ouvWrWtqDhw4IHJBQYGp0fv169dP5NGjR5t9+vfvL3KtWrVMTSXE+RM/SfHZbf369SK3atVK5DfffNPso697VQTnTnxVufPnlFNOMdtatGgh8rRp0xJ1OBWN9/zhG24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACID7cANVyPLly0XW/ZvxkpmZabb94x//EHnVqlWmJjc3V+TCwsLIx965c6fIu3fvNvv07dtX5KysLFMTiu6r9eGexZVDu3btRNa99z4jR4402/bu3Svyk08+KfKYMWPKcHTW7NmzzTbddz5x4kSRs7Oz4/LcQFUTdW489NBDZlt6errIgwcPjusxARVRjRp2+fj++++LrK97t912W9Bjquj4hhsAAAAAgABYcAMAAAAAEAALbgAAAAAAAmDBDQAAAABAACnFxQm9H3uVu/k7SuS9+TvKTJw/+fn5piAtLU3kJ554wtQk++CKeCsoKBA5NTXV1EyYMEHkW265JZaH5vyJryDXH9/gv0OHDolcp06dEE+NknH+xE9SfHabOXOmyGUZgPbqq6+abcOHDy/zMZUTzp34qnLnz5IlS8y2efPmiXzuueeKfNxxxwU9pgrEe/7wDTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABGDvXA6gUli5cmVkTadOnRJwJMmtdu3aInfv3t3UdOnSJVGHgwSrVatWeR8CgDgYP368yCeccILIGzdujHyMpUuXxvWYgIroxBNPjGkb/g/fcAMAAAAAEAALbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIgKFpQCXVuHHjyJrt27cn4EiSW40a8m10wYIF5XQkAICyeuONN0ROTU0V+f333zf7XHfddSL/7ne/i/+BAaj0+IYbAAAAAIAAWHADAAAAABAAC24AAAAAAAKghxuopFq2bGm26R60c845J1GHAwBApdW6desS//uIESPMtpSUFJGPO+64eB4SgCqCb7gBAAAAAAiABTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABJBSXFycyOdL6JOh3KVEl6AUOH+SC+dPfHH+JBfOn/jh3EkunDvxxfmTXLznD99wAwAAAAAQAAtuAAAAAAACYMENAAAAAEAAie7hBgAAAAAgKfANNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAA1Evlke/fuLf5trl27tqlJSUkpMaNiKC4ujtxWrVo1/nhxlJ+fL37B9erVMzXbtm0TuVo1+29qBw8eFPnAgQOmpkGDBiLXrVtXZN956XsurSznd7K8B+jzJyVZfvAE4fpTtXH+hFPZzp1Dhw6Zbfo6l5eXZ2q2bNkicsOGDUVOT083+6Smpopc0V92fHZLvMp2/uCIef94fMMNAAAAAEAALLgBAAAAAAiABTcAAAAAAAEktIdb94Gi8vL1l9BzEpbuq/bRPWe7du0KdTjBJOvrKFl/7kTh+gOUTWU7d3zzRKpXr17qx9HvyXXq1Imsqej47JZ4le38QRh8ww0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAggIQOTQMQ1p49e0QeNWqUqZk9e7bIRx11lKnJzs4WOTc3V+S0tDSzT15ensgHDx40NUVFRSKPHTvW1LRp00bkJk2alPjfnbMD5WrUsG9t+nhq1aplavTwmEOHDolcXFxs9tHDeHwDaMoysAdA7Bj8hH/yvU+/9957Io8cOdLU6Pd7nd9//32zT//+/UVOTU2N9TABJBG+4QYAAAAAIAAW3AAAAAAABMCCGwAAAACAAOjhBqqQFStWiLx7925Tc8opp4h82mmnRT7uV199JXLt2rUj9/H1WtepU0fk1157LXK//Px8kX/99Vezj+7h3rt3r6nZvn17ifs451zNmjVF1j3nurfdOecyMzNFrl+/vqn5+uuvRZ43b56padWqlcj0fQNA6RUUFJht77zzjsi6P9s523+tZ6JcdtllZh/9Xh7L9RRA8uEbbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIgAU3AAAAAAABMDQNqKQKCwvNtg8//FDktWvXmprRo0eL3L59e1OjB5PpwWUbNmww+wwZMkTkfv36mZqWLVuK7Btuph97yZIlIv/4449mn2rV5L8drlu3ztRozZo1M9vS0tJEPnDggMi+39X06dNF3rhxo6nRP+d3331navSQND1EDRXTwYMHzTY9kEn/bfXrFUdO/x0YOpi88vLyzLaffvpJZD3A0znnevfuLXJOTk6J2TnnFixYIHL37t1NTY0aFfujdnFxscgpKSnldCSIN/2+6BsWGMvf27dfVI1+3LJe90JdP3fu3CnysmXLTI0ewHjGGWeYGj1E+HC/T676AAAAAAAEwIIbAAAAAIAAWHADAAAAABBAxW4sAXBYur/Yp3PnzmZbLP06uqdr06ZNIi9fvtzsM2zYMJG7detmaho2bBj53O3atRNZ933H0psZS01mZmZkja/PT9O96r5ep8svv1xk39+AvrnKYf/+/SIPHDjQ1CxcuFDk++67T+Tbb7/d7FPR+zwTxdd/u337dpGbNm1qanQfHZLXzz//bLY1aNBAZH2dcc65k046qcT83nvvmX30tXLHjh2mJjs7+/AHWwFw7ak6du3aJbK+Fun5PM7Z+Qa+GT0//PCDyHquj3P2s0+tWrVEbt68udknIyND5Llz55qaF154QWTfZ8v69euX+Nzr1683++jr8kcffWRqevbsKXKfPn1MTazXHr7hBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAU1qASso3GCwtLU3kY489NvJx9NAXn9WrV4tcVFQUuU9qaqrZVq1a9L/x6UFlrVq1Etk3lCyWn0EPhtEfO8pkAAAZ4klEQVRDNZyzv9NYhsnooRorV640Ne3btxdZD+Nxzj8ICuXL97patGiRyL4Bgnqg4cGDB0Xet2+f2cd3viSDNWvWiHzhhReamv/93/8V+fvvvzc1bdu2FTmWwYmoGvQ1Qb+mnHMuNzdXZN+QQn2d04PVdu/ebfa54447RD7rrLNMTUUfmobKoaCgQORly5aZmsWLF4s8YcKEEh/DOfv61INBnbODK301mv78pIfvOmeHnflqLr74YpF9nyPHjRtX4rFMmTLFbNu4caPIeuCcc8717dtXZN91RX9OONznRr7hBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAA6OFG3MTax4DEmTt3rtl29dVXl/px0tPTRfb1cH/zzTci695m5/x905rux9uxY4fIW7duNfvofqK9e/eaGt1707lzZ1PToEGDyOPT9Ov8u+++MzW//PKLyDVr1ox8HJS/zZs3m20//vijyC1atDA1GzZsEPnFF18UecSIEWafZOjh9s1feOKJJ0TW/drO2XPD1wMfyxwHVE36deV7nelrRMOGDU1Njx49SnyerKwss033nE6fPt3U6Bkevvd/IMqCBQtE9vUtFxYWiqw/0zRp0sTs07VrV5F9r0/dG+77HKa36c+Jul/bOTt3yHde6tkJvj7qZ555RuQuXbqIfMkll5h91q9fL/KHH35oaoYPHy5yvXr1TI2e0XI4fMMNAAAAAEAALLgBAAAAAAiABTcAAAAAAAHQw51kfH1u9I5WTr4+Nd3bqPufnbN9NL6+tPz8fJHz8vJE9t2DUfdwz5o1y9Qcd9xxJR6Lc/ZeiLon2tdXq3tofPdT1D1Ijz/+uKnR/bgZGRki+86VnJwckX33kdRq165ttnEelj/d+z9//nxTo++7O3ToUFOj7/E7bdo0kd955x2zz4033iiyvh99ZaTPy6VLl5oa/bvx0T17vr553zmF5KDve++7RuheVt8chdatW4use0cbN25s9tHXJ91D6zs+ergRxTcnx3c90urWrSuyfs22bNnS7KP7qH0GDhwosp5L45xzy5cvF3nnzp0id+rUyezTpk0bkdetW2dq9HP51jH6vNOf3Xz070L3sjvnv+e35usp9+EbbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIgAU3AAAAAAABVMqhaXoAhW9IhR7WomsyMzPNPrE0x1ckvsEBBQUFIufm5opcq1Yts48ezpOdnW1qGPJR8cQyZEsP9PLtp4dsOOdcenq6yOeff77IegCZc85t2LBB5GeffdbU6Mcpyzm3ceNGs+3dd98VuW3btqZGD9744IMPTI0erHPFFVeIrH8vztnBIOvXrzc1ejhcZXuvSRZ6KM0999xjanr16iXy73//+8jH1e/DvmuWHhbTsWNHU1PRB+vpa/PTTz8tsu/nPu2000T2DVbr0aOHyPXq1TM1Ff13g3D05z3fZyPfIDVNDxnV2TcoSg8vnThxoqkZM2ZM5HMDv6UH4Drn3LJly0T2DVbbtm2byPo1u3LlSrOPfj/1rY/04/iOT58vegipb5itHjqoryG+x/HR1xad9cBe5+zPtGLFClMTz7UPn/oAAAAAAAiABTcAAAAAAAGw4AYAAAAAIIAK18Ote3G+//57U6N7ZKZOnWpqdu/eLbLuAfD1l5555pkit2rVytT4esdC8PW66T5AX6/bvffeK/LatWtF9vWO6p6Jhx9+2NTcdNNNIvt6wemhS6zq1aubbfr1mZaWZmoGDBggclZWlqnRPdA9e/YU+YcffjD73HjjjSJ/++23piYvL89s0/r06SPyX//6V5F9MwYeeOABkX29dp988onIuvfOOfu+obPv/NHPpfvAnXPu2muvjXwcJJbvPXbBggUi+/rJrr76apE7d+5savQsDd2n9t5775l9Bg4ceNhjrYh8PXzTpk0TefLkySIfffTRZh99nfX1D+rfse9ao/t2uR4lD907Om7cOFOjr5d6do1z9rWnP4+mpqaaffR7uZ7p4XtuIMqaNWvMtuOPP17khg0bmhq9ZtKP43vf1ueCb96BXkPVqGGXj/v37xdZ95jr66Jz9jrs+1ym+WY06HNVP86iRYvMPqtWrRJ5+/btkc/to5/rcJ/v+NQHAAAAAEAALLgBAAAAAAiABTcAAAAAAAGw4AYAAAAAIIAKNzTtq6++EnnUqFGm5sQTTxRZDwrz+fLLL0W+6667TM2cOXNE1oOYnLM3T4/XTdH1zerffPNNU/PQQw+J7Bsu8C//8i8iP/nkk5HPrX83EyZMMDV6EMjdd99tauJ5g3jExx//+EezrX79+iLHMlxID8jo0KGDqdFDn/TAPufskI9LL73U1OgBffp4fXSNb7DascceK7LvHMvJyRH57bffjnxuPcDDN5xj8ODBIvsGjiAs/X6pB7w4Z/+WepCmc3YgSu3atU2Nfi/Ug/987916oFhFG/qlh+boIYnOOTdr1iyRBw0aJHLLli3NPqtXrxbZN8xKD1bjWoPf0kPT9LAm5+x7ru/8OvXUU0XWA6Y+/fRTs49+v/cNpYplEBSSmx7Q+f7775sa/XkkIyPD1OhBr3pgn+96pc+X/Px8U6MHBvo+w+jPYfpa6RseqN/L69atG1nju3brc0wP/vUNmNO/q9tvv93U+IbgarEOweUbbgAAAAAAAmDBDQAAAABAACy4AQAAAAAIIKGNhLo/bteuXaZG/z/0vj6bW2+9VeRjjjnG1Oj+guHDh4u8ZMkSs4/u4Z43b56p2bNnj8i678452yOh+xZWrlxp9tF91L4eJN1vcMcdd5iaMWPGiJyWliayr5fotNNOE7lXr16mRv/Or7rqKlOjfxe+XgyEpfuAfBLVT3bttdeabfo1oV9Xzvl7jOJBP/fVV19taqZPny7y999/L3IsfbW+/iKtovXnJgP9nrp582ZT8/zzz4us5wk451y/fv1E9v0tdU+X7kErLCw0++jrY3nS/drOOffKK6+I/PHHH5uarKwskX0921q7du1E1r9f55zr0aOHyL5eQM6p5KV7uGPRqVMns033eerrabdu3cw++rXou74WFBSIrPthkVx8n+8nT54ssn6/dc7OUvLNt9Hv3frziO+59fXI9zlSz9bw9TbrmoMHD4pcq1Yts49+Lt8MBF3j+xn0z7l06VKRfT3c+vrumxkWz3k7fMMNAAAAAEAALLgBAAAAAAiABTcAAAAAAAGw4AYAAAAAIICEDk3Tje6//vqrqVm8eLHIjz76qKk56qijRK5Xr17kc+tm/e7du5saPTTNN7Rg1qxZIvuGy4wbN67EY/n666/Nttdee01k3+CoE088UWTf0Cc9JE3z3aBdD15IT083NTk5OSKvX7/e1OihBAxNK38PPfSQ2XbdddeJ7Bu8oelBMKtWrTI1U6dOFfnKK680NSNGjBA51IC0WPgGeOjBf3poWiyaNm1qtulBIBVpQFay0ANcfvjhB1OjhxkNGTLE1PiGxUQ9l75O6AGYzjm3detWkZs1a2ZqQg0G08f7n//5n6ZGv2avueYaU6OPTz+uj74eXnLJJaamffv2JT4PklteXl5kTSzDQqPel32fjWJ5LcYyzBRVlx7otXr1alPz+OOPi+x7vY4ePVpk32tv0qRJIuvPd74hYHroYKtWrUzN0KFDRf7ll19MzYIFC0TWAwV954H+3fh+bj10tHHjxpE127dvF9k36PmWW24ROZbPwj76feNw7wl8ww0AAAAAQAAsuAEAAAAACIAFNwAAAAAAASS0h1v/f+2+Hl/9//z7emoWLlwo8sCBA02N7s/Uvcu+PgF9fL5+HX3jed374Huck046SWTdJ+qc7atYtGiRqdm2bZvZpumeuVh+n7qX8dtvvzU1ugfV1weinwth+frx9et+z549pkb34/v6/vXr6NNPPxXZNz/gj3/8o8gXXnihqcnMzDTbKhL9u9C/B9/rXvPNddDo4U48/bfUPdPOObd582aRfX+nWPu1fqtNmzYiL1++PPL4fH2psZw/uk9Ov0/k5+ebfd566y2RdT+hc7HNX+jcubPIuifONxNh2bJlIl988cWmJhZl+bugatCzF3z0eeCbxaBr9GvI93kvlt7wWOYMoerS7+27du0yNbqX2TcjY+TIkSLreVbOOXfFFVeIPG3aNJF9n2H063zQoEGmRs+mWbJkianRn32++uorkX3nSizv23pt4esx79Onj8j6PWH48OFmn7L2bGuxXmv4hhsAAAAAgABYcAMAAAAAEAALbgAAAAAAAkhoD7f+//B99xmdO3euyL77COteh7Vr15qaY445RuR27dqJvG7dOrNPQUFBZI3uxbjppptMzYABA0TW94dr1KiR2Uf/Lu68805TM2PGDJF994h95JFHRO7WrZvIun/XOee++OILkfXv1znb2+TrfYilvxVh6V4SX5+l7rPx9YLrnn3dr6PvX+ic7R1q0aKFqfE9V0Wij9nXK6Tl5uaK7PsZs7KyImsQlu6/3LJli6nR54Y+D2Kl3wv19cfXC3r77beLfN1115ka3SP9888/mxo940TXfPnll2YfXaOfxzn73nLRRReZGn0d811DNd2r7uuH43xBSXSPtO/1UpYZM0VFRSJ/8803pkZ/JtSf95yzs1WQXAoLC0X2vf/rXubjjz/e1Oiebd/rqnXr1iIPGzZMZN95oLdlZ2ebGq1jx45mmz4X9MyrOnXqRD63b26K7nn/+uuvTc3pp58u8vXXXx/53InGVQwAAAAAgABYcAMAAAAAEAALbgAAAAAAAmDBDQAAAABAAAmdcqWHofiGS/To0UPkd99919ToG7mPHj3a1GRmZoqcnp4u8q+//mr22bZtm8i+gTl//etfRf7Tn/5kanw/VxQ9VOmpp54yNSeffLLI9913n6m5++67RdY/g29Ygx4M4huapn/url27mpqy/NwoO99gmNTUVJF9A4hC0cdTGQcdHThwoMT/Hsvv0zeoDuVPv8f269fP1MQyLEa/RmIZiOQbBKPpIW6+wYR6II4eFuWcHYCmB1z63t91Tffu3U1N06ZNRdYDfJyzQ3MWLVok8vr1680++me44IILTE0s7yWJfK9DxdK4cWORfa8X/TnHN3CwQ4cOIufn54vsG9Crz+0bb7zR1JRlYBuqDv3+etxxx5kaPSTNd8346aefRPYNntaDyg4dOlRi9m3bvXu3qdFrB99ATD1UUA8h1dnHd3z6d6EHjDrn3KhRo0RO5JA0fXyHuxZVvk/EAAAAAABUAiy4AQAAAAAIgAU3AAAAAAABJLSHOxb6/3339ahddtllIvt6yXS/ju4teOutt8w+uv/MZ8iQISLrntl4adCggdk2YsQIkXW/kXO2D/2aa64RubCw0OzTvn37Ep/HOeduvfVWkStjf24yiqWv3tczo88fPfNg+vTpZh/dQ1PRFRQUmG2ff/65yPq9xdebk5OTI7LuoXLO9nXTb5p4uqerbdu2pkb3Yw8fPtzUvPTSSyL36dPH1Oi/b40a8lKbkZFh9tFzMfTryjnnjj32WJH1bBLnnDv77LNLPBZfD7c2b948s03PEFm4cKGp2bRpk8gvvviiyLm5uWYf3X9bt27dyOMDfistLU1k32c5fQ766J5t/Xr2vTb1NVbPinCOz0vJTr9GfLNC9PXopptuMjU33HCDyL7XtP7s9vjjj4vs+7ynP+e0bNnS1Ojrxp49e0yN/pzYsGFDka+88kqzj36unj17mhp9DdPXQef8a6ZEifXzHO8CAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAKocEPTYqEHEA0cODByn7POOktk343T9+3bJ7KvEd43pCZR9ECfM844w9ToYSHnnHOOyLt27Yp83MzMTFPD0I+Kp7i42GzTQ18OHDhgarZu3Sqyb5CeHoihh1/oAUqHO57y4jsWfX77BkPNmDFD5EsuuaTUz71+/frI46lIv6tkod/P9WvaOeceeeQRke+66y5To1837dq1MzV6mKbvfVfTg3X0MDHn7Huzb/haPAby/fDDD2bbuHHjRO7evbupqV69ush6QM/zzz9v9unYsaPI5XmNReWkh+v+27/9m6l58MEHRb7ttttMzR133CGyPpd8A6f0ZyP9+fRw+yF5+YbvjRw5UuQzzzzT1Hz//fci+z5HPPzwwyL3799f5KFDh0Ye35o1a8y2BQsWiLxy5UpTowe/PfbYYyKfeOKJZp+jjz5aZH0NqUpYRQEAAAAAEAALbgAAAAAAAmDBDQAAAABAAJWyh1uLpb9Y1/huGK/7gCo6X6+e/rnS0tJKzPGk+0ni0UuII1NUVBRZ4+vh3r17t8iLFi0SeeHChWafyy+/XORJkyaZGt2vEy+6V3316tWmZt26dSLr43XOuQ4dOoh85513Rj63/l3s2LHD1GzYsEFkX2+9nqWAsGK5buiZCM4599Zbb4mcm5trajZv3izyihUrRN67d6/Zp1WrViJnZWWZGv0abtKkianRPdC+fkFN9wZu2bIlcp+rr77abNPX0L59+4pcv359s4/vWgyUhn6NjxkzxtToGQn79+83NbrnVM8q2bRpk9mnadOmIt96662mxtfXjeTl+2zctWvXErNzzl1wwQUi+2YD6NkEuifa99z6s7ueBeWcnYvw6aefmho9Y0Q/brNmzcw+yTQfKnl+UgAAAAAAEogFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEkOK7cXpACX0ylDumpsWXOH/27dtnCs455xyRly5damqWL18ucvPmzU2NHpqxePFikX0Dk/SgstTUVFOjB3j4hvjpQVV60FKbNm3MPvrn9A270gPkRowYYWr+9re/iayHQPneL9euXSty7969Tc31118vsh5s4pz9OVOYOhhv4o/n+1tOmTJF5OHDh0c+aMOGDc22goICkfWfUp8HzjnXqFEjkX2DljIyMkTWww19z62HksVyXnbq1MnUPPjggyI3btzY1OjnSuRL2DO0k/Mnfir9Zzf9+vjss89MzUUXXSSyHm5Yp04ds8/UqVNFHjhwYFkPsSLh3ImvSn/+6M+NPXv2NDX6fJk+fbrIgwYNMvtU0bdp7w/FN9wAAAAAAATAghsAAAAAgABYcAMAAAAAEECN6BIAFZHuSXbOuVWrVon89NNPm5qsrCyRq1Wz/+6mt51yyikiz5492+zTuXNnkX191Pq5N2zYYGp0r92BAwdE1v3kztle6xNOOMHU6P72MWPGmJqaNWuabb/l6zdq0aKFyL4+Jf1z617Xwz02wvH9vocMGSKyr8//1VdfFVn3TDvnXGFhYYm5QYMGZp+cnJzDH+z/p19HW7duNTV6/oLu2Z4xY4bZR89S0Oeyc/6+84qE8wcl0a8P36yNCy64QOS3335bZF8PN5AM2rZtK3J2drap+eWXX0TW55xvbkoyvW/zDTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABEAPN+LGcx/UcjqSqkn/fnfu3Glq8vLyIh+nVq1apX5u3dPtu3e37q0+dOiQqdE9qBs3bjQ1ubm5Iuv7U/vu3d2sWTORdU+3c/ZniOrXjpW+9+T8+fNNzUknnVTisTjH+VMR6Nfa+PHjTc3o0aNF3rRpk6lZt26dyHXr1hW5T58+Zp/09HSRff1uem7D5s2bTY0+f/Q9tX29d77XI1CV+V7z7du3F1nP2vDde16f68nep4qqSV97fNc9/ZmvQ4cOIif7dSa5f3oAAAAAAAJhwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAATA0DXHDYJCw9u3bJ3JOTo6pKSwsFFkPUIoX3/CLY445ptSP4xvg1KVLlzIdU3nRg+p8g+DOP/98kcsyuA6Jl5qaarbpQTA6O+dc7969RdbvjfF6r/QNENSDnwBYsQw309fTXbt2RT7O+vXrTU2rVq3KcohAhaEH0c6dO9fUfPfddyI3adIk6DFVNnzDDQAAAABAACy4AQAAAAAIgAU3AAAAAAAB0MMNVBI1asjTtVGjRpH71KxZ02zz9V+j7DIzM0UeO3asqdF/Ox/dC8hMhMqLcwyo2HznqJ69oK+f27ZtM/vccMMNIk+dOtXUtGjRIvK5gYpMnws9evQwNXobr3OJ3wYAAAAAAAGw4AYAAAAAIAAW3AAAAAAABMCCGwAAAACAABiaBlQSemhFenq6qdGDt5o3b25qGGQRXw0aNBD5qquuMjXNmjVL1OEAAMqgf//+Iq9evVrkN954w+xTWFgocsuWLU0NAzBR1fA5svT4jQEAAAAAEAALbgAAAAAAAmDBDQAAAABAACm65zOwhD4Zyh2NS/Elzh/dO+acc7/88ovIWVlZpiYjIyPOh4XfOnTokNkWS7+Tfi9OofEv3rj+JBfOn/hJynNHvyfv2LEjch/fbJVK2O/KuRNfSXn+JDHv+VPp3gUAAAAAAKgMWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAMTUPcMPQprIMHD4pfcPXq1cvrUBDA/v37Ra5duzbnT3xx/anCuP4ExblThXHuBMf5U4XFev7wDTcAAAAAAAGw4AYAAAAAIAAW3AAAAAAABJDoHm4AAAAAAJIC33ADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAEwIIbAAAAAIAAWHADAAAAABAAC24AAAAAAAJgwQ0AAAAAQAAsuAEAAAAACIAFNwAAAAAAAbDgBgAAAAAgABbcAAAAAAAE8P8A7aYkwKCxwTwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1296x360 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "model = Autoencoder()\n",
    "model = model.to(device)\n",
    "model.load_state_dict(torch.load('autoencoder_quickdraw-1_i_%d_%s.pt' % (num_epochs, device)))\n",
    "model.eval()\n",
    "torch.manual_seed(random_seed)\n",
    "\n",
    "for batch_idx, (x, y) in enumerate(train_loader):\n",
    "    features = x.to(device)\n",
    "    decoded = model(features)\n",
    "    break\n",
    "\n",
    "\n",
    "\n",
    "##########################\n",
    "### VISUALIZATION\n",
    "##########################\n",
    "\n",
    "n_images = 5\n",
    "\n",
    "fig, axes = plt.subplots(nrows=2, ncols=n_images, \n",
    "                         sharex=True, sharey=True, figsize=(18, 5))\n",
    "orig_images = features.detach().cpu().numpy()[:n_images]\n",
    "orig_images = np.moveaxis(orig_images, 1, -1)\n",
    "\n",
    "decoded_images = decoded.detach().cpu().numpy()[:n_images]\n",
    "decoded_images = np.moveaxis(decoded_images, 1, -1)\n",
    "\n",
    "\n",
    "for i in range(n_images):\n",
    "    for ax, img in zip(axes, [orig_images, decoded_images]):\n",
    "        ax[i].axis('off')\n",
    "        ax[i].imshow(img[i].reshape(28, 28), cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy       1.15.4\n",
      "pandas      0.23.4\n",
      "PIL.Image   5.3.0\n",
      "torch       1.0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "autoencoder-conv-2.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}