pytorch_ipynb/basic-ml/softmax-regression-mlxtend-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.5\n",
      "IPython 7.10.2\n",
      "\n",
      "torch 1.3.1\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Runs on CPU or GPU (if available)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Softmax Regression with MLxtend's plot_decision_regions on Iris"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of softmax regression (multinomial logistic regression) and demonstration for how to use PyTorch models with MLxtend's [`plot_decision_regions`](http://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/) function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings and Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Device\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Hyperparameters\n",
    "random_seed = 0\n",
    "learning_rate = 0.05\n",
    "num_epochs = 10\n",
    "batch_size = 8\n",
    "\n",
    "# Architecture\n",
    "num_features = 2\n",
    "num_classes = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class label counts: [50 50 50]\n",
      "X.shape: (150, 2)\n",
      "y.shape: (150,)\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### DATASET\n",
    "##########################\n",
    "\n",
    "data = np.genfromtxt('../data/iris.data', delimiter=',', dtype=str)\n",
    "X, y = data[:, [2, 3]], data[:, 4]\n",
    "X = X.astype(float)\n",
    "\n",
    "d = {'Iris-setosa': 0, 'Iris-versicolor': 1, 'Iris-virginica': 2}\n",
    "y = np.array([d[x] for x in y])\n",
    "y = y.astype(np.int)\n",
    "\n",
    "print('Class label counts:', np.bincount(y))\n",
    "print('X.shape:', X.shape)\n",
    "print('y.shape:', y.shape)\n",
    "\n",
    "# Shuffling & train/test split\n",
    "shuffle_idx = np.arange(y.shape[0])\n",
    "shuffle_rng = np.random.RandomState(123)\n",
    "shuffle_rng.shuffle(shuffle_idx)\n",
    "X, y = X[shuffle_idx], y[shuffle_idx]\n",
    "\n",
    "X_train, X_test = X[shuffle_idx[:70]], X[shuffle_idx[70:]]\n",
    "y_train, y_test = y[shuffle_idx[:70]], y[shuffle_idx[70:]]\n",
    "\n",
    "# Normalize (mean zero, unit variance)\n",
    "mu, sigma = X_train.mean(axis=0), X_train.std(axis=0)\n",
    "X_train = (X_train - mu) / sigma\n",
    "X_test = (X_test - mu) / sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### Data Loaders\n",
    "##########################\n",
    "\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "\n",
    "class MyDataset(Dataset):\n",
    "\n",
    "    def __init__(self, X, y):\n",
    "    \n",
    "        self.X = torch.tensor(X, dtype=torch.float32)\n",
    "        self.y = torch.tensor(y, dtype=torch.int64)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        training_example, training_label = self.X[index], self.y[index]\n",
    "        return training_example, training_label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.y.shape[0]\n",
    "    \n",
    "    \n",
    "train_dataset = MyDataset(X[:100], y[:100])\n",
    "test_dataset = MyDataset(X[100:], y[100:])\n",
    "\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset,\n",
    "                          batch_size=batch_size,\n",
    "                          shuffle=True, # want to shuffle the dataset\n",
    "                          num_workers=4) # number processes/CPUs to use\n",
    "\n",
    "test_loader = DataLoader(dataset=test_dataset,\n",
    "                         batch_size=batch_size,\n",
    "                         shuffle=False,\n",
    "                         num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "class SoftmaxRegression(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, num_features, num_classes):\n",
    "        super(SoftmaxRegression, self).__init__()\n",
    "        self.linear = torch.nn.Linear(num_features, num_classes)\n",
    "        \n",
    "        self.linear.weight.detach().zero_()\n",
    "        self.linear.bias.detach().zero_()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        logits = self.linear(x)\n",
    "        probas = F.softmax(logits, dim=1)\n",
    "        return logits, probas\n",
    "\n",
    "model = SoftmaxRegression(num_features=num_features,\n",
    "                          num_classes=num_classes)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "##########################\n",
    "### COST AND OPTIMIZER\n",
    "##########################\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/010 | Batch 000/012 | Cost: 1.0986\n",
      "Epoch: 001/010 training accuracy: 44.00%\n",
      "Epoch: 002/010 | Batch 000/012 | Cost: 0.9489\n",
      "Epoch: 002/010 training accuracy: 66.00%\n",
      "Epoch: 003/010 | Batch 000/012 | Cost: 0.8236\n",
      "Epoch: 003/010 training accuracy: 72.00%\n",
      "Epoch: 004/010 | Batch 000/012 | Cost: 0.8275\n",
      "Epoch: 004/010 training accuracy: 95.00%\n",
      "Epoch: 005/010 | Batch 000/012 | Cost: 0.6650\n",
      "Epoch: 005/010 training accuracy: 72.00%\n",
      "Epoch: 006/010 | Batch 000/012 | Cost: 0.6465\n",
      "Epoch: 006/010 training accuracy: 78.00%\n",
      "Epoch: 007/010 | Batch 000/012 | Cost: 0.3825\n",
      "Epoch: 007/010 training accuracy: 95.00%\n",
      "Epoch: 008/010 | Batch 000/012 | Cost: 0.4917\n",
      "Epoch: 008/010 training accuracy: 77.00%\n",
      "Epoch: 009/010 | Batch 000/012 | Cost: 0.6003\n",
      "Epoch: 009/010 training accuracy: 81.00%\n",
      "Epoch: 010/010 | Batch 000/012 | Cost: 0.5938\n",
      "Epoch: 010/010 training accuracy: 93.00%\n"
     ]
    }
   ],
   "source": [
    "# Manual seed for deterministic data loader\n",
    "torch.manual_seed(random_seed)\n",
    "\n",
    "\n",
    "def compute_accuracy(model, data_loader):\n",
    "    correct_pred, num_examples = 0, 0\n",
    "    \n",
    "    for features, targets in data_loader:\n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "        logits, probas = model(features)\n",
    "        _, predicted_labels = torch.max(probas, 1)\n",
    "        num_examples += targets.size(0)\n",
    "        correct_pred += (predicted_labels == targets).sum()\n",
    "        \n",
    "    return correct_pred.float() / num_examples * 100\n",
    "    \n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
    "        \n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "            \n",
    "        ### FORWARD AND BACK PROP\n",
    "        logits, probas = model(features)\n",
    "        \n",
    "        # note that the PyTorch implementation of\n",
    "        # CrossEntropyLoss works with logits, not\n",
    "        # probabilities\n",
    "        cost = F.cross_entropy(logits, targets)\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        ### LOGGING\n",
    "        if not batch_idx % 50:\n",
    "            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
    "                   %(epoch+1, num_epochs, batch_idx, \n",
    "                     len(train_dataset)//batch_size, cost))\n",
    "            \n",
    "    with torch.set_grad_enabled(False):\n",
    "        print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
    "              epoch+1, num_epochs, \n",
    "              compute_accuracy(model, train_loader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 84.00%\n"
     ]
    }
   ],
   "source": [
    "print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a ModelWrapper class for plot_decision_regions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All we need is a simple class that implements a `predict` method. That's it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ModelWrapper():\n",
    "    def __init__(self, model, device):\n",
    "        self.model = model\n",
    "        self.device = device\n",
    "        \n",
    "    def predict(self, X):\n",
    "        features = torch.tensor(X, dtype=torch.float32, device=self.device)\n",
    "        logits, probas = self.model(features)\n",
    "        _, predicted_labels = torch.max(probas, 1)\n",
    "        \n",
    "        return predicted_labels.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mymodel = ModelWrapper(model, device=torch.device('cpu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3iUVfbA8e+ZzGTSC0mANEggdKSIFAuKDVERWEWkiOu6ithd3bXuT9etbtFdLKuiYlsUsbsIChasq1JEUHpPIL2QXuf+/pgkJmHSJ5lJcj7Pw7N537nvfU9YPLm5VYwxKKWU6v4sng5AKaVU59CEr5RSPYQmfKWU6iE04SulVA+hCV8ppXoITfhKKdVDtDvhi4ifiHwrIt+LyI8i8oCLMleKSKaIbKn+c3V736uUUqp1rG6ooww4yxhTKCI24AsRWWOM+bpBuVeNMTe64X1KKaXaoN0J3zhXbhVWX9qq/+hqLqWU8jLuaOEjIj7AJiAJeNwY842LYpeIyOnAbuBXxphkF/UsAhYB/Omffxx35uzJ7ghPKaV6jJPjTpfGPhN3bq0gImHAW8BNxpgf6tyPAAqNMWUishiYY4w5q6m6vkvbYHZlb3dbbEop1RPMHfHzRhO+W2fpGGPygPXAtAb3s40xZdWXTwPj3PlepZRSzXPHLJ2o6pY9IuIPnAPsbFAmus7lDGBHe9+rlFKqddzRhx8NvFDdj28BVhpjVonI74GNxph3gZtFZAZQCeQAV7rhvUoppVrBrX347qR9+EopTxEjBBKC3WJHaLRL3GMMhjJHGUXkY6R+Dm+qD98ts3SUUqo7CSSEkIAQsBi8MN+DAbvDDsVQyLEWP6ZbKyilVAN2i917kz0447IYZ5ytoAlfKaUaEMR7k30NodXdTZrwlVKqh9CEr5RSXuib9d9y+VlXMv+MK1j+71fcUqcmfKWU8jJVVVX8675H+dvzf+aFdc/y0bufcHDPoXbXq7N0lFKqHRbPvp28vOLj7oeFBfDk6w+1qc4dW3YR2z+GmH4xAJx10RS+WPslCYP6tytWTfhKKdUOeXnFDF78r+Pu737y1jbXmZWeRe+Y3rXXUdFR7Niys4knWka7dJRSysu4XBDrhllDmvCVUsrLRPWNIuNoRu11Zmomkb0j2l2vJnyllPIyQ0cPIeXgEVKTU6kor+Dj/67n1HNPaXe92oevlFJexmr14dbf38Svr7gLR5WDC+ZMI3FwQvvrbX9oSinVc4WFBbgcoA0LC2hXvZPOnMikMye2q46GNOErpVQ7tHXqpSdoH75SSvUQmvCVUqqHcMcRh34i8q2IfC8iP4rIAy7K2EXkVRHZKyLfiEhCe9+rlFKqddzRwi8DzjLGjAbGANNEZFKDMr8Eco0xScA/gb+64b1KKaVaod0J3zgVVl/aqv80XCY2E3ih+uvXgbNFxNt3m1ZKqW7FLX34IuIjIluADGCdMeabBkVigWQAY0wlcAw4btmYiCwSkY0isvGNl95yR2hKKdUlPfibvzNz3GyunHq12+p0S8I3xlQZY8YAccAEERnZoIir1vxxm0UYY5YaY04yxpx0ycKfuSM0pZTqks6ffR5/f+Evbq3TrbN0jDF5wHpgWoOPUoB4ABGxAqFAjjvfrZRSnpSXc4z7r7mHY7ktP1S8KaMnjiI4NNgtddVwxyydKBEJq/7aHzgHaLiP57vAz6u/ng18bFxuB6eUUl3TutdWU5m8m7UrV3s6lEa5o4UfDXwiIluBDTj78FeJyO9FZEZ1mWeBCBHZC9wG3OWG9yqllFfIyznGhtXrWHJJNBtWr3NbK9/d2r21gjFmKzDWxf376nxdClza3ncppZQ3Wvfaai5KEgb18eOipGLWrlzNpdfO83RYx9GVtkop1Q41rfsF40IAWDAuxGtb+ZrwlVKqHWpa9xFBzg6TiCArFyVJu/vyH7jpT1x/8c0c3p/M7Elzee/VNe2OVXfLVEqpdvj+q818crSUV7YerXe/V9bmdnXr3P/ove0N7Tia8JVSqh3++MLfPR1Ci2mXjlJK9RCa8JVSqofQhK+UUj2EJnyllOohNOErpVQPobN0lFLKC2UczeBPt/2VnMxcLBbhonkXMvuqi9tVpyZ8pZTyQj5WH2747WIGjxxEcWEx11x0HSdNHkfCoP5trlMTvlJKtdPX67/ljZffIDU5jej4vlwy/xImTZnQrjojekcQ0dt5TlRAUAD9B/YjMy1LE75SSnnK1+u/5emnlpIwM4Z+iSPJO1DA008tBWh30q+RmpzGnu17GT5maLvq0UFbpZRqhzdefoOEmTH0SgrF4mOhV1IoCTNjeOPlN9xSf3FRCfdd9wA33Xc9gcGB7apLE75SSrVDanIaYYn1T6YKSwwmNTmt3XVXVlRy3+Lfcc6sszl92uR216cJXyml2iE6vi95Bwrq3cs7UEB0fN921WuM4a93/oP+Sf257OrZ7aqrRrv78EUkHngR6As4gKXGmCUNykwB3gEOVN960xjz+/a+WynlnbZ8sZX3V64l82gWUTGRTJszlTGnjWrVM4NHDGL3j3taVYcnXDL/Emef/Uxnyz7vQAEH3znKNdcuale92zb+wNo3P2TA0ER+ef61AFxzx1VMOnNim+t0x6BtJXC7MWaziAQDm0RknTFme4NynxtjprvhfUopL7bli628+txKEmZFk5AwjGMHC3n1uZUAjSbshs/sX5vCB+99wPAFSYwf3rI6PKVmYPaNl99gV/IhouP7cs21i9o9YDtq/Al8evBDd4RYyx1HHKYCqdVfF4jIDiAWaJjwlVI9wPsr15IwK5rwgc4ToMIHhsAs5/3GknXDZ7K255I4NxoJdWDxsbSoDk+aNGWC22bkdCS39uGLSALO822/cfHxySLyvYisEZERjTy/SEQ2isjGN156y52hKaU6SebRLEITgurdC00IIvNoVoufKc4sJTQpiMqKihbXoZrntoQvIkHAG8Ctxpj8Bh9vBvobY0YDjwJvu6rDGLPUGHOSMeakSxb+zF2hKaU6UVRMJMcOFta7d+xgIVExkS1+JiDKj2N7C7HabC2uw50MBkynvKrtTHWcreCWhVciYsOZ7JcbY948Lq46PwCMMatF5N8iEmmM0R/XSnWytgyoNufNp95h3dsfUVpUio+PlYBUP0ZcMZDQhCCOHSzk4NupXPaLOY0+P23OVGcf/SxnSz5yeDgHVhxh+IIkHFWOFtXhTmWOMuwOO1gMSKe8snUM4BDKHGWtis8ds3QEeBbYYYx5uJEyfYF0Y4wRkQk4f7PIbu+7lVKt05YB1ea8+dQ7fPDeByQujCY0KYhjewvZsyyF3S8kA87W+2W/mNNk/TWfvb9yLbuPJhMVE8l5F57H7i/3sOG1HS2qw52KyIdisFvsiBdmfIOhzFHmjLMV3NHCPxVYCGwTkS3V9+4B+gEYY54EZgPXiUglUALMNcZ4+y9MSnU7bRlQbc66tz8icWE04UOdi4/ChwYz6Ko4Dr2UyRNrHmlxPWNOG+U1A7JGDIUco9Cbs1Qbfg65Y5bOF8292hjzGPBYe9+llGqfzKNZJCQMq3cvNCGI3UeT21xnaZFzgLVenUlBlBa1vU7VMXSlrVI9SFsGVJvjF+gcYK1X595C/AL92lyn6hjirT0r36VtMLuydSq/Uu605YutvPTEf4iYEow9yoeyzCrS1uYS5BtMWVmZyxWuza14re3Dn/tTH/7+l1MJsgYjVnH5TEsGjjticLktvCWOlpo74ueN9rjo9shK9TBVpQ7SPsmmrLACi49QXlDBwGvi6DM84rgVri1Z8XrxtTMBWPfSR5QWJWPztWHztzHkl/1qZ+nUfaYlA8cdMbjcFt4Sh7tol45SPcj7K9cy7MpETvvtOM5+cBL+vfwYdFVc7YrWhitcXa14TZgVzfsr19ar9+JrZ/LEmkd47rOlDBieyKhFgwkfGOLymboDx43V2ZIynfX35Q1xuIsmfKV6kOZWtDZ3Da1fNdvwmZasxG3Lat2O4C1xuIsmfKV6kOZWtDZ3Da1fNdvwmZYMHHfE4HJbeEsc7qIJX6keZNqcqRx8O5Xcffk4qhzVK1pTMccsLbrO3ZfPwbdTmTZnar16t3yxlQdv/ge3z76L/OwCdq84VPuO3H357F5xiPzsgkY/b1hnwzgbe2971Y37wZv/wZYvtjb599VRcXQWnaWjVA/T3L7zrZ2lU3dgs2aQdsfzBwgKCqKkqBT/QD8KCwsZdmWiy889NUvHVdw12ze0dkaRN2lqlo4mfKVUuzx48z8IP9e/dvUuQO6+fHLXlXDXI79u9nNP8da42quphK9dOkqpdnHHIK0neGtcHUkTvlKqXdwxSOsJ3hpXR9IuHaVUk1z1YQO19+x2O8fy8/AJFsoKK7AH2ajIcRDRp1e7+vA7+ntxFZerPvyuRlfaKqXaxNVK05ee+A9VpQ6GXZlYe/5s1icV9D2jNwF9fClOLyd5VQYBo2yMnJrIsYOF7F5RzKEVGZSVJWO32/HxsxA/O8rlStzO/F7qxtXZWzB7gnbpKKUa5WqlacSUYBz2ytp7WdtzGTA/mtD+QcQPjie0fxAD5kWTtT239pnBc/sTEhHMQ68/SEhEMIPn9u/01auuvpe6cd31yK+7dbIHTfhKqSa4Gti0R/lQVvjTytuGq3ErKyoITQqiOLO0tow3DOL2xEHahtxx4lU88CLQF3AAS40xSxqUEWAJcAFQDFxpjNnc3ncrpeprbo59w/735ubZ1wxs1p26WJZZhT3op5W3NatxA3r5A2C12ZzXUT9tj+xqELdune4aLG1qzryr96Zvz6a4qJjbZ9/VJebYt5c7WviVwO3GmGHAJOAGERneoMz5wKDqP4uAJ9zwXqVUHTV91OHn+jP+/mE4+pfxwXsf4H+qhfH3DyP8XH9eeuI/PP/PF5ss8+pzK2tXnLpaaZq9vgBLmbXR1brmmIUDK1KJHB7ucnVqR61ebfj9N/e9pG7LZPvyvcSf08dl+e7IHSdepQKp1V8XiMgOIBaoO8VmJvBi9bGGX4tImIhEVz+rlHKDhscXutrpMmJKMGmfZDdZpu6Rh67Oml143eXH3Wt4/ux5F57H7h/3sOGB48+jdVWnOwZLmzu+seF7i4uK6X9OLEnn93NZvjty6ywdEUkAxgLfNPgoFqh73llK9b16CV9EFuH8DYB7/3Y3Iy8Y4s7wlOrWGh5f6Oxbj6Uk/ae+dNf97/XLNDzysLGzZtuTFDvi/NqWHN9Y9723z76LAVPjmizf3bht0FZEgoA3gFuNMQ2PUnc1L/S4BQDGmKXGmJOMMSddsvBn7gpNqR6huZ0wofH+99bshumtWruQqicuvHJLC19EbDiT/XJjzJsuiqQA8XWu44Cj7ni3Uspp2pypzvnss5wt1cjh4ex/+QjxF/WmorS89jjDkpxyPrnvGyqKKsFAwfIiRl4xCEeVo96iKE8PZLZ207KG33/dhVTuKN8duGOWjgDPAjuMMQ83Uuxd4EYRWQFMBI5p/71S7tWwj9put2MTG5mf5tWugC0vrKSKKuLPicSvjy+l6eUcfC2dw69lcJgMjy2KaqgtRwu2dmygo8YSvFm7t1YQkdOAz4FtOKdlAtwD9AMwxjxZ/UPhMWAazmmZvzDGbGyqXt1aQan2cbUb5I5PdpP2WS5j7kyqvZe7s4BDL2XyxJpHvGYHSW+Joyvq0K0VjDFf4LqPvm4ZA9zQ3ncppVrO1SCmXx9fKour6t0LTQqitCi50Wc8MZDpLXF0N7rSVqluytWgZGl6OdYAn3r3ju0txC/Qr9Fn9GjB7kM3T1OqC3vzqXdY+9aHlBSU4B/sT1xcLEeOHKW0qBSbrw3bASujFg2u7Y/PXl9ESXoZW/66l8riKqwBPpRnV3DhpRcALR/IfPOpd1j39keUFpXiF+jHyLEjKCgsaPJUrOYGYBvuZJn2fJrLnSxbU6eqTxO+Ul3Um0+9wwfvfUDC5X0J6m8nb0cxB98+SO9Twok/L5FjewvZ/3Iqu549jFiFqJhIppxzBus//JSI0wNrB22z1xcxYEQi0LKBzJr3Ji6MJjQpiJwd+Xy38jviT+3L+GuOH2BtyQBsa3eybMugrtKEr1SXte7tj0hcGE1Qoh++PmDvbSNxQV9S1+VgsQrhQ4MZMB/ngOzbjwDOwdARVwz8aTB0OOT2y6+3urS5RVE17w0fGgyAX5QvA+ZHc3RNNsNmDTxuxWpzK2DB9SrZwXP7Vw/SPnBcDC2pUx1P+/CV6qJKi0oJSvDHR8AigDGEDAigLKu8toxzQPanVbTu2DGytMi5O2YNR6WDkIEBlObUeW8rd8dsbVy682XbaMJXqouyB9jJ31uItfq/YhEhf18x9kjf2jJ1B2TBPYOhfoHO1bk1LFYL+fuK8etV572tPOJQV8l2Dk34SnmJLV9s5cGb/8Hts+/iwZv/0eyujQn94zj4ahp5e4qoqjSUZZZz4OU0ghP9cVQacncWcGBFKpOnTWbJTQ9TkFfolp0qz511NgdWpJK7swBHpaE0s5z9L6cSNbxXm3fHbG1cHbXjZnenffhKeYG2DELaBewFFrY/eYSqKoOPj2Apg/SPc0n7KBe/QD/Om3Ue/jYfso8e5Is313P+VdOB9q0uvfjamQCse+kjSouS8Qv0Y+zYsRSkFbR5d0xdJds59BBzpbxAR60szc8tYOn1f+bx6YHcsKqIa5+4l+CwoOYfVF1WUytttUtHKS/QUYOQX771KTOSLAzqY2dGkoUv3lzfrvpU16YJXykv0BGDkPm5Bfy47kvmjXP+IJk3Logf131JQV5hM0+q7koTvlJeoGYQMnt3LukH0sjendfoIGR+bkHtIGxTalr3EUHOobqIIGttK7+ldbhLawekVcfQQVulvEDNYOPyJS+Tl5ZNWN8IFtwy3+Ug5JdvfYqlwSCsKzu//ZFv08p4ZVt6vfshmT8CtKgOd9BVsd5DE75SXmLAiET6WIXXF/fnhlVFDBw54LgyNd00j1/cmxtWfclpF09pdBD2V0/e6fJ+7UBuC+pwB10V6z20S0cpL9GSAVZ3DMJ29kCuror1Hl6b8F9++D/87Rd/Z/uGnezctIv05AxPh6RUh2nJAKs7BmE9MZCrq2K9h7vOtF0GTAcyjDEjXXw+BXgHOFB9601jzO+bqvPia++jMiyJja+uAhFSUg4xcHR/fH2dhy0PPHEQo6aMdkf4SnWI/NwCnrvvaa76w6Jmu0xqWt2bDhRyzUtHWPbzOGYkWfho+Qcc2nmIq/6wqLZMmJ+QfjidyJio2hZ63X74owdS+fsv/8wdz91LenJGvS2EI8NCGRZczrVvpHI4p4J+vWwMC7Z1aF9+Tzw71lu5ZeGViJwOFAIvNpHwf22MafG/qP/tyzY/HDlWe12Ql8OOz96uPVpr4/rVhITasFiEgPBA5t27AB+rj+vKlPKANctWsfe9tSRdOLXZZPrPxX8lPy2T1OQsBoRb2J/rIDo+kgqHhT62UpIunMrOb38kPy2TksISKotLsAb44x/kT0jfqHr99Utu+Aflu3dQHBlHaUAVCbOiaxPtd09tp6KkksRL+hIYbacotYyU9zLpExTBA6/+ocP+LnTv+s7ToUccAhhjPhORBHfU1ZjgsF5MmHFV7fWYaQvISk0BYNtHr/GPKx/C4mOhvLKCGTfMICDY+R9DTGJ0R4allEutGVwF5wDrl2u+4d0/Psmymf7MXFHM6VfMYNNra3h4urOOa5+4F2NM9crZaJcrZ48eSCX9h128PieIKSuOMPpXJ9QbLO03sy9pn2QzeEr1mbZDICq+N7nrSjr076O5LZdV5+jMWToni8j3wFGcrf0fGxYQkUXAIoA7/vgQSZNnNVqZzddOdP+BAERfdRc1s5WP7vuRz5c/jNXHh/TMDGIGRhLZtxdWP1/OXHA2VptOTFIdr/7AaEmLukxW/OFZFo2yMbqvD5efYOOJB1/g1vP61qsDaLLe1/6xnPkjrYyKtmK3gsNeXu8d9igfygor6t3Ts2J7js7KfpuB/saYQhG5AHgbGNSwkDFmKbAUju/SaamYgSO47L5nAagoL2PTqheprCghbW8KD1/9ML0iQwEQq4V5v11AQHBAm78ppVypad3/9jJny3reuCDmvdp0K//LNd8QIBXcMMH57/GGCTb+s62C2CBHbR2Xvfw5DgO/XRDmst6a1v3iq5zvGB7lQ9aBY/SKjcBWPfZVllmFPchW7906gNpzdErCN8bk1/l6tYj8W0QijTEdOi/L5mtn0sXX1F5npBykrNT5q2vqrs38c9ESbDYr/eIjiPayf/ASHsTkeWci0mh3nOpkLR2EbWqFa8NWfs0Aa3lJCYtH24gOdo5DRQc5W/kPvJvGKVEVRMZEcW5MOT+kV1FVFcy8xw/wyOX9mZFk4cPlH3B45yEqy8qZP9JK32Dn5LvfjPPl2nVZHLbbSZzUr/pM2wIsZVZy9+W7dQBV++i7Brftllndh7+qkUHbvkC6McaIyATgdZwt/kZf3tYWfmsZY9i27lWOpR2E/KNcf85gJo5I8Hii/fzHQ7y9YQ/dJd8fzi9h+AXjEbruN/T1e1+S8tW3DDz7NOb+en6j5WoGYBtqOLgKPw2w7supws96/N9NaaUhyG7BGuBPZVkF5VUGi0Xoa68krcxKeGQIFQ6hj62M3akl2HDUe76s0lBm9yG0T0RtIgbcmpzrrqRt+ENEk37n6/BBWxF5BZgCRIpICnA/YAMwxjwJzAauE5FKoASY21Sy70wiwqipcwEoys/jpc2f88TaNST0DmH6hAGMHxzjkbgmj+jP5BH9PfLujpCTX8SGnSmeDqPN8otKeOvrLdx6cgh/fv9/+FRZCO0V4rLsiHEnNFrP2qdX1359LLeAXRu2s+AEGweOGU6bfR4hYcG1n5eVlPHZOx8zc6gv7+ws54w551NaUs4nr3/A5CE2lm+rIGncaLas/5axQ305nGc4fdZZ2P3sjb4/Y4fz/4MTTxgO1WEWpeXhcDiwWNq2LEdX0nYdXrsffme18F2pqqzEUVXFd+8uxa80E19HKX+6/BT87LbmH1bd0sPL18KRTdx2eigPf3aMyr5juWXu2e2q89J7nmSk9TD3TwnkgfVF/FDZj9f+vLj28yUrPoQj33HL5BCWfJ4PsWP5Yuvees+8lxLEghG2emVumXtOq+LYciCdf6/fhn+Af5u+jxWvfUDclKh6vxUbY0hZn8ncS89rU53tUVRewelXnddj9/3v8BZ+d+NjteJjtTLh0psAyMk4yqKXnqe8tJizBti5aMJAoqsHf1X3l5VXyKpPN7ByjrP1fcWJgcxZuYlfzpxMRGhgm+rcdSid7bsP8NRVQditwk2T/Jm87ACH03MZFN+brLxC3v9iMyvnBGO3WrhqfBAzXtpARm5e7TPzR9l5eWsO88ck1JaZs3Izv5x5eqvimjgkjolD4tr0fQAc2L4dv7EBRNU5vCVzXz6xmX48uXBKm+ttq/yiEv700joqvbMt2yolZRU4YiPpN/r4fZUaNaLxjzTht0Cv3jFMvvIeAA5u38RtK99lRHgFoQF2rrtwLFZd8NXlZOUVcu2D/2Hp3QubTY4vvvcV05MsRFYPwkYGWZmeZOF3z/yX5e9/w/tLbmHC8AR2HUpn2i1LWPvorQyK733cNVB7b1Bcb+aPtNInyEJpRSV9g3yYP9LKLQ+/gr/dzphBcUxPcnaxXPJcMkvnxBBqKeTUIT5EBzn/va3dU8GCUTZKi/IhPKo2rhdWfcltCzrvbNdbZ07hjhX/hZkQkRBE9sFC9r+Tzt/mXtRpMdQVEujPX69o329f3mTjrhSSD6W6pS7t0mkDYwwlRYXkZRxh3/vPEhroy/kn9OGCCQM9HZpqoYeXr2XVuk+Zfu4ZzSbHGbc/xtGM4yeUpWTmExPowAT04ruX7uNndzzO4f176Tcgibf+dsNx10DtvYN5Bj+bYBwGhzFYRBCLUGUsDO/rR06FL1aLIaeghDBrOXmVvhwrKsUm4OPj/EFQWuHAZnGOQ8X2DquNK6Z3JO8+dKMb/7aat+br7fzrnfUcSMshsW8vbp05hfMnDe/UGFS1U27SLh13EhECgoIJCBpKzPV/B+D9D1/l9WXf4yjI4NbzhxEcYGdgXJSHI1Wu1HTRPHFxJNet2sDPp5/aZCvfVfL8etsB5t75L5bNDGDWimxe+2gz23bu4805gVy8ch8ffL2j3vWe5AwcDlPnXjHL/3Izdy9ZzhPTA7huVTFP/vYaFv/x6UavX/v7fW3uQupo508argm+C/D53e9+5+kYXErJLfldRkGZp8NosT4DRhI/ejIRQyfxya5cPtyZyxdfbyI8yE5sRHDzFahO8+Qb6xlsS+Oi4YFkF5TzfVolJ49q3W9nF/36US4ZVMmCE+xkFjv401tbuXK0lbkn+JFVVMWf39rK5SOk9nr510f4cMN2zulbUHvv4dU7uHSYtTaOZ9fv49y48kav2xKn6oHiJz7Q2Edeuz1yVxUYEsbIMy7i5NmLibvkfp7aVMYty77ijuc+o6SsvPkKVIeqad1fcaKzpXzFiYGs+nQD2ceKWlzH19sOkJudzQ3jfQGYN9KGXSq5cozzevZwG47yEq4Z55weed0EP7bu2MeW7Xu5boIfAIvH+5Gbnc2p/Z0zvy4c4se2nfuYP8a50nbe6AC27dzH9KF+bY5TqYa0hd+BbHY7cSMm0Hv02fjEjuLZ19bw1sbDZGVmEupvxddqxVf39ulUNa37swc5E2uAr6XFreesvEIW/m4ZL67+itmDKzmtn5WFbxWTV2IYHuXD8CgLUUFWntxQwogoCxNjfbD5GML8fUjOq6Ci0nB99dYJfj6GnBIHK7eW8MbWArKLqxjRq4rxsTaCAuwUFRVRWlbG4Xzh5ISAenEO6teHhb9bxjnjhxHg59vhf2eqi2miha+Dth6w7/uvKMw8Qtb2/zF1RASXnzVSE38naWwAtiUDnTUDvVuSC/H1gYoqQ3yohUN5DuxWwRiwWi2UlFfh6yMYQACLCFXGUFFl8Pd1zrBxOBwYA2VVhsRwH5KPGXytgo+Phd7hwWTkFlBV5cCBhdjIn7oEY3pHMuXEwS0ecFY9UBODtsW8l3oAABooSURBVJrwPaiyopyjB3ZxYO1zRIX6ceaw3syYdNyecsoLZOUVMueOJU0MqN5ab0C1YfmGn7e0THNxtOQZ1cM0kfC1D9+DrDZf+g0+gTNufJjhC//MR3nRXPPcd/zikY/Ym5JJWnZ+85WoTlEzF39IbzvTkyzc+dhr9a5fWPVlk+Ubft7SMu54RqkamvC9yMhzLmPCwt8yZuHvePg7K3euzuCe59fzwrrvSc7I9XR4PVbDgd6GA6wNB1RbMjDclsFjdww4q55NE74XCggOYez5l3PavFsIn/ZrDg+cy91v7OLO5z+jqKRrD2S3RFZeIZfc9aRHE1ndGGpa1eBc9bry+3zmj7RChXOr7borXKHxlbl1W+MtKdNQW55Rqi4dKfRyYZF9AOh7zR/Iz83iuhcfxe4jTIiz8bNJSQQH2LH7dq9N3V587yty05I7fYuAxmJYv3k3RzPKeOyrPMKs5Xy2vwRfq/D81kx6h5fWPhOTvpvbFkytLf/ytox6ddZ8DrSoTENteUapunTQtovav/kzsg/vIu/Qj1x8Yl/Cgvw476Qkj+/j317eMCjpKgZjjMfjUqpFmhi01Xn4XVR4dH9ih46j34lnctD05YccH17/71rEOBgSF+Hp8NrMHatgOyKG7/ckezwupVpEV9p2XzZfO3EDhzLytPOZdP0SPsyLZvGyjVz92EccyczrUn3+3jAo6SqGNz/6hnc++VYHS1WX55aELyLLRCRDRH5o5HMRkUdEZK+IbBWRE93xXnW8E865jHFX3M+ohQ9w30cFXPvyHv7vxc/471c7OJKZ5+nwmtTWQcldh9JJnHUPe5Iz2lymZpD2iTfXHxdDlK2UM2LKj4vr369/Um9w2RsGm5Vqilv68EXkdKAQeLGRM20vAG4CLgAmAkuMMRObqlP78N0n8+hhCo/lcvCz10gMquDO2eMJCWzb6UYdqa2rYF1tQ9zaMjWraHMq7Vil/rmwR7IKsPlA7/D6m+BVGgu9rGW1K15bs+WyUh2mo7dHNsZ8Vn2IeWNm4vxhYICvRSRMRKKNMe7Z1V81KSqmH1Ex/UgcNpqCvBxufOmfBPpaGBtt56pzR2KxSJvPM3WntuzhvutQ+nHbENccNtLSMvW3Sy7mtb/f3ooVr84tlqdPHtOqLZeV8oTO+q88Fkiuc51Sfa8eEVkkIhtFZOPbK17opNB6luCwXky++g+ceMUD7A8/mauX7+Gyhz5kzdfb+XLrfrx11lZj7nr8deaPtDKqr435I63c8ehrrS7jjhWvza28VcobdFbCd/UrxnGZxRiz1BhzkjHmpFlzf94JYfVsA086k1MX3M6ZN/yNj21n8HJ6f6785xruWLaeLfvSPB1es2pa7jVbDl83wbkCtm4/fXNl3LHitbmVt0p5i85K+ClAfJ3rOOBoJ71bNcPma2fw6AmMOXMGp938OAMv/yuP/i+fa5/4hIzcAk+H16ialnvNGa/R1efC1m3BN1fGHSte39tR2OTKW6W8RWettH0XuFFEVuActD2m/ffey2KxMGnurygtLuI3bz1FRWkRgwKLmDoqlgExEURHhno6RAC+25XMt+UVPPtd/dlHNt/kFpdxx4rXI1kFWHA0uvJWKW/hrlk6rwBTgEggHbgfsAEYY54U5/LPx4BpQDHwC2PMxqbq1Fk63iX10D7yMo6QunkdScEVRIb4ceNF42oP1FZKeQndD1+5izGGyopyMpP3cmDdC0waGMri88d4OiylVA1N+Kqj7Nv4CWnbv6Y4J53rz04gLNCf0YNiu/yePkp1WZrwVUerrChn96bPKM3PpWLPZwyMDmPaiQmMTerr6dCU6lk04avOVF5WSmV5OVvfexZ74RH+b/aJXjPQq1S3pwlfeUpZaQkb33gcKkpJ8C/i0pMH0js8iKgG2xQopdxEE77yBqkHdpJ9eDfp27/hpGgLoYF2rpo6Wmf6KOVOmvCVN3E4HBTm5ZCXeZRDH75AWJCd6WOimTpugKdDU6rr04SvvN337y8nZ88mbp46kN5hQSTGROhMH6XaQhO+6goqKyvY/vkqyosKkJRNjB0QyaQh0YweGO3p0JTqOjThq66muCCfkqICdn+yktCKTPx9HNw3dxJ+9u51YLtSbqcJX3V1uRmp/LjqKQYEVXD1ucMJDrB75SEuSnmcJnzVXRzd+wNpuzeRc2g3ZybaCQ20c+nkYVitPp4OTSnv0NEnXqmO8Zcb51FYePz2xEFBwdz92CseiMjzYpJGEpM0EmMMGSkHOXgsm9WP/oeoUH+mjoph2kk600epxmjC92KFhQUMuPrR4+7vf+YmD0TjXUSEPvGJ9IlPZODIkwBY9f5y3lq2GUdJHvdfOpbYqDCd6aNUHZrwVbcxetoCAIry87j/w5X4Zm3izBHRDIuP4IQBuqePUprwVbcTGBLGpIsXkZ+bxY85Waz+cg29Pt5FgK+Fe+ZMJMDP19MhKuURbkn4IjINWAL4AM8YYx5s8PmVwN+BI9W3HjPGPOOOdyvVmJDwSELCI4kbOBSAvOwMFj//KH5WC6f092feGUOx++o0T9VztDvhi4gP8DhwLs6zazeIyLvGmO0Nir5qjLmxve/rydKS91NVVUVuVgb3Xjm99n5PHsRtjbCI3ky++g8AbP/2QxY8vo6LRgQTEezP1JOS8LXpL7yqe3PHv/AJwF5jzH6A6nNrZwINE75qpaCg4HoDtLlZGdiCeuEXGceAqx+qva+DuK03aMI5JI0/mz27f2BbUQEvL3mFuIgAzhwZy/njB3o6PKU6hDsSfiyQXOc6BedB5Q1dIiKnA7uBXxljkl2UUXU0bLXfe+V0l7N2VNuICP2HnADAkBNPAeC995fz3+c24Cgt5I/zxhMRGqgzfVS34Y6E7+q/hoaruf4LvGKMKRORxcALwFnHVSSyCFgEcMcfHyJp8iw3hKdUy9XM9CkuyOf2d58loPAwsyfEE987nMH9ens4OqXaxx0JPwWIr3MdBxytW8AYk13n8mngr64qMsYsBZaCrrRVnhUQHMJp839FXlY67x85RPonXxBRto1gf1/uuGQCQQF2T4eoVKu5I+FvAAaJSCLOWThzgfl1C4hItDEmtfpyBrDDDe/tcppbOXvdeWMwPnW2CDAGEHBUEhEdT05GGnn/uhqLzY+oC2+pLXYsO6vF71CtExbZh7DIPgwePQGA/NwsrnvuYUL8bZwU788vzj3BwxEq1XLtTvjGmEoRuRH4AOe0zGXGmB9F5PfARmPMu8DNIjIDqARygCvb+96uqLmVs8bHh/gb/1N7vzzrML6R/Ti67EYGXP0o1r3bEYuN9Fd/S/aqh2vLGUdli9+h2ickPJLTr/0zANu/WcvPl22mJOco1581gJNHJug0T+XV3DIPzRizGljd4N59db6+G7jbHe/qyXysNuyR/fAN7sWoG5+ova/J3DMGT5zK4IlTKS8r5a3vvmTpI28zODqISUN0Tx/lnXTisVLt5Gv3Y+Sks3GMn0JVZSWr1r/Oqme/xlFRwh/mTSAiNNDTISoFaMLvVNmpyeT85bLaa0dFGeJjBQyLLxgHBo4uuxGx+dFnrrPbwDgcteUz31uCcVRSVZjL1seuq71fWfBTH77yHIuPDxYfH0af55zpU1pcyG1vPoGjvJShISUsOGMoCdERHo5S9WSa8DuTj5W4G16ovTy67EZirnqstq++bp+9xebc70UsltryjopSYq56jIqsZPx796u9f+ixKzrve1At5hcQxGmX/waAI/t2cN8Hq4mXHwgPtLPo/NF6gIvqdJrwO5EgmMryevfqH0AjlGcdpqooj5THr6ht3ZuqSvY/cxOO4mOUZxwADGVZh2ufsshPPxQars6te195TuzAYcQOHEZJUQHFhQVc/9wjRATaGBUbyC+n6kwf1Tk04XcisViw+dafv113Fac1tA8Wmy9isXDS3Str7+9/5ib+9Pwq7r1yOv2Shh1Xb1lEZO3XOvXSu/kHBuMfGMwZi537C+76dh1XLdtAcV4Gt5ybRHhIAEP69dbVvapDaMJXyoMGTTiXQRPOpaK8jBVfr6P0wDHkv2u55JRBnHuizvRR7qUJv42aW+B084yJVDrq7zBRVV7G4WduAItzcVVVUS7l6fsBnF011d07xlHFhj/Nrn3OVFWw+PxxAFj3bMdgsNp+2tNdF151fTZfOyNPd+6AWlI0g7c//y/vbPoCMZX836Xj6B2uXXKq/TTht1FzC5wqHYb+N75Y77ODj15B5IW3QfWv6+mv3E32mkcwdbYeEgQcVcTd/FNyTnlsIfE3/Ye0V+4h+/1HqMzPwh4aVfu5LrzqXvwDgxkzzblYvbS4iDveeAwfRzmJgaXMO20QEaGBhAUHeDhK1RVpwu9EFv9g8LHiG+mcYSNWX6Kv/BeOijIsNnvt/6Y8fgUigjGmXl9un7l/QkRIfvRyXXjVQ/gFBHLqwjsBOLJ7G//ctpPM3Zs4Y4A/86cM18SvWkUTvlJdROzgE4gdfAKOsy8h88ghbnjuCfoEWxkeE8zV543ydHiqC9CEr1QXY7FY6BOfSJ/r/wbA7m/Xcc2yrynJz+X2C4bQJzyY6MgQnemjjqMJ300aHj/ocFRRUV6GAFbfxrfSNbUDtQ4qj2UABuNwkPr8rT+Vqaps5GmlnDN9mHAulRXlPP3xm1QUF2DP/JbZpw5iyugET4envIjUX/jjPbx9P/yGs2Fqjh/08QtgxNUPsfEvcwDBmCosNbNyyssQHx9qzowxVRWIT/3dFX0CwxCrLxHn/XT8b/rK/6PuOTOmqoLImJ9W2tadgaOzdBRAQV42e79ei6Rswtcq3DFrLNGRoZ4OS3WGU25q9Fc7Tfhu0tjxgzWLplryzMa/zCHuhhdrB29rJD96OU+u2eT+oFWPUFZawsaV/8JOBfGB5Vw3bRT+dhsBfr7NP6y6niYSvnbpKNXN2f38OfUK5+7kR3Zv4e61G8g9sofpw0MJDfRl+sQhWK0+zdSiugNN+Er1ILGDxxA7eAwOh4P9B3ZTdCyH1x9ZwZkj+/LLqaM9HZ7qYG5J+CIyDViC88SrZ4wxDzb43A68CIwDsoHLjDEH3fFupVTrWSwW4gYOBWDIiaewZ8PHLHrmU8qKC7njomEk9O1FgJ+vzvTpZtqd8EXEB3gcOBfngeYbRORdY8z2OsV+CeQaY5JEZC7OQ8wvO762rqstu1Q2fMZRUUbyo5cfX7Cqwi0xKtWYgePPYuD4s6iqrGTJ6hepKDpMcNEhpo6OY1BsL4b06+3pEJUbtHvQVkROBn5njDmv+vpuAGPMX+qU+aC6zP9ExAqkAVGmiZd3tUFbpbqbvKx0cjJSObJpHVFV6fxqxhjieod7OizVnA4etI0FkutcpwATGytTfej5MSACqHdUk4gsAhYB3PHHh0iaPMsN4Sml2iIssg9hkX0YMHwMFWVl3LviHwRZ9hLtX8Wds8djEcHHx9J8RcpruCPhu/pp0rDl3pIyGGOWAktBW/hKeROb3c5pP78XcM70uWb5RxRkpTJ3XCSRwXZOGz0Qm8708XruSPgpQHyd6zjgaCNlUqq7dEKBHDe8WynVyWpm+hhj2PTjZkqLCnj6n69xzqhYrjpPZ/p4M3ck/A3AIBFJBI4Ac4H5Dcq8C/wc+B8wG/i4qf57pZT3ExEGjnSe0zBi4hT2bP6MRUs/wFFZzm0XDmN4/z4ejlA15JaVtiJyAfAvnNMylxlj/iQivwc2GmPeFRE/4CVgLM6W/VxjzP6m6tQuHaW6JkdVFRvefprywlxCKzOZd0oi0RGhJMZEeDq0nkG3VlBKeUJ2+lEyDu8hc+e3xJJBiL8vt846Sbd16Eia8JVSnlZRVkbBsWx+eHMJQ6J8uXfORF3Y1RE04SulvMmR3Vs58NUqivNzWDCxD/0igzlhYIzO9HEH3TxNKeVNYgePInbwKIwxfL7xM8pTCsldtYaR/cIZP6gPZ49J9HSI3ZK28JVSXqGivIyKsjJ2ffYWjqM/cP15Qxk1oK+nw+p6tEtHKdWVOBwONr71JFVFeQRV5HDtuUMJDw4gJkoPcWmWdukopboSi8XChEuuByA7NZmndm0i++BOhgYcIzzYjyvPPoGggMaPDlWuacJXSnm1iOh4IqKdi/nzc7PIKyrg2qf/Te8QO2P7BXPF2SM9HGHXoV06Sqkua/dXqzmy5VMWTOzLsLhexPcJ15k+2oevlOqujDHs+OoDygpyKdz7PyYP6c2QuF6cfkJ/T4fmGZrwlVI9QWlxEYXHcjm0+SNsGduxiuHe2eOIDAvydGidRwdtlVI9gV9AIH4BgURe+HMASosLuX3lI4SZXH49YzSB/r70Cgn0cJSeoy18pVS3l5OWwsHNH3EsLYWx4SX0DQtk5imDCAn093Ro7qddOkop5ZSVmkxpcRF71jxNdKgvY/qHsfCsbjTTRxO+Ukq5tuurNWRs+5S54/sycUgMoYF+WLvyTB9N+Eop1bQfPn2H4px0ylJ+5KITY4mLDObkEf08HVbracJXSqmWKS4sIDsthbQdG7ClbcHf18pvfjaWPr1CPB1ay3RUwheRXsCrQAJwEJhjjMl1Ua4K2FZ9edgYM6O5ujXhK6W8QVlJMRtWPIS/j4PEEAe3XDQWP7vN02E1rgMT/t+AHGPMgyJyFxBujLnTRblCY0yrJsJqwldKeZvk7RvZ9fl/OTXWMKBPKJNP6E9okJfN9OnAhL8LmGKMSRWRaGC9MWaIi3Ka8JVS3caR/bsoLS7k4MfL6RduY2S/Xt6zp08HJvw8Y0xYnetcY0y4i3KVwBagEnjQGPN2I/UtAhYB3PHHh8YlTZ7V5tiUUqqz7P5mLfk7PqeitIR7Zo1kYGyk545vbE/CF5EPAVenENwLvNDChB9jjDkqIgOAj4GzjTH7mnqvtvCVUl1NZUU5G99ZRnnqdn5xeiK9ggMYMziuc4Noz9YKxphzGvtMRNJFJLpOl05GI3Ucrf7f/SKyHhgLNJnwlVKqq7HafJk0ezFFBcdYvW8neT/sxHfdWsIC/bjxwtEeP8ClvV06fwey6wza9jLG3NGgTDhQbIwpE5FI4H/ATGPM9qbq1ha+Uqo7MMZQXlbKty//jRCbg/hguOOS8R3X5dOBffgRwEqgH3AYuNQYkyMiJwGLjTFXi8gpwFOAA7AA/zLGPNtc3ZrwlVLdUfL2jez89G2mJFgZmxjJ8IS+7p3powuvlFLKuxzc8T3FBbmkfv0uw/r4kRQdygJ37OmjCV8ppbyTw+GgsqKcQ99/Qd62jxBHJbdNH8GQ+Ki2VagJXymluoaqyko2vPUkVcX59DI5XHvucIb079PyCjThK6VU15N19BB7v1lHcM6P9O0VxOVnDCGu93Ez3+vTE6+UUqrriYzpT+TPrqa0uIiK8jLufuMRevlWERNsadNMH23hK6VUF5O8czPJ36yhOD+XK0+J5bSR/QgO9HN+qF06SinV/Rhj2PG/daRufJ+TB4QQ0yuIS+/6t3bpKKVUdyMiDD9lKoMnnEnRsTy+2buVS5sorwlfKaW6OKvVRmhEFKERZzdZztJJ8SillPIwTfhKKdVDaMJXSqkeQhO+Ukr1EJrwlVKqh/DaWTpBdiuRwb6eDkMppboNr114JSKLjDFLPR1HS3SVWLtKnNB1YtU43a+rxNpV4qzLm7t0Fnk6gFboKrF2lTih68SqcbpfV4m1q8RZy5sTvlJKKTfShK+UUj2ENyf8rtQ31lVi7SpxQteJVeN0v64Sa1eJs5bXDtoqpZRyL29u4SullHIjTfhKKdVDeGXCF5FpIrJLRPaKyF2ejqcxIrJMRDJE5AdPx9IUEYkXkU9EZIeI/Cgit3g6JldExE9EvhWR76vjfMDTMTVFRHxE5DsRWeXpWJoiIgdFZJuIbBGRjZ6OpzEiEiYir4vIzup/qyd7OiZXRGRI9d9lzZ98EbnV03G1hNf14YuID7AbOBdIATYA84wx2z0amAsicjpQCLxojBnp6XgaIyLRQLQxZrOIBAObgFne9ncqzgM6A40xhSJiA74AbjHGfO3h0FwSkduAk4AQY8x0T8fTGBE5CJxkjMnydCxNEZEXgM+NMc+IiC8QYIzJ83RcTanOV0eAicaYQ56Opzne2MKfAOw1xuw3xpQDK4CZHo7JJWPMZ0COp+NojjEm1RizufrrAmAHEOvZqI5nnAqrL23Vf7yrRVJNROKAC4FnPB1LdyAiIcDpwLMAxphyb0/21c4G9nWFZA/emfBjgeQ61yl4YXLqqkQkARgLfOPZSFyr7ibZAmQA64wxXhkn8C/gDsDh6UBawABrRWSTiHjr6tABQCbwXHU32TMiEujpoFpgLvCKp4NoKW9M+K4O4PXKVl5XIyJBwBvArcaYfE/H44oxpsoYMwaIAyaIiNd1lYnIdCDDGLPJ07G00KnGmBOB84EbqrsivY0VOBF4whgzFigCvHb8DqC622kG8JqnY2kpb0z4KUB8nes44KiHYuk2qvvE3wCWG2Pe9HQ8zan+dX49MM3DobhyKjCjum98BXCWiPzHsyE1zhhztPp/M4C3cHabepsUIKXOb3Sv4/wB4M3OBzYbY9I9HUhLeWPC3wAMEpHE6p+gc4F3PRxTl1Y9GPossMMY87Cn42mMiESJSFj11/7AOcBOz0Z1PGPM3caYOGNMAs5/nx8bYy73cFguiUhg9UA91V0kUwGvm1VmjEkDkkVkSPWtswGvmlTgwjy6UHcOeOF++MaYShG5EfgA8AGWGWN+9HBYLonIK8AUIFJEUoD7jTHPejYql04FFgLbqvvHAe4xxqz2YEyuRAMvVM98sAArjTFePeWxC+gDvOX8mY8VeNkY875nQ2rUTcDy6obefuAXHo6nUSISgHMm4bWejqU1vG5aplJKqY7hjV06SimlOoAmfKWU6iE04SulVA+hCV8ppXoITfhKKdVDaMJXSqkeQhO+Ukr1EP8PJaExXzJuWi0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.plotting import plot_decision_regions\n",
    "\n",
    "plot_decision_regions(X, y, mymodel)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch       1.3.1\n",
      "matplotlib  3.1.1\n",
      "numpy       1.17.4\n",
      "torchvision 0.4.2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}