SOLU-8l-old-Observations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8e886a88-4bb6-474b-a55b-6c7734f0fec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import einops\n",
    "import tqdm.notebook as tqdm\n",
    "\n",
    "import random\n",
    "import time\n",
    "\n",
    "# from google.colab import drive\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "import os\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from functools import *\n",
    "import pandas as pd\n",
    "import gc\n",
    "import collections\n",
    "import copy\n",
    "\n",
    "# import comet_ml\n",
    "import itertools\n",
    "from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer\n",
    "import dataclasses\n",
    "import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a5cbf732-a372-42eb-af42-61b8c1403aeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from easy_transformer import EasyTransformer, EasyTransformerConfig\n",
    "import easy_transformer\n",
    "from easy_transformer.experiments import (\n",
    "    ExperimentMetric,\n",
    "    AblationConfig,\n",
    "    EasyAblation,\n",
    "    EasyPatching,\n",
    "    PatchingConfig,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f376b45c-c276-45ce-855f-efc120c5ad91",
   "metadata": {},
   "source": [
    "### SOLU: Softmax Linear Units\n",
    "Here we'll study a transformer trained with the [softmax linear unit](https://transformer-circuits.pub/2022/solu/index.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c751652e-072c-4941-bdb3-3b88d0a7ed20",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c4a6c5c-2e2f-4350-8972-9a7c5f0cf895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: solu-8l-old\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moving model to device:  cuda\n",
      "Finished loading pretrained model solu-8l-old into EasyTransformer!\n",
      "Moving model to device:  cuda\n"
     ]
    }
   ],
   "source": [
    "model_name = \"solu-8l-old\"\n",
    "model = EasyTransformer.from_pretrained(model_name).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "4fad34cc-c527-4492-a46d-b565f8a4e6fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EasyTransformerConfig:\n",
       "{'act_fn': 'solu_ln',\n",
       " 'attention_dir': 'causal',\n",
       " 'attn_only': False,\n",
       " 'attn_types': None,\n",
       " 'checkpoint_index': None,\n",
       " 'checkpoint_label_type': None,\n",
       " 'checkpoint_value': None,\n",
       " 'd_head': 64,\n",
       " 'd_mlp': 4096,\n",
       " 'd_model': 1024,\n",
       " 'd_vocab': 50278,\n",
       " 'd_vocab_out': 50278,\n",
       " 'device': 'cuda',\n",
       " 'eps': 1e-05,\n",
       " 'final_rms': False,\n",
       " 'from_checkpoint': False,\n",
       " 'init_mode': 'gpt2',\n",
       " 'init_weights': False,\n",
       " 'initializer_range': 0.025,\n",
       " 'model_name': 'SoLU_8L_v21_old',\n",
       " 'n_ctx': 1024,\n",
       " 'n_heads': 16,\n",
       " 'n_layers': 8,\n",
       " 'n_params': 100663296,\n",
       " 'normalization_type': 'LNPre',\n",
       " 'original_architecture': 'neel-solu-old',\n",
       " 'parallel_attn_mlp': False,\n",
       " 'positional_embedding_type': 'standard',\n",
       " 'rotary_dim': None,\n",
       " 'scale_attn_by_inverse_layer_idx': False,\n",
       " 'seed': 42,\n",
       " 'tokenizer_name': 'EleutherAI/gpt-neox-20b',\n",
       " 'use_attn_result': False,\n",
       " 'use_attn_scale': True,\n",
       " 'use_local_attn': False,\n",
       " 'window_size': None}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.cfg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "657a6e09-1fef-45ca-897f-9f3705101402",
   "metadata": {},
   "source": [
    "### Induction heads\n",
    "We look for induction heads by feeding in a random sequence of tokens repeated twice and looking for heads that attend from a second copy of a token to the token just after the first copy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5b670aaa-0978-4b32-af06-93ba29453167",
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 100\n",
    "rand_tokens = torch.randint(1000, 10000, (4, seq_len))\n",
    "rand_tokens_repeat = einops.repeat(rand_tokens, \"batch pos -> batch (2 pos)\").to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "3ec19b07-063d-471e-99e1-0b7efef48d39",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mask out the heads with the highest induction score\n",
    "induction_scores_array = np.zeros((model.cfg.n_layers, model.cfg.n_heads))\n",
    "\n",
    "def calc_induction_score(attn_pattern, hook):\n",
    "    \"\"\"Pattern has shape [batch, index, query_pos, key_pos]\n",
    "    \"\"\"\n",
    "    induction_stripe = attn_pattern.diagonal(1 - seq_len, dim1=-2, dim2=-1)\n",
    "    induction_scores = einops.reduce(\n",
    "        induction_stripe, \"batch index pos -> index\", \"mean\"\n",
    "    )\n",
    "    # store these in a common array\n",
    "    induction_scores_array[hook.layer()] = induction_scores.detach().cpu().numpy()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a1e512c4-4a2e-40f6-b814-b2acca5bc14b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_attn_hooks(hook_name):\n",
    "    split_name = hook_name.split(\".\")\n",
    "    return split_name[-1] == \"hook_attn\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "85d99541-e15e-430c-93e1-dbd4dc989b30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "coloraxis": "coloraxis",
         "hovertemplate": "Head: %{x}<br>Layer: %{y}<br>color: %{z}<extra></extra>",
         "name": "0",
         "type": "heatmap",
         "xaxis": "x",
         "yaxis": "y",
         "z": [
          [
           0.007221605163067579,
           0.007005648221820593,
           0.0052811382338404655,
           0.007684369571506977,
           0.006177148316055536,
           0.00543549656867981,
           0.008061942644417286,
           0.007398218382149935,
           0.009174970909953117,
           0.007798024453222752,
           0.006897738203406334,
           0.007003214210271835,
           0.004043221473693848,
           0.00687289796769619,
           0.007462042849510908,
           0.007293550297617912
          ],
          [
           0.003388745244592428,
           0.0008520383853465319,
           0.004962692968547344,
           4.1972579925553255e-12,
           0.0019923218060284853,
           6.17564967396421e-21,
           6.461049224526505e-07,
           0.0012802411802113056,
           0.004572327248752117,
           0.0017938436940312386,
           1.9262912098749752e-10,
           6.782847418722442e-19,
           1.6778492004144056e-37,
           4.964051495163699e-14,
           0.002202582312747836,
           0.004415472503751516
          ],
          [
           1.0545743965584276e-11,
           0.0013257188256829977,
           0.0028420784510672092,
           2.6150491494458805e-11,
           2.7385923390133016e-11,
           8.579650721089196e-16,
           0.00035306380596011877,
           0.002317853271961212,
           1.4175520846038125e-05,
           0.002394661772996187,
           0.0028601076919585466,
           2.2160580556374043e-05,
           0.0021006136666983366,
           0.006980125792324543,
           1.828982278129221e-12,
           0.0001239109260495752
          ],
          [
           0.003600881202146411,
           0.0216167401522398,
           0.002407241612672806,
           9.495644007984083e-06,
           0.0035184475127607584,
           0.0014035761123523116,
           0.004660090897232294,
           5.77428863834939e-07,
           1.2221848825859913e-23,
           9.531353839520307e-07,
           6.683242759208952e-07,
           0.00018383732822258025,
           0.0006168605759739876,
           0.001556359464302659,
           0.0013503115624189377,
           0.0007486341055482626
          ],
          [
           0.005284776445478201,
           0.006088730879127979,
           0.6402723789215088,
           0.001268303021788597,
           2.7627231247606687e-05,
           1.0487074177945033e-05,
           1.0515276471778634e-06,
           0.0016490078996866941,
           0.007343673147261143,
           0.0012581143528223038,
           0.0013831725809723139,
           0.004518447443842888,
           0.00033377474755980074,
           3.1605391995981336e-05,
           0.0002842126996256411,
           0.6168648600578308
          ],
          [
           0.0025591261219233274,
           0.03251533582806587,
           0.0016285560559481382,
           0.1571081280708313,
           0.32328101992607117,
           0.20798426866531372,
           0.11704714596271515,
           0.5587158203125,
           0.24138742685317993,
           0.0008683035266585648,
           0.00014592513616662472,
           0.0019469247199594975,
           0.0020449066068977118,
           0.05343598872423172,
           4.6549466787837446e-06,
           0.08330928534269333
          ],
          [
           0.09291791170835495,
           0.003473499557003379,
           0.016125187277793884,
           0.07268991321325302,
           0.02634909190237522,
           0.0014974861405789852,
           0.005410476587712765,
           0.005121956113725901,
           0.006888541392982006,
           4.132581307203509e-05,
           0.019403278827667236,
           0.0061025796458125114,
           0.05686185136437416,
           0.024155929684638977,
           0.021906157955527306,
           1.8004451703745872e-05
          ],
          [
           0.007500350009649992,
           0.011111307889223099,
           0.009122894145548344,
           0.059650395065546036,
           0.012209426611661911,
           0.026963815093040466,
           0.012048366479575634,
           0.025827638804912567,
           0.006679396145045757,
           0.03958643972873688,
           0.04641525447368622,
           0.030032876878976822,
           0.01838570646941662,
           0.00953657552599907,
           0.02023862488567829,
           0.013849987648427486
          ]
         ]
        }
       ],
       "layout": {
        "autosize": true,
        "coloraxis": {
         "colorscale": [
          [
           0,
           "rgb(247,251,255)"
          ],
          [
           0.125,
           "rgb(222,235,247)"
          ],
          [
           0.25,
           "rgb(198,219,239)"
          ],
          [
           0.375,
           "rgb(158,202,225)"
          ],
          [
           0.5,
           "rgb(107,174,214)"
          ],
          [
           0.625,
           "rgb(66,146,198)"
          ],
          [
           0.75,
           "rgb(33,113,181)"
          ],
          [
           0.875,
           "rgb(8,81,156)"
          ],
          [
           1,
           "rgb(8,48,107)"
          ]
         ]
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "xaxis": {
         "anchor": "y",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0,
          1
         ],
         "range": [
          -0.5,
          15.5
         ],
         "scaleanchor": "y",
         "title": {
          "text": "Head"
         }
        },
        "yaxis": {
         "anchor": "x",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0.09090909090909094,
          0.9090909090909091
         ],
         "range": [
          7.5,
          -0.5
         ],
         "title": {
          "text": "Layer"
         }
        }
       }
      },
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg4AAAFoCAYAAAA//nPjAAAAAXNSR0IArs4c6QAAIABJREFUeF7tnXm8TdUewH/XTErJa57T4HkiJU1SSqk00KCkWRQhmTJlTFTGpCgVUUmRiKIJlQxp0IRGJSEhurcy3PfZ+957pnuvs+7e63f2Ofd8zz+9d631W2t/12/t/T1rr71PRnZ2drbwgQAEIAABCEAAAgYEMhAHA0oUgQAEIAABCEDAJYA4kAgQgAAEIAABCBgTQByMUVEQAhCAAAQgAAHEgRyAAAQgAAEIQMCYAOJgjIqCEIAABCAAAQggDuQABCAAAQhAAALGBBAHY1QUhAAEIAABCEAAcSAHIAABCEAAAhAwJoA4GKOiIAQgAAEIQAACiAM5AAEIQAACEICAMQHEwRgVBSEAAQhAAAIQQBzIAQhAAAIQgAAEjAkgDsaoKAgBCEAAAhCAAOJADkAAAhCAAAQgYEwAcTBGRUEIQAACEIAABBAHcgACEIAABCAAAWMCiIMxKgpCAAIQgAAEIIA4kAMQgAAEIAABCBgTQByMUVEQAhCAAAQgAAHEgRyAAAQgAAEIQMCYAOJgjIqCEIAABCAAAQggDuQABCAAAQhAAALGBBAHY1QUhAAEIAABCEAAcSAHIAABCEAAAhAwJoA4GKOiIAQgAAEIQAACiAM5AAEIQAACEICAMQHEwRgVBSEAAQhAAAIQQBzIAQhAAAIQgAAEjAkgDsaoKAgBCEAAAhCAAOJADkAAAhCAAAQgYEwAcTBGRUEIQAACEIAABBAHcgACEIAABCAAAWMCiIMxKgpCAAIQgAAEIIA4kAMQgAAEIAABCBgTQByMUVEQAhCAAAQgAAHEgRyAAAQgAAEIQMCYAOJgjIqCEIAABCAAAQggDuQABCAAAQhAAALGBBAHY1QUhAAEIAABCEAAcSAHIAABCEAAAhAwJoA4GKOiIAQgAAEIQAACiAM5AAEIQAACEICAMQHEwRgVBSEAAQhAAAIQQBzIAQhAAAIQgAAEjAkgDsaoKAgBCEAAAhCAAOJADkAAAhCAAAQgYEwAcTBGRUEIQAACEIAABBAHcgACEIAABCAAAWMCiIMxKgpCAAIQgAAEIIA4kAMQgAAEIAABCBgTQByMUVEQAhCAAAQgAAHEgRyAAAQgAAEIQMCYAOJgjIqCEIAABCAAAQggDuQABCAAAQhAAALGBBAHY1QUhAAEIAABCEAAcSAHIAABCEAAAhAwJoA4GKOiIAQgAAEIQAACiAM5AAEIQAACEICAMQHEwRgVBSEAAQhAAAIQQBzIAQhAAAIQgAAEjAkgDsaoKAgBCEAAAhCAAOJADkAAAhCAAAQgYEwAcTBGRUEIQAACEIAABBAHcgACEIAABCAAAWMCiIMxKgpCAAIQgAAEIIA4kAMQgAAEIAABCBgTQByMUVEQAhCAAAQgAAHEgRyAAAQgAAEIQMCYAOJgjIqCEIAABCAAAQggDuQABCAAAQhAoBAC5U++OxA2WZ+MDqRdk0YRBxNKlIEABCAAgbQkUL52+0COO2v5qEDaNWkUcTChRBkIQAACEEhLAuVPvSeQ485aNiKQdk0aRRxMKFEGAhCAAATSkkD5OvcGctxZS4cF0q5Jo4iDCSXKQAACEIBAWhIof1rnQI47a8kjgbRr0ijiYEKJMhCAAAQgkJYEyp/eNZDjzvrooUDaNWkUcTChRBkIQAACEEhLAuXPuC+Q485aNDiQdk0aRRxMKFEGAhCAAATSkkD5M3sEctxZHw4KpF2TRhEHE0qUgQAEIACBtCRQ/uxegRx31vsDo9pduPhzGTRqkmzctEVqVq8qQ3q2liqVK+Xr27r1m+S+QePky5U/yOGHHCADu7WU6iccZfUYEAerOAkGAQhAAALFiUD5evcHcjhZC/uH2v1ze6Y0ur6LPNLnLqlTq5qMGDdV1m3YJMP6ts3Xt5vaD5LzzjxZWlx9obzy+nz55IvVrmTY/CAOPmlm7sj2GaGYVd8tIhkKx5ThBE0d1k5PM7J1euziLaGDI1tr/JTSws00jXxTSOHIkBmKnc5OoXmijFkqlPafHOXP6avdzQLjZy0It/vGu0tk2uwFMu7hnCc8tm3PlPpNO8hHs8ZImTKlQ/V//nWDOOLw9kvDpUQJ/8de2IEjDj5TAnGIBpidrXAed67CzoUyxT6qF2Glc4IzflofjS67gpaCUlnC6bMSa8QhnMFWxOHc8Dd/rblRUNys98IrHWOfmymbNm+VHu1bhIo64jBxVA858rADQ397e+FymTRtrnuL4sOlX8hhhxwgvTveJMceeYjVriMOPnEiDmGA7klc6xur0jdsn8NfeHXnuuCwUPlkSIaznKHw0ROHnBUjFXnIUJBVBbahkE5/szVI5HDYrbXUpclEKbYVcTgveq+BUlfzhc16N7y3YsSTL8vOXbuk853NQuUaXtdZRg1oJ9WOOzL0t+lzFkr/4RPliSH3ymm1TpRJr8xzVyqmP233GBAHn1mAOEQAdM9aPoEWVl3nPKvUWSdshmTv1lh+yemy+yVb4aMmDq5V2r/A56w4KIBQDql2q8IRVrVBVIaiEN6KODQI5umGrHfCT3OMmzRTnE2PfTrdEqJ0RuM28uITffKtOIyZ8Kq88lTOKsnu3dlS+8KW8u4rI2S/SntbI4w4+ESJOMQA1PkibP+K43Pc41Z3Vxx09mW410mli6XmNUely84+khJqOOIOs6cCiisOTl7sztZZ2fF0rAFXsiIOFwTzPoWst8Lvj5g7f5lMnjZPJozs7hJ1nqxo1Lyru8ehdOlSIcorv/tZ2nYfLm+9lPO66l27dkvtC++Q92c8KntXrGBtNBAHnygRhwiASisO7rfKFLtV4fqT1m0brW/ZirKj5joO6JI6G0V9nhoKr64pDm7aIQ558K2IQ8Ng3uCYNS/8xsq/Mv+WRs27yJBeraVOzRNl8OjnZXtmlvu0xPdr1snadRulXt2T3MNuclsvufHqC6XJxfXkuZfnysx5i2TqOLsbPBEHg7PDmrUbpMeDT8rXq3+SQw+qIv273ia1qld1ayIOYYCaexycW8Iq31gNxt9rEbVv784dEK3NokpPguQxtD2G3KrIn51sjgwzsSIOFwbzmxFZc6N/I+Oj5V9Jv6ETZOOmzXKqIw89Wsm+lSrKlBnviLMiMX5Yjmis+v4X93r1y7qN7qZI53rF5kivZ3Ef9W5s94CcVaeG3N78Upm/6FP3JRxvvvCIlC5VEnGI5Jq7IdD2xcFtQiWoj6QwqKomDopPEmj1WXX4VIMbDLSHImp7HJzFFzZHhkbEijg0CuZXKrPeCOZXOU3SmRWHOJQ2bf7TXSJaNGuMlCrprImKXH1HH+nW9nqpU+tExCGWn9IeB1YcokHr3LrJUNtYl3Nt19nzkWpSmfN0sZ7tcKvC8orDxSNMrqXWy2TNucd6TFsBEYc4JJevWC39h02QV58JP87Suf/jUrd2Nbmm8bmIQ8yKg95TFUoXHVszKV8cvYuwcwHWeRxT70mQvNGzfrlMxXd8uLfd9N7jgDhYFodLRqmdJfYUOGt2+0DaNWkUcYhD6cNlX8jIJ1+RKWP7hEr2HPyUHH/s4XLzNRchDglYcXAXMVJsj4PbZ6XVFxeH9StwDmTVR0hNzkhFLZOK4uCSVhnAnPc4aCZeUccn4PJWblU0Hh3IUWTNujuQdk0aRRziUHLe891ryHh5/bnwIznte49yd7Cy4lAAPK2LZYq9HdD9Qqn1TgvFp0z03nbpXCqVkkPnGmxy/vRcRkscnA6x4mB5xeGyMZ7H2U/FrJlt/FRXrYs4xMG7ees2ueDaTvLBa6OlXNkybulLb7xPBnS9TWrXOJ4VhwSsOLhNpODFQWujoXP9zXDfQ2//QqzW59C3bMt9TsEVB/Y4qF7TooJbWXG4/InEdTiipazX7gykXZNGEQcDSrd3ekhOOekEueOGxvLme0tk5FOvyJzJQ9zNkjyOGQPQ8nUhJ7r74LvBSCVXEa2LsOtQSu+10FtxUHS/FJNKbXHgcUzLKw5XjgvkxJL1aqtA2jVpFHEwoOS86rPbA2Ply5U/uj8e8sB94d83RxwSIQ56F0qD4fdWJAVvVajvy/BGMn6tFBOHHBXW63SOYqeeaMcf6KKXsLLi0OSpojdsoUbW9JYWouiEQBx8ckUcYgBqvC0xFd8O6Jy61c7dGZJRQudNTdnujy/Z77jWZTIlXwCl+OZINkdGn4+siEPT8T6vEt6qZ0273VvFBNRCHHxCRhwiAeY8gmj9IpGiv0egtTnSvbQrPWWiKQ65v3Plc8YVsMKltN/DbkejoznvcbCvZzl39fiRK7u3Kipc9bRmKhQaO/OV2wJp16RRxMGE0h7KIA4FnMh9Mi2outaFUqGroZB6Kw5aj6em5uOYKfcjV8o3KlSERHOiKMa2seKw1zXPKPaw8NB/Tb01kHZNGkUcTCghDmaUlH7kyv1qloLfKrXEIefXMXVuKWj12b2r7wS3vhyVek/cqG+O5JXTofOVDXGoeO2zZuc/y6W2vxT+CW3LoX2HQxx8ImTFITErDioXHJ9jv8fqzpLxLr2LmsoLoHLFT/Mbq21vSMU9DrrikK35o6yaM0Yltg1x2LvZBJW+xQu6bcrN8YoE9u+Ig0/0iEMMQI3NkYrfsH0Of+HVNZ+qUPx1TNUVB4W7+jnioLP6opkbGe4mVJ0PP3IV5mpDHPa5fqLOQMWJ+ucLNwXSrkmjiIMJpT2UQRz0Vxzci4PSewt8Dv8eq2u9E0HtPQ6u7GhdhLXeHJkhovSEiWZuqD6Oya2K0NDZEIdKzZ/TTIVCY299/sZA2jVpFHEwobQncdgZ/Vhc7M7x8P+PPiHn/T1vWTj8/cPZbZ3z1z1/kSrgBB/xp8j4OSep8GthCvuuk7fsG30rOq90+OnwnHvsEVByEbitxNzGzteP7Gz3iQA3RGyDTpzQ0wLhgwktR0d1JeelUIXeNs999MA56hDHQq6J+Y47tlxuu4W1FVU/cgxcZDl/iAxZEP/IEnllI//rRHJXA3LhOz9yVdAY5+yqz8udaIbh3kT/TmUk1shrTvTI59aJTPDcdty+R8LJaz/0YGeYgdPpnDHOyfP8qMOw82+IjSmdFytCKgt9cmMPYxo918L9ijqs2LmVe8h5syDyeKLGwB24/D8MmjcnQz/CUsD9oXyjl+HsE8nll/vfyEWcvPJ5v1URNa6x/Y9MhtBUjh6TqDyNGOIoxgUkSeSqVfg4C2gwJm0KP81GD15B59jwKSmnbF4ZG+Kw7w2TfF4lvFXfMrmFt4oJqIU4+IT8946Yp94LOkEVctM49mKSTybyzqER552ci21ep6OutDl/zP33Ak+guX/cHepP6MyfUy3UoZxTelT/Yq9m0d6Q26VsKZHv5ns4VjTqPXW0wOBRC92h00PU1aXQy0bE2Tvnf+Z1M/riG0YYPhGF1S5yGHMepov4hUN3TMLjkfOvBZ3icgYvHDWiWr7uR/yhgKHOHygiL2I3heQ26OIqLG1i8jTfKx0iEqLQU3ls/kfmbuTFNuqqFh7vgv4cFqboYYwcx1CEKLBRUyIqqdz8dvFGgMkd9DD1mAGJStlC8jp6yNw2Yy/24bzIU6foKV3gfp58iRrZt9y+xJ5nItMzcvyj8nLP8zCEKCbV3XAFTLfIeenMsVC58ITKf8bN/bIRdfcmt79uvHznnuiG88rkSG/OmSuneraULx2lP57O9vu1mOypnt9Kmyfd4DeEWn3EwSfarB0+AxRSPZTu/vM+Xwvuby/FnmQsHYb78IP1T/gEqxA670u89dCpF9BZwVEZQJ2nKfT2nqbe0CWox0qnjRy5UjiG8qX9B6180/P+g3iI8MfE5h5qJaYK4uCTs5Y45H3rsH8az5Zd2Xp3WDXEIe/bg8rVJ/fLi33OPhMroOpqmyPzVmQUHo9h7PSTpbAVBmsta1iDiNgQh/1vfsHaYRYl0KYJ1xeleELLIg4+caeaODjzU/PiUKKg9UufjBUXSEI94+ITs6xsYcwiQ4SWnC3HdcLlvyWk0AghXQJK13e1wDbEocotLwYy+r8/e10g7Zo0ijiYUNpDGS1xUL1VEXXT2CeAmOrOfU3rF2H3nnwBN1Qtdd16fy31K+FhlJaLcy7uudcG27DzNuPajptw+MndYOQWC42eJvOtiv/cOkXjkOPG3PhMs7hlgiqAOPgkryUO7slW6WQY3hzp8+DzVS9oc6S9NrS+7ShhtnfgCYqUtxFOhXOuUNqOHblvLkGY0rYZ22OXB9IdQ6UvMzZWHA64/aVAxnzD+GsDadekUcTBhFIAKw454qDzLXuX0pt/nT5r7HFw4mqdtCJPXj5ToXhUV1x1cAEpWZpS2OIxppaOQnUOKgW3IQ4HtpxqiWDRwqx/6pqiVUhgacTBJ+yUXHFQvBJr3KpQOqdEjTwXnhwcWt/8QrAVQCuE9HlWKJ7VlRYFop8ht4zOhjgcdMfLlntlFu63J682KxhAKcTBJ3RVcVD6hqZ3q0JnxUFTHBQ3+/vMrGCq64mD3uMrbI5MTK7ozcNscV7BrRHfhjgc3PqVxACOaWXd2KsCadekUcTBhNIeyqiKg8pXqWzZrfisvvP2X63laI0Ti5Kb+cyqYKqr7nFQBK0yTYIZgqRuVXP+aQmrDXE45M5pgYzLr080DaRdk0YRBxNKAYiDezJUOiOm3Aug8l6v63OsCq6e821Hi7VKlxWDaj6qq9VtpWmi1d2UjasrDjpYbIjDoXdN1+lcnKhrH28SSLsmjSIOJpQCEAe3SaXrmd6timwpodBprRNWzrDmvVzbZyIUk+pq4qAlwjyOmbDMU52HSsFtiMNhbV5NGOPIhn4Zc2Ug7Zo0ijiYUApIHPQex9S7n6CxOTIPv9K5hRcIReR36A2BPudFvuqKm0mUHj6yTSDl42nNv1x/V+FjQxwOv3uGSt/iBf159BXxigT274iDT/R6exycX3W0vwjrTH5nxcF+5JwVEpU9Dsq3KnJ+q5GPmjTkoVWCrBSWhCiAgIY85DmlRmwb4nBEu9cCyYU1j14eSLsmjSIOJpQCWXHIXUJXOCvq3arIeWmVQpdVdlyHrmd6CzA+syvx1VNNHjRyLfHUU6dFjYt76FdlFTDYEIcj289U6Fn8kD+Nuix+oYBKIA4+weuuODids39q1BSHEkrrxjonrPDg26fsM7ECqM5TFQFAT5EmQ0KpNVGUJrgNcTjqnlmBjNKPIxoH0q5Jo4iDCaViteKQLbsdGVGaqCpvjlS8VaF1HvSZVoFUz7k4KC+/KABXCBkI/2RuNC8rlE4baucjG+JwdMfXAxmaH4ZfGki7Jo0iDiaUAhEHpd+qyBZxH8dU+pTQWSRRvZ4pLZIoEdYLG7ooaF0dFK/wiqH1gKdSZEV5d08ZSq86tyEOx9w7O5CR+n7YJYG0a9Io4mBCKShxULoI692q0PuRK61rWd6vgHPhyXnzrxZn94eMlCAjfj5PYobVtXIj2cXh2E5zDAnZLfbd0IvtBrQYDXHwCVNvj0MKrjhoPVWh+iNXOYuwStc0n9mV+Opa73FQE4fcNXTGTz9XVO9iKVmJjRWH47oEIw6rH0Yc9LM6oBa0xCHvRKtxQlRbcXDEQWkclM4rCEPseGmuOmhsrUEclGZcTFjlWxU6S13ZUr60/zPo8V3fSAzjmFZWPdQokHZNGmXFwYRS3FsV4W+t2c4McJ5JjNhNFPq2VejVL3rrkfP9N7RFzX/e5+t96JXTubGdbxLu/3Q7mtfJ6IbDf43cPBfxv51DzhYpqXR7JXJXt9tN96jCZ7M85OGDzf233MdDowjHDJHPFEh89Yg8inxbduy2xth0Cy/pF5CIuYHyOIcZ56ZFHu+Y3A4negEYQvmUs6LjxlbI58iQBW7tDKdCKGXce+pObsRUiMpzrVeRh9p0zhbh3rv/y/23nL/n/Uvun1x2ef873LXoslEnnogxzYc9cppH8InCkft/cv6TLdnZGaFTW9RoF5R4EY9lh+Zu7vhHpEVUd92YEYkX+vKUJ7ORSZkv3XKIubfDIpZGbKw4nNDtzcTPcRFZOeSiQNo1aRRxMKG0hzJ/7/QZIIDqWsvR7rxXuDAEgCjpmyzUQX32XHP4tPqs8201B6RWPquxCP00us9EKKB6CS0Y9rsailiulP/g1boHIw5fP4g4+B+9JI2AOEQPTAqeW5I0s/bcLa0LD+KQmHzWGj930ULpm0G6isN/e8wN5Bzx1aALA2nXpFFWHEwoseJgTAlxMEblq6DWhQdxQBwKS8x0FYfqPef5mqteK3/5QEOvVdXrIQ4+EbPikJgTrc9hKnbVEYeIIdWCwa2KqHmTruJQo3cw4rBiAOJQ7E7ceQeEOCAOQSS31rWSFYfE5LPW+HGrInr8bOxxOOn+t4KY4vJ5/wsCadekUVYcTChxq8KYErcqjFH5Kqh14UEcEAduVUQTqNnnbV9z1Wvlz/qd77Wqej3EwQDxdz+ulb5DJ8jK79ZIlcqVpPNd10mDs052a7LikJgTrcEwpVURxIFbFYUlPJsjw2RsrDic3C8YcfikD+KQ0if1K27tKVdfWl9uaNpQPlj6hdzbd7QsmP6olC9XBnGIGVlWHBKT6ogD4oA4xJ9rNsShdv934jekUGL5/Q0UotoJyYpDHI47d+2S6XMWSpOL60mpku7rjaTupXfJ1HH95IhDD0AcEAc7M7GIURAHxAFxiD9pbIjDKQPejd+QQomPe5+nENVOSMShiBxXfP29dLj/UXlryjApUSIDcUAciphBdoojDogD4hB/LtkQhzoPBCMOS3siDvFHOAVK/LJuo7Tq8oj0vucmOePU6m6P2eMQPXDcqkhMIiMOiAPiEH+u2RCH0wa9F78hhRJLepyrENVOSFYcDDmu/O5n6dD7Ubnv7uZy7pm1QrUQB8TBMIWsFkMcEAfEIf6UsiEOdR+cH78hhRKLu9dXiGonJOJgwPHnXzfIHZ0fkUHd75DaNY6LqoE4IA4GKWS9COKAOCAO8aeVDXE4Y0gw4rCoG+IQf4STuMQt9wyWZpefJxc3qJuvl4gD4hBE6iIOiAPiEH/m2RCHMx9aEL8hhRIfdj1HIaqdkKw4xOHo7Gu46PouUrp09M+sPXL/XXJBvVPY4xDDjz0OdiZmvCiIA+KAOMSbJSI2xOGshxfGb0ihxAdd6ilEtRMScfDJkRUHVhx8ppCn6ogD4oA4xJ86NsSh3tBgxGFhp2hxWLj4cxk0apJs3LRFalavKkN6tnZfSFjYZ8kn38itHQfLzIkPyjFHHBwfVhFKIA5FgFVQUcQBcfCZQp6qIw6IA+IQf+rYEIdzhr0fvyGFEgvuPTsU9c/tmdLo+i7ySJ+7pE6tajJi3FRZt2GTDOvbtsCW//13h1zfZoArGc+O7I44KIyPr5CIA+LgK4E8VkYcEAfEIf7ksSEO9Yd/EL8hhRLzO54VivrGu0tk2uwFMu7hzu7ftm3PlPpNO8hHs8ZImTKl87X+2DPTJTtbZO6CZTKi/92Ig8L4+AqJOCAOvhLIY2XEAXFAHOJPHhvicN7IYMTh3Q5hcRj73EzZtHmr9GjfInTQjjhMHNVDjjzswCgQP/78m9xz/2h5aWwfubpVX8QhfpokvgTigDgkPutEEAfEAXGIP/NsiEODUR/Gb0ihxDvtzwxFHfHky+L8/EHnO5uF/tbwus4yakA7qXbckVGt39ZxiLS+6XKpe3I1ufyWnoiDwtj4Dok4IA6+k8hDAMQBcUAc4k8cG+Jw/qOL4jekUOLtdmeEoo6bNFPWrd8kfTrdEvrbGY3byItP9IlacXj1jfdl2WcrZWC3291yiIPCwNgIiTggDjbyqKgxEAfEAXGIP2tsiEPD0cGIw7y7w+Iwd/4ymTxtnkwY2d09aGfTY6PmXd09DpGvCmjXa5QsX7FKSpYo4Zbb8ud22btiBXmgW8uoNx7HJ7fnEjxV4ZMg4oA4+EwhT9URB8QBcYg/dWyIw0WPfRS/IYUSb7Y9PRT1r8y/pVHzLjKkV2upU/NEGTz6edmemeU+kvn9mnWydt1GqVf3pHy9YMVBYWBshMzaYSNKwTH+3bVbJXiZUjk2qvHJ0Agq4u4Q1vpovbQqFfusxZi4EAiCgA1xaDRmcRBdlzfaRL+p+KPlX0m/oRNk46bNcqojDz1ayb6VKsqUGe+IsyIxflhXxCGQkfLQKOIQDQ1xCPNAHDxMKKpAwCIBG+JwyRPBiMPsO/P/xIFFNL5CcavCFz4RxAFxKCyFEAefk4vqEPBJwIY4NB67xGcvvFWf1fo0bxUTUAtx8AkZcUAcEAefk4jqEFAiYEMcLhu3VKl3ew47s1WdQNo1aTSpxWH37mxZ9f3PcuyRh+T7kSmTg0tEGcQBcUAcEjHTaAMCRSdgQxyueDIYcZhxB+JQ9BF3N8RlS+2LWsmcyUPkoP9U9hRDuxLigDggDtqzjPgQ8EbAhjhc+dQyb437rPVqy1N9RtCrntQrDs5hj39htjg/bd3qhsZy8IH765HwGBlxQBwQB4+Th2oQUCZgQxyajv9YuZcFh592+ymBtGvSaNKLg/OSC+clFs6PepQqWVJKly4ZdVzL3hhncpxqZRAHxAFxUJteBIaALwI2xOHqp4MRh5dvQxw8D/47H3wipUs5slDwg3716tbwHNtGRcQBcUAcbMwkYkDAPgEb4nDNM8vtd8wg4tRbaxuUCqZI0q845GFxfuBj/cbNcuhBVYIhVUiriAPigDgk1ZSkMxAIEbAhDs0mfBII0Sk3nxxIuyaNJr04OLcoBo2aLK+/vUh27dotX773rPyxZZt0GfC4PNTrTtl/v31MjlOtDOKAOCAOatOLwBDwRcCGOFw/MRhxeOEmxMHz4PcaMt79QY82t1wpzdsMcMUhM+sf6T98gvz997/uT4YG+UEcEAfEIcgZSNsQKJyADXFo/tyngSB+/sZagbRr0mjSrzjUb9pBXn1moOxXaW+pfu4trjg4nz+3Z8pF13WWRbPGmBynWhmhx+M+AAAgAElEQVTEAXFAHNSmF4Eh4IuADXFoMSkYcZjUAnHwPPinXNRK3p8xWsqXKxMlDlu2bpcLmt0rPFVRdLT8yFWM7Cj9wAavnC56blIDAjYJ2BCHmyZ/ZrNLxrEm3lDTuGyiCyb9ikPrrkPdN0d2bHWN1GrY0l1xWLd+kwwaNUl27totjw/umGhmUe2x4sCKAysOgU5BGodAoQRsiMPNz38eCOEJzfP/THYgHSmg0aQXB+flT/f2fUxWffez7Ni5SyruVV62/5UlNaodI8P6tJFDAn7KAnFAHBCHZDmd0Q8IRBOwIQ63vhCMODxzPeLgO59XfPODrFm7XkpkZMgRhx4o1U84yndMGwEQB8QBcbAxk4gBAfsEbIjDbS+usN8xg4hPXxfsO4r21MWkX3G47s5+ctmFZ8rFDU6XyvvubYA7sUUQB8QBcUjsnKM1CJgSsCEOLad8Ydqc1XJPNfuf1Xg2gyW9OIyZMEPmzV8q3/30q5xVp4Zc1vBMaXD2yVKubBmbHDzHQhwQB8TB8/ShIgRUCdgQh1ZTgxGHcdcgDr6Tw7lNMXf+Mpk7f6n8+PNvckG9U+TyC8+SurWrSUaG0rZ4g14jDogD4mAwUSgCgQAI2BCH1i9/GUDPRcZeXT2Qdk0aTfoVh9iDcDZITp+9QIaOfcndJOm8grrlDY3lmsb1AxEIxAFxQBxMTjWUgUDiCdgQh7te+SrxHReRx6/6byDtmjSaEuKQnZ0tH3++SmbO+1DmvrdUypQp7e57uOKis+TnXzfKkNHPy7ln1pLu7W4wOWarZRAHxAFxsDqlCAYBawRsiEPbacGIw2NNEQfPiTDiyZdl1luL5Pc/tkqDs2rLlY3Ocvc6lCxZIhRz9Q+/yHV39peP30z8T2xn7sj2fGxBVdz/tHZqTW9eOlotNoHDBLSyzpF0rY/zRFSqfbRwpCAK1aHbr34PlfhZHwzyHbfd9K99x/AS4NEm1bxUS0idpF9xuO6u/nJlo7PlkvNPl30qVoiCsmPHTildupT8++8OeXzia9Kh5VUJgRbZCOIQjRxxSEwKal3eEYfo8UMcEpPPySwOHV4NRhxGXok4WM8+Z3/DpTfeJ/OnjbQeuygBEQfEoSj5Yqss4mCL5J7jIA6J4ZzM4tBxxjeJgRDTyvArTgykXZNGk37F4edfN8gDIyfJV6t+lH/+3RE6pqy//3FfRT396YEmx6lWBnFAHNSSaw+BEYfEUEccEsM5mcWh02vBiMPQyxEHz9nXsvPDslf5cnLpBWfIgOETpF+X2+TLlT/IB0u/cH+nwvnVzCA/iAPiEET+IQ6JoY44JIZzMotDl1nBiMPDjREHz9lX5+LW8s7U4bJ3xQpyQbNO8taUoW6sN99bKvMXfSqDut/hObaNiogD4mAjj4oaA3EoKjFv5REHb9yKWiuZxaHb6yuLejhWyg+59AQrcTSCJP2titMbt5E5k4e4KwsXXtdZZk180H0c09nEdcZlbeWjWWM0uBQY0/kp70tu7CYdbr9Kml3RwC2DOCAOCUvAiIYQh8RQRxwSwzmZxaH77GDE4cFLEAfP2deh96OybXumjBrYXroOfEIOOmB/uaHJ+fLxitXy2DPTE7o5sseDT8qST7+RO5pfijgUMqI8VeE51YtUEXEoEi7PhREHz+iKVDGZxaHnnFVFOhZbhR+4+HhboazHSfoVhz+2bHNf8NS7402y9rffpW334bJuwx9Stkxpuf/em91HNRPxWfLJNzJmwqtS9ahD5bijD0UcEIdEpF2hbSAOicGPOCSGczKLQ+83ghGHAY0QB2vZ59yi+G3jZtmvUsWE/dCV876Ia1v3laF928rz095CHPYwmqw4WEv1PQZCHBLEWQk0L4CKHr9kFoc+b65OTLLFtNLvouMCadek0aRfcSjsIDb8vkWcJy5ee/YBk+P0VWbMs6+6eyra3tpEBo54DnFAHHzlk43KStczN8+1Prw5MkwWcUgdceg3Nxhx6HMh4mD9XOTctnA2S3753rPWY0cGdH6Js1O/MfLCmN7upkzEYc+4WXFQTcdQcK3LO+IQPX5aHoU4pI44DJj3bWImdUwrvRtWDaRdk0ZTdsUhUeLw7EtvyNiJr7mvtnY+f2X+7f5ORvMmF8g9d1zNUxUxWYY4mEw7/2UQB/8MTSIgDiaU/JdJ5lsVD7wdjDj0PB9x8J9ZMRESJQ6xHWfFgRUH68nsISDi4AGahyqIgwdoHqokszg8+E4w4tC9AeLgIZX2XAVx8I6UX8f0zi5ZaiIOiRkJxCExnJNZHIa8+11iIMS00u28YwNp16TRpL1VcWqjVnvsvzOh//7nX/U9DvEg8gKoaELcqoiXMXb+HXGwwzFeFMQhHiE7/57M4vDwe8GIQ5dzEYciZ9fstxcb1bnk/LpG5bQKIQ6Ig1Zu7Sku4pAY6ohDYjgnszgMnf99YiDEtNKp/jGBtGvSaNKuOJh0PhnKIA6IQxB5iDgkhjrikBjOySwOwxcEIw4dz0EcEpN9AbSCOCAOAaSdIA6JoY44JIZzMovDyIU/JAZCTCsd6h0dSLsmjbLiYEJpD2UQB8TBZwp5qo44eMJW5EqIQ5GReaqQzOLw6PvBiEO7sxEHT8mUCpUQB8QhiDxFHBJDHXFIDOdkFofHPvgxMRBiWml71lGBtGvSKCsOJpRYcTCmxFMVxqh8FUQcfOEzrow4GKPyVTCZxeHxRcGIw11nIA6+kiqZK7PiwIpDEPmJOCSGOuKQGM7JLA5jPwpGHFqfjjgkJvsCaCVrh16jWf/uUglevmxJlbhO0M9+2qISe+Jnv6rEdYI2rFpZJfZx+1dUiesErdN+ikrsx+5rqBLXCdr85CPUYmsF1hI0rf46cbf8pXNS2m+v0mrd/nXz3yqxj/lPOd9xn1z8k+8YXgLcUfdIL9USUodbFT4xIw7RABGHMA/EITo3EAefJxvD6ohDGJQNcRi/JBhxuP00xMEw5VOvGOKAOBSWtYgD4hDEGQ1xsCsOzyxdE8Qwyq11kneFjhUHnymBOCAOiIPZJGLFwYyT31KIg11xmLAsGHG4+VTEwe9cSNr6iAPigDiYTU/EwYyT31KIg11xmLjsZ79D4qn+Tace7qleIiqx4uCTMuKAOCAOZpMIcTDj5LcU4mBXHCZ9HIw4tDgFcfA7F5K2PuKAOCAOZtMTcTDj5LcU4mBXHJ5f/ovfIfFUv3ntwzzVS0QlVhx8UkYcEAfEwWwSIQ5mnPyWQhzsisMLy9f6HRJP9a+vfaineomohDj4pIw4IA6Ig9kkQhzMOPkthTjYFYcpnwQjDs1ORhz8zoWkrY84IA6Ig9n0RBzMOPkthTjYFYeXPtV7+dyexvraWof4TQW1+qw4+ESLOCAOiIPZJEIczDj5LYU42BWHlwMSh6tjxGHh4s9l0KhJsnHTFqlZvaoM6dlaqlSulC9d3nl/uQwd+5Jb7oRjD5d+XW6TY4442G9aRdVHHHziRBwQB8TBbBIhDmac/JZCHOyKw7TP1vkdEk/1m9YMX+z/3J4pja7vIo/0uUvq1KomI8ZNlXUbNsmwvm2jYq/fuFkuv6WHjH2ok5xU7Vh59Olp8umXq+WZ4fd56kNhlRAHnzgRB8QBcTCbRIiDGSe/pRAHu+Iw/fNgxKHJSWFxeOPdJTJt9gIZ93Bn9+C2bc+U+k07yEezxkiZMuHfEHHE4fOvv5OG55zqlvt69U/StscIeWfqcL9pxYqDTYKIA+KAOJjNKMTBjJPfUoiDXXGY8flvfofEU/0rTjooVG/sczNl0+at0qN9i9DfHHGYOKqHHHnYgYXGH//CbFn57Rp5qPednvrAioNVbOFgiAPigDiYTS7EwYyT31KIg11xmLlivd8h8VT/shphIRjx5Muyc9cu6Xxns1Cshtd1llED2km14wr+Maz3l6yQAcMnynOP9pQDquzrqQ+Ig1VsiENhOPl1zDAZfuQqOksQB6WTUExYxMGuOMz6IhhxaPy/sDiMmzRT1q3fJH063RI6uDMat5EXn+hT4IrDrLcWyeMTZsjjgzvKEYcWviLhNSPZ4+CVXG49VhxYcWDFwWwSIQ5mnPyWQhzsisPsLzb4HRJP9S/53wGhenPnL5PJ0+bJhJHd3b85T0w0at7V3eNQunSpqPjOUxWjxk+Tp4Z2KfCpC0+diamEOPikiDggDoiD2SRCHMw4+S2FONgVhzlfBiMOF1cPi8NfmX9Lo+ZdZEiv1lKn5okyePTzsj0zy30k8/s162Ttuo1Sr+5JsnXbX9Lktl7u3ofDDv6P31QqtD7i4BMt4oA4IA5mkwhxMOPktxTiYFcc3vxqo98h8VT/ov9GX/g/Wv6V9Bs6QTZu2iynOvLQo5XsW6miTJnxjjgrEuOHdZXpcxZKryHj861CvPfyCLesrQ/i4JMk4oA4IA5mkwhxMOPktxTiYFcc5gUkDg1jxMFvXtisjzj4pIk4IA6Ig9kkQhzMOPkthTjYFYe3vv7d75B4qn9BtSqe6iWiEuLgkzLigDggDmaTCHEw4+S3FOJgVxze+SYYcWhwIuLgdy4kbf3V67PU+nZY5fIqsXfu3q0S1wn6x/YdKrErVYjeOWyzkXKlS9oMF4q1OztbJW6qBi2RkaHS9X936uVzqZI6fd6SqTNPHMAVyujMFaXhc3OibKkSKrlRzgKKd7/ZpNK3eEHPO3H/eEUC+3dWHHyiRxyiASIOYR6IQ3RuIA5hHohDdG4kszjMX/mHz6uEt+r1T6jsrWICaiEOPiEjDohDYSmEOCAOheUG4pA64rBwVTDiUO94xMHn5Tl5qyMOiAPiYDY/WXFgxaGwTEnmFYf3V202S3DLpc4+fj/LEe2FY8XBJ0vEAXFAHMwmEeKAOKSiOHy4OhhxOPM4xMHszJKCpRAHxAFxMJu4iAPikIrisOjbLWYJbrnUGVXt/jCVze6x4mBAc8eOndJv2ASZO3+pVNyrvHRoebVccdFZbk3EAXFAHAwmkYggDohDKorDRwGJw+mIg9mJJVlLjX56unz741p5sEcr9799Hn5anh/TW8qVLYM4xAwaT1WEgbA5Mjo5EAfEIRXFYfF3waw41D2WFYdkdQKjfp1/zb3ue8CPOvygfOVZcWDFgRUHo2nEikMEJp6qiM6ZZN4cueT7rWYJbrnUacdUshzRXjhuVcRh+ef2TDmnSXvpfGcz92dNy5YpI+1vbyoNzq7NrYoC2LHiwIpDYVOKFQdWHFJxxWFpQOJQB3GwZzqJjrT2t9/dnzNtd1tTadm8saz45ntp1eURmTnhQTmgyr7cquBWRaEpya0KblUUlhysOKTOisOyH4JZcTj1aFYcEn29t9aes+JwRuM2svj1x92Nkc7n9nsfkmsvP08uOrcO4oA4IA6Gs40VB1YcUnHF4eMf/zTMcLvFTjlqH7sBLUbjVoUBTEccpj7ZTw47OOf30W/rOERaXNXQvV3BHodogNyq4FYFtyrin1RYcUidFYflAYlDbcQh/kRK5hKDRk2SzKx/pG/nW+SrlT9Kq65DZdbEB6VK5UqIAysOrDgYTl5WHFhxSMUVh09+CmbF4eQjWXEwPLUkZ7Ft2zOlx+CnZMknX0vlffeRLnc1Y3NkIUPFigMrDqw4xD+PseKQOisOn67ZFn9AFUrUOmJvhah2QnKrwidHblVwq6KwFGJzZDQZVhxYcUjFFYfPAhKHmoiDz6tzEldHHBAHxMFsgiIOiEMqisPnPwez4nDS4aw4mJ1ZUrAU4oA4IA5mExdxQBxSURxW/LLdLMEtl6pxWEXLEe2F41aFT5aIA+KAOJhNIsQBcUAczOaKUwpxMGeVciURB8QBcTCbtogD4pCK4vDF2mBWHP53KCsOZmeWFCyFOCAOiIPZxEUcEIdUFIcvAxKH6oiD2YklFUshDogD4mA2cxEHxCEVxeGrX/8yS3DLpf57yF6WI9oLxx4HnywRB8QBcTCbRIgD4pCK4vD1umDEodrBiIPZmSUFS2XuyFbrdbZeaLU+a73YplKF0mp93rlrt0rsUiVLqMR1gmr1uUwpvT5n/rNLhYfm+zJKK41hyRIZKiycoFqxd+3WOyFpSWXFsv45fxOQOJyIOKjNkcADIw7RQ4A4hHkgDtG5gTiEeWhd3BGH6JyzIQ4rf8sM5DpzwkEVAmnXpFFuVZhQ2kMZxAFxKCw9EAfEobDcQByiySTzisOq9cGIw/EHIg4+L8/JWx1xQBwQB7P5yYoDKw6FZUoyi8PqgMThOMTB7MSSiqUQB8QBcTCbuYgD4pCK4vDthiyzBLdcquoB5S1HtBeOWxU+WSIOiAPiYDaJEAfEIRXF4buNwYjDsf9BHMzOLClYCnFAHBAHs4mLOCAOqSgO3wckDscgDmYnllQshTggDoiD2cxFHBCHVBSHH37/2yzBLZc6uko5yxHtheNWhU+WiAPigDiYTSLEAXFIRXH4MSBxOApxMDuxpGIpxAFxQBzMZi7igDikojj8tCmYFYcj92fFwezMkoKlEAfEAXEwm7iIA+KQiuKw5o9/zBLccqkjKpe1HNFeOG5V+GSJOCAOiIPZJEIcEIdUFIefAxKHwxEHsxNLKpZCHBAHxMFs5iIOiEMqisMvm4NZcThsP1YczM4sKVgKcUAcEAeziYs4IA6pKA5rt/xrluCWSx26bxnLEe2F41aFT5aIA+KAOJhNIsQBcUhFcfg1IHE4BHEwO7GkYinEAXFAHMxmLuKAOKSiOKzbGsyKw8GVWHEwO7OkYCnEAXFAHMwmLuKAOKSiOPz25w6zBLdc6qB9SluOaC8ctyp8skQcEAfEwWwSIQ6IQyqKw/qAxOFAxMHsxJKKpRAHxAFxMJu5iAPikIrisGFbMCsOB+zNioPZmSUFSyEOiAPiYDZxEQfEIRXFYeP2nWYJbrnUfyqWshzRXjhuVfhkiTggDoiD2SRCHBCHVBSH3wMShyqIg9mJJRVLIQ6IA+JgNnMRB8QhFcVh01/BrDjsvxcrDmZnFkpBAAIQgAAEkojAHwGJQ2XEIYmygK5AAAIQgAAEDAlsztxlWNJusf0qlLQb0GI09jhYhEkoCEAAAhAoXgS2ZAUjDvuWRxyKVyZxNBCAAAQgkBYEtgYkDpUQh7TIr0IPcuHiz2XQqEmycdMWqVm9qgzp2VqqVK6U3lBS6OiHjX1Jnn3pDSlRokSo11Oe6CMnHHt4Ch1F+nX169U/Scc+j8nZp9WQXvfcGAKwZu0G6fHgk+L8+6EHVZH+XW+TWtWrph+gJD/inbt2yYgnX5ZnXpwj7894VPartLfb44WLV8hd9w2TUqXC38i73HWd3ND0ApUj+vPv3Spx4wXdp1z4fBOvbKL/nVsVysT/3J4pja7vIo/0uUvq1KomI8ZNlXUbNsmwvm2VWya8LQL9hj4rxx1zuDRvcr6tkMRRJvDJF6tl4IjnpOrRh8ree1WIEocb2z0gZ9WpIbc3v1TmL/rUlfo3X3hESkdciJS7R3gDAu16jpQTqx4hTzz3miyYPiokDrPfXizzFiyV4f3uNojiv8i2f4IRh73LRotDMn0BRRz859UeI7zx7hKZNnuBjHu4s1tu2/ZMqd+0g3w0a4yUKZO8bwZTxpJS4Tv3f1zqn15TLrvwzJTqdzp3ds3a9e6q3sSpc+X3P7aGxGHT5j+lUfMusmjWGClVMucb69V39JFuba+XOrVOTGdkSXfs33y7xhWHGg1ujRKHl157V1Z884MM6HpbQvq8PSBxqBghDsn2BRRxUE69sc/NlE2bt0qP9i1CLTniMHFUDznysAOVWye8DQKtuw6V7Oxs+e7HXyWjRIZce9m50qrFZTZCE0OZwBMTX4sSh+UrVkv/YRPk1WcGhlp2xLBu7WpyTeNzlXtDeC8EYsXhqedfl3nzl0nWP//Klq3b3FtRPTvcKHtVKOclfNw6f/2bHbeMRoG9ymSEwibbF1DEQWPEI2I69+ice3Wd72wW+mvD6zrLqAHtpNpxRyq3TngbBMZNminly5WVqxufK7+u/11adX5Eut3dXC6sf6qN8MRQJBArDh8u+0JGPvmKTBnbJ9Rqz8FPyfHHHi43X3ORYk8I7ZVArDjMW7BMPvvqO7m12cXuqpEjfkcdfqArDxofzZf87am/FUqHxSHZvoAiDhqZFhHTueisW79J+nS6JfTXMxq3kRef6MOKgzJ7rfCPT5zhjmn/LolZKtU6jnSIGysOzt6HXkPGy+vPDQ4dfvveo6Re3ZNYcUjShIgVh9hufvz5Kuk15CmZM/khlSPI2hHMikP5CHFIti+giINKqoWDzp2/TCZPmycTRnZ3/+g8WdGoeVd3j0Pp0sn7SlFlLCkVfvmKVVL9hKOlbO6elNFPT5ctf26P2nCXUgeURp2NFYfNW7fJBdd2kg9eGy3lypZxSVx6433u/fLaNY5PIzKpc6ix4vDdT7+6G14PqLKvexCLP/laHhg5SV579oHUOagi9jTZvoAiDkUcwKIW/yvzb3cz1pBeraVOzRNl8OjnZXtmlvtIJp/UIHBD24Fy+in/lba3NJFf1m2UW+8ZLH073yr16tZIjQNI417GioOD4vZOD8kpJ50gd9zQWN58b4mMfOoVmTN5SGizZBrjSspDjxUH59v3V6t+lBH975bsbJF7+z7mPhp9b+trk7L/NjqVbF9AEQcboxonxkfLv5J+QyfIxk2b5VRHHnq0kn0rVUxAyzRhg4Dz3H/fR56Rr1b/JHtXrODeC29xVUMboYmhRMAR9BdnvCO7d+92N7aWLFlSrmlc370P7txm6vbAWPly5Y9y+CEHyAP3tZTqJxyl1BPCeiGwZet2Offqe9yqO3bsDK3OvjVlqFTcq7wMGD5R3lv0qSt7Dc6uLV3bXC/ly+WsIBXHT7J9AUUcimOWcUwQgAAEIFCsCCTTF1DEoVilFgcDAQhAAAIQ0CWAOOjyJToEIAABCECgWBFAHIrVcHIwEIAABCAAAV0CiIMuX6JDAAIQgAAEihUBxKFYDScHAwEIQAACENAlgDjo8iU6BCAAAQhAoFgRQByK1XByMBCAAAQgAAFdAoiDLl+iQwACEIAABIoVAcShWA0nBwMBCEAAAhDQJYA46PIlOgQgAAEIQKBYEUAcitVwcjAQgAAEIAABXQKIgy5fokMAAhCAAASKFQHEoVgNJwcDAQhAAAIQ0CWAOOjyJToEIAABCECgWBFAHIrVcHIwEIAABCAAAV0CiIMuX6JDAAIQgAAEihUBxKFYDScHA4FgCLw8a748M2WOvP7c4GA6QKsQgEDCCCAOCUNNQxDQJ9Bv6LPy28bN8vjgjlGN/f7HVqnftIM8P6a31PzvsdY7gjhYR0pACCQtAcQhaYeGjkGg6AQQh6IzowYEIFA0AohD0XhRGgJJTaAo4jB52lvu7YUtW7fJkYcdJB1aXiXnnF7TPb7NW7dJv6ETZPHyr2Tnrt1y8v+qSp9Ot8ihB1Vx//2TL1ZL/2ETZM3aDVLrf1Wl7snVZMabH3CrIqmzg85BwA4BxMEOR6JAICkImIrDgo8+k94PPS1jHuwoJ1Q9XBYu/lzu7TtGZjwzUI449EDpNnCsbNi0WR65v42UKV1Keg0ZL//u2OneAtmxc5dccO290vSSc+TOmy6Xld+ukXv7PiZly5ZBHJIiC+gEBHQJIA66fIkOgYQScMThpZnvSYkSGfna3b07O7TH4c5uw6TGiUdL21ubhMq17jpUTqp2jPu37X9luX+vuFd5979z5y+TgSMmyoLpo2Tpp9/I7Z0eko9mjZEK5cu5/z5o1CT5YOkXiENCR5vGIBAMAcQhGO60CgEVAo44fPfTr9K7401R8bds3S633DM4JA6XtOgmP/2yPl8frrjoLBnU/Q5Z/cMvMuqpV2TV97/Irl275J9/d7grDotff1xmzVskD415wZWIvM/kafPk+elvIw4qo0pQCCQXAcQhucaD3kDAFwHTWxWNb+ou1152rtx0zUX52svOzpaGzTpJvdNrSre210u5smXknQ8+ke6DxrniMG32Ahk+bqosfPXRUN0JU9+Ul157F3HwNXpUhkBqEEAcUmOc6CUEjAiYikOb7sOl8r77yMBut4firlu/SQ78T2VZ//tmdw/DnMlD3P0OzufRp6fJpFfmueKwcPEKadtjeNStir6PPCtLP/sGcTAaJQpBILUJIA6pPX70HgJRBEzFwdkc2bHPYzKifzs589Tq8umX34ojE87mx/8ef5SccVlb6dm+hVx16Tny9vvLZfwLs+XLlT/Ih689JhkZGXLe1fdI8yYXSMvml8oXK3+Qng8+JeXKsTmSdIRAOhBAHNJhlDnGtCFgKg4OEGcF4dmX3hDn5VCHHLi/tGpxmVzZ6GyX1fQ5C2XEky+7exsanHWydGlznbtHwtkr8daUofLxilUyaOQk+XndRqld4zhpcFZtmTj1TXnzhYfThjUHCoF0JYA4pOvIc9wQgAAEIAABDwQQBw/QqAIBCEAAAhBIVwKIQ7qOPMcNAQhAAAIQ8EAAcfAAjSoQgAAEIACBdCWAOKTryHPcEIAABCAAAQ8EEAcP0KgCAQhAAAIQSFcCiEO6jjzHDQEIQAACEPBAAHHwAI0qEIAABCAAgXQlgDik68hz3BCAAAQgAAEPBBAHD9CoAgEIQAACEEhXAohDuo48xw0BCEAAAhDwQABx8ACNKhCAAAQgAIF0JYA4pOvIc9wQgAAEIAABDwQQBw/QqAIBCEAAAhBIVwKIQ7qOPMcNAQhAAAIQ8EAAcfAAjSoQgAAEIACBdCWAOKTryHPcEIAABCAAAQ8EEAcP0KgCAQhAAAIQSFcCiEO6jjzHDQEIQAACEPBAAHHwAI0qEIAABCAAgXQlgDik68hz3BCAAAQgAAEPBBAHD9CoAgEIQAACEEhXAohDuo48xw0BCEAAAhDwQABx8ACNKhCAAAQgAIF0JYA4pOvIc9wQgAAEIAABDwQQBw/QqAIBCEAAAhBIV71AyNEAAAH+SURBVAKIQ7qOPMcNAQhAAAIQ8EAAcfAAjSoQgAAEIACBdCWAOKTryHPcEIAABCAAAQ8EEAcP0KgCAQhAAAIQSFcCiEO6jjzHDQEIQAACEPBAAHHwAI0qEIAABCAAgXQlgDik68hz3BCAAAQgAAEPBBAHD9CoAgEIQAACEEhXAohDuo48xw0BCEAAAhDwQABx8ACNKhCAAAQgAIF0JYA4pOvIc9wQgAAEIAABDwQQBw/QqAIBCEAAAhBIVwKIQ7qOPMcNAQhAAAIQ8EAAcfAAjSoQgAAEIACBdCWAOKTryHPcEIAABCAAAQ8EEAcP0KgCAQhAAAIQSFcCiEO6jjzHDQEIQAACEPBAAHHwAI0qEIAABCAAgXQlgDik68hz3BCAAAQgAAEPBBAHD9CoAgEIQAACEEhXAohDuo48xw0BCEAAAhDwQABx8ACNKhCAAAQgAIF0JYA4pOvIc9wQgAAEIAABDwQQBw/QqAIBCEAAAhBIVwKIQ7qOPMcNAQhAAAIQ8EAAcfAAjSoQgAAEIACBdCWAOKTryHPcEIAABCAAAQ8EEAcP0KgCAQhAAAIQSFcCiEO6jjzHDQEIQAACEPBAAHHwAI0qEIAABCAAgXQlgDik68hz3BCAAAQgAAEPBBAHD9CoAgEIQAACEEhXAohDuo48xw0BCEAAAhDwQOD/fvV2bX4cagYAAAAASUVORK5CYII=",
      "text/html": [
       "<div>                            <div id=\"64788e13-2ff2-4916-8269-73d3bc091adf\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"64788e13-2ff2-4916-8269-73d3bc091adf\")) {                    Plotly.newPlot(                        \"64788e13-2ff2-4916-8269-73d3bc091adf\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.007221605163067579,0.007005648221820593,0.0052811382338404655,0.007684369571506977,0.006177148316055536,0.00543549656867981,0.008061942644417286,0.007398218382149935,0.009174970909953117,0.007798024453222752,0.006897738203406334,0.007003214210271835,0.004043221473693848,0.00687289796769619,0.007462042849510908,0.007293550297617912],[0.003388745244592428,0.0008520383853465319,0.004962692968547344,4.1972579925553255e-12,0.0019923218060284853,6.17564967396421e-21,6.461049224526505e-7,0.0012802411802113056,0.004572327248752117,0.0017938436940312386,1.9262912098749752e-10,6.782847418722442e-19,1.6778492004144056e-37,4.964051495163699e-14,0.002202582312747836,0.004415472503751516],[1.0545743965584276e-11,0.0013257188256829977,0.0028420784510672092,2.6150491494458805e-11,2.7385923390133016e-11,8.579650721089196e-16,0.00035306380596011877,0.002317853271961212,0.000014175520846038125,0.002394661772996187,0.0028601076919585466,0.000022160580556374043,0.0021006136666983366,0.006980125792324543,1.828982278129221e-12,0.0001239109260495752],[0.003600881202146411,0.0216167401522398,0.002407241612672806,9.495644007984083e-6,0.0035184475127607584,0.0014035761123523116,0.004660090897232294,5.77428863834939e-7,1.2221848825859913e-23,9.531353839520307e-7,6.683242759208952e-7,0.00018383732822258025,0.0006168605759739876,0.001556359464302659,0.0013503115624189377,0.0007486341055482626],[0.005284776445478201,0.006088730879127979,0.6402723789215088,0.001268303021788597,0.000027627231247606687,0.000010487074177945033,1.0515276471778634e-6,0.0016490078996866941,0.007343673147261143,0.0012581143528223038,0.0013831725809723139,0.004518447443842888,0.00033377474755980074,0.000031605391995981336,0.0002842126996256411,0.6168648600578308],[0.0025591261219233274,0.03251533582806587,0.0016285560559481382,0.1571081280708313,0.32328101992607117,0.20798426866531372,0.11704714596271515,0.5587158203125,0.24138742685317993,0.0008683035266585648,0.00014592513616662472,0.0019469247199594975,0.0020449066068977118,0.05343598872423172,4.6549466787837446e-6,0.08330928534269333],[0.09291791170835495,0.003473499557003379,0.016125187277793884,0.07268991321325302,0.02634909190237522,0.0014974861405789852,0.005410476587712765,0.005121956113725901,0.006888541392982006,0.00004132581307203509,0.019403278827667236,0.0061025796458125114,0.05686185136437416,0.024155929684638977,0.021906157955527306,0.000018004451703745872],[0.007500350009649992,0.011111307889223099,0.009122894145548344,0.059650395065546036,0.012209426611661911,0.026963815093040466,0.012048366479575634,0.025827638804912567,0.006679396145045757,0.03958643972873688,0.04641525447368622,0.030032876878976822,0.01838570646941662,0.00953657552599907,0.02023862488567829,0.013849987648427486]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}<br>Layer: %{y}<br>color: %{z}<extra></extra>\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]]},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('64788e13-2ff2-4916-8269-73d3bc091adf');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "induction_logits = model.run_with_hooks(\n",
    "    rand_tokens_repeat, fwd_hooks=[(filter_attn_hooks, calc_induction_score)]\n",
    ")\n",
    "px.imshow(induction_scores_array,\n",
    "          labels={\"y\": \"Layer\", \"x\": \"Head\"},\n",
    "          color_continuous_scale=\"Blues\",\n",
    "         )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf094deb-fb94-4c50-b76e-74dc382652a1",
   "metadata": {},
   "source": [
    "It appears that the max score for the random tokens is around 0.6. Above that threshold there appear to be two attention heads `L4H2` with score 0.6402724, and `L4H15` with score 0.6168649. A third attention head is slightly less activated, `L5H7`, with score 0.5587158. Is this an induction head, or some other kind of attention head?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ecd63b-32ce-443c-97ba-ef51df6f860e",
   "metadata": {},
   "source": [
    "**Validation**: We can ablate the top few heads by the metric defined by `calculate_induction_score`, and show that performance drops substantially."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e7cd0ee0-79e4-4196-9822-7d2a2047f339",
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh_score = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "240f3607-0c4d-4d65-b7f1-941d359f111e",
   "metadata": {},
   "outputs": [],
   "source": [
    "induction_logits = model(rand_tokens_repeat)\n",
    "induction_log_probs = F.log_softmax(induction_logits, dim=-1)\n",
    "induction_pred_log_probs = torch.gather(\n",
    "    induction_log_probs[:, :-1], -1, rand_tokens_repeat[:, 1:, None]\n",
    ")[..., 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "b53c87ae-2064-4227-94ac-8b8be0f7d885",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False,  True, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False,  True],\n",
       "       [False, False, False, False, False, False, False,  True, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False],\n",
       "       [False, False, False, False, False, False, False, False, False,\n",
       "        False, False, False, False, False, False, False]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The chart above seems pretty sparse so we print this to see what the attention mask might look like after pruning\n",
    "induction_scores_array > thresh_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9b4084d2-3263-4ed1-a8fe-bb51a055a8e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mask out the heads with the highest induction score\n",
    "attn_head_mask = induction_scores_array > thresh_score\n",
    "\n",
    "def prune_attn_heads(value, hook):\n",
    "    \"\"\"Value has shape [batch, pos, index, d_head]\n",
    "    \"\"\"\n",
    "    mask = attn_head_mask[hook.layer()]\n",
    "    value[:, :, mask] = 0.0\n",
    "    return value\n",
    "\n",
    "def filter_value_hooks(name):\n",
    "    return name.split(\".\")[-1] == \"hook_v\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c784f464-bb87-45c0-b844-8d5b6587eca2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original loss on repeated sequence: -6.096261024475098\n",
      "Loss on repeated sequence without induction heads: -9.745206832885742\n"
     ]
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "coloraxis": "coloraxis",
         "hovertemplate": "Head: %{x}<br>Layer: %{y}<br>color: %{z}<extra></extra>",
         "name": "0",
         "type": "heatmap",
         "xaxis": "x",
         "yaxis": "y",
         "z": [
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           255,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           255
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           255,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ]
         ]
        }
       ],
       "layout": {
        "autosize": true,
        "coloraxis": {
         "colorscale": [
          [
           0,
           "rgb(247,251,255)"
          ],
          [
           0.125,
           "rgb(222,235,247)"
          ],
          [
           0.25,
           "rgb(198,219,239)"
          ],
          [
           0.375,
           "rgb(158,202,225)"
          ],
          [
           0.5,
           "rgb(107,174,214)"
          ],
          [
           0.625,
           "rgb(66,146,198)"
          ],
          [
           0.75,
           "rgb(33,113,181)"
          ],
          [
           0.875,
           "rgb(8,81,156)"
          ],
          [
           1,
           "rgb(8,48,107)"
          ]
         ]
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Mask"
        },
        "xaxis": {
         "anchor": "y",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0,
          1
         ],
         "range": [
          -0.5,
          15.5
         ],
         "scaleanchor": "y",
         "title": {
          "text": "Head"
         }
        },
        "yaxis": {
         "anchor": "x",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0.004166666666666652,
          0.9958333333333333
         ],
         "range": [
          7.5,
          -0.5
         ],
         "title": {
          "text": "Layer"
         }
        }
       }
      },
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg4AAAFoCAYAAAA//nPjAAAAAXNSR0IArs4c6QAAIABJREFUeF7t3XmcTuX/x/HPGPuSSChRopBIJITIyDrWrGPJMvZ9G7LvDGYwZJc9RSjZSSgpS7ITSmTLHhm73+M+fk0N+jpnznU292v++T1+X5/r+lzneR2+7+91ztx3wN27d+8KPwgggAACCCCAgA6BAIKDDiVKEEAAAQQQQEATIDhwIyCAAAIIIICAbgGCg24qChFAAAEEEECA4MA9gAACCCCAAAK6BQgOuqkoRAABBBBAAAGCA/cAAggggAACCOgWIDjopqIQAQQQQAABBAgO3AMIIIAAAgggoFuA4KCbikIEEEAAAQQQIDhwDyCAAAIIIICAbgGCg24qChFAAAEEEECA4MA9gAACCCCAAAK6BQgOuqkoRAABBBBAAAGCA/cAAggggAACCOgWIDjopqIQAQQQQAABBAgO3AMIIIAAAgggoFuA4KCbikIEEEAAAQQQIDhwDyCAAAIIIICAbgGCg24qChFAAAEEEECA4MA9gAACCCCAAAK6BQgOuqkoRAABBBBAAAGCA/cAAggggAACCOgWIDjopqIQAQQQQAABBAgO3AMIIIAAAgggoFuA4KCbikIEEEAAAQQQIDhwDyCAAAIIIICAbgGCg24qChFAAAEEEECA4MA9gAACCCCAAAK6BQgOuqkoRAABBBBAAAGCA/cAAggggAACCOgWIDjopqIQAQQQQAABBAgO3AMIIIAAAgggoFuA4KCbikIEEEAAAQQQIDhwDyCAAAIIIICAbgGCg24qChFAAAEEEECA4MA9gAACCCCAAAK6BQgOuqkoRAABBBBAAAFbg8OXq76TboMnSZrUKWXt/JESGBjvgR3YfeBXqdmsn/af7/hqqsQPDFS2S3/3Xz4nXDJlSKdsXiZCAAEEEEDAXwQcCQ6JEyWUyL6tpFih1x5wHhw1WxYs3SDXrt8gOPjLXch1IoAAAgh4RsCR4FC0QC5JmiSxFh7+/XPr9m0pXrW9ZH8pk2zauofg4JnbiIUigAACCPiLgCPBoUe7ejJs3FxZv3C0pEyRLMZ67cbt0qH3WGnX5D2JmDAvVnD4dvMumTT7Szn4y+9y4+YtyZQhrTSoWVYqlS4cM37/oaMyavJ82b3/iPwVfU3SP51KKpQqLM3rVZR48QLkYY8qFi3/RnoN+0gGfxAqFUv9M5e/3ABcJwIIIIAAAkYEHAkOaz6NkOD6H0iXlrWkVqUSMett33us3L5zR4rkf1X6j5wZExyOHj+t1ZcPKiQNa5WVRAkTyMp1m2X0lAUyZUQXKfRGTvGdVrzzXnvJ/UoWad2wiiRLmkR27DmkzdO8fkVpXLvcA8Fh/aYd0qbnaOnWuo6EVAky4kYtAggggAACfingSHDYtGScDBo1S377/ZR8MqGPBn/p8l9SrGo7iejdUs6evxgrOPjedzhx+pykfzq1JE2SKGajCgW3lCpli0pYq9ry+8kzUrp2FxnULVQqlykSU3Pg8DFJljSxPPfM07GCw4VLV6RRh3BpVq+CNK1bwS83n4tGAAEEEEDAqIBjwWHP/l8ltPNwWTxjsGR5/ln59Iu1MnrqAlm/MEoWLl0fKzj4LmrV+q0y/8t1cuT3U3Ljxk25e/eunL94WSqWeksGf9BEbt++I3VaDZBfj52S2pWDtFOIvK++JAkSxI8x+ftRxYeD20v3oZOlcukiWujgBwEEEEAAAQT0CTgWHJInTSKlanWSckEFpWOzGhLScoC8mj2zdG9bVwsR/35UseH7HdKi20jtfYb3a5SR1E+mkICAAHkvtLcUzv+qFhx8P1ejr8usz1bJ6g1bZd/B37TTCd97Cx2aVpfkyZLEnDj4TiB8tWXeeVNG9G6hT4oqBBBAAAEEEBDHgsMTyZNq7ygsXrlRpo3qJmXrhMm8iX0lZ7YXHggOHft+KDv3HpY18yJjtsz3TsObZZtr/+X/d3D4935evHRFVq3fIsPHfyoli+aTId2bxASHJnWCtT6+dyr6dHxfalR8h1sBAQQQQAABBHQIOBocfC89lq3TVXtPYee+X2Tx9EHaku8/cfA90rhw8bIsmNL/gccOvhMFXyg4ceqsbN9zSMoHFYx12T3Dp2ovSX45c8gDL0cOGj1LPlu6QeaO6yXZs2bSwUUJAggggAAC/i3gaHDw0ddtPUh27D2kPU5oVKvcQ4PDh9MWyYRZi2V0/zbyavYXZeOWXbJ41UaJvnZDe7dh/NAOcvL0OandcoDUr1ZKKpUpoj2mOHTkuPQZPk3effsN6dWh/gPBwfeuREirgXI1+prMn9RPe4mSHwQQQAABBBD4bwHHg8NnS9ZL34jp2kdQp03z5EODg++/2H3vPKz/7ie5c/euFM6fS3q0qys//LhP+kZMk3RpUmknCl9/t12mfrxMCwzXb9zUPsehdPE3peX7lSRhwgQP/RwH36nHe6F9tE+x5H0H/qoggAACCCDwvwVsDQ5sBgIIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwsJWbZggggAACCHhbgODg7f1j9QgggAACCNgqQHCwlZtmCCCAAAIIeFuA4ODt/WP1CCCAAAII2CpAcLCVm2YIIIAAAgh4W4Dg4O39Y/UIIIAAAgjYKkBwMMl9/eYdXTMEBARIgIjcuXtXVz1F7hIICBBh69y1J0ZWw/4Z0XJXre/fTSP/aiZKEM9dF/AYrobgYHJTr90yOQHDEUAAAQSUCSSOr2wqJvoPAYKDyVuD4GASkOEIIICAQgGCg0JMgoM1mAQHa1yZFQEEEIiLAMEhLmrGxnDiYMzrgWqCg0lAhiOAAAIKBQgOCjE5cbAGk+BgjSuzIoAAAnERIDjERc3YGE4cjHlx4mDSi+EIIICAlQIEByt1781NcNBhfPT4H9J9yGTZd/A3yZA+jfQPayR5cmbVRnLioAOQEgQQQMAmAYKD9dAEBx3G9doMksL5c0njkPKyftNPMjhqtqycO0ISxA8kOOjwowQBBBCwS4DgYL00weERxucu/CllQrrIpiXjJH5goFZdrUkf6dqqtuTPk53gYP09SgcEEEBAtwDBQTdVnAsJDo+g+3HXQekfOUM+nzYwprJz//FSIG8OqR5cnOAQ51uPgQgggIB6ARXBIcnrbdQvTMeM0dvH6KhyvoTg8Ig9+G7rbhk9eYF8OrFPTGWPoVPk5SwZ5f3qpQkOzt/DrAABBBCIEVASHPK1c0Q0ettoR/oabUpweITY9t0HpWf4VFk6a2hMZdteUVK0QG5OHIzebdQjgAACFgsoCQ75O1q8yodPH70l0pG+RpsSHB4hduHSZSlZo5NsXDxWEidKqFWXr9dNBoQ1kry5XubEwegdRz0CCCBgoYCS4PBmZwtX+N9TR28e4Uhfo00JDjrEGncaJvlyZ5MmdYJl5brNMnrKAlk+J1x7WZJfx9QBSAkCCCBgk4CS4FCwq02rjd0m+vtwR/oabUpw0CF28vQ56Tpoouw5cEQyPptWBnULlZzZXtBGEhx0AFKCAAII2CSgJDi89YFNq70vOHw3xJG+RpsSHIyK3VdPcDAJyHAEEEBAoYCS4FC4h8IV6Z8qeuMg/cUOVhIcTOITHEwCMhwBBBBQKKAkOBTtrXBF+qeK/qa//mIHKwkOJvEJDiYBGY4AAggoFFASHIr1Vbgi/VNFr3emr/4V3qskOBgVu6+e4GASkOEIIICAQgElwaH4AIUr0j9V9Lpe+osdrCQ4mMQnOJgEZDgCCCCgUEBJcCjhzLsG0WudebfCKD/BwagYJw4mxRiOAAIIWCegJDgE/fOBf9at9MGZo7/qZme7OPciOMSZ7t5AThxMAjIcAQQQUCigJDi8O0zhivRPFb06TH+xg5UEB5P4BAeTgAxHAAEEFAooCQ6lIxSuSP9U0Ss76S92sJLgYBKf4GASkOEIIICAQgElwaHMSIUr0j9V9IoO+osdrCQ4mMQnOJgEZDgCCCCgUEBJcCjnzLdURi9z5ls5jfITHIyK3VdPcDAJyHAEEEBAoYCS4BA8VuGK9E8VvaS1/mIHKwkOJvEJDiYBGY4AAggoFFASHCqMU7gi/VNFf9lSf7GDlQQHk/gEB5OADEcAAQQUCigJDpUmKlyR/qmiv2imv9jBSoKDSXyCg0lAhiOAAAIKBZQEhyqTFa5I/1TRi5roL3awkuBgEp/gYBKQ4QgggIBCASXBoepUhSvSP1X0wsb6ix2sJDiYxCc4mARkOAIIIKBQQEVwSFrtI4Ur0j/V1c8a6S92sJLgYBKf4GASkOEIIICAQgEVwSF5jekKV6R/qivzGugvdrCS4GASn+BgEpDhCCCAgEIBFcEhRc0ZClekf6rLn76vv9jBSoKDSXyCg0lAhiOAAAIKBVQEhydqz1S4Iv1T/Tm3vv5iBysJDibxCQ4mARmOAAIIKBRQERxShsxSuCL9U136uJ7+YgcrCQ4m8QkOJgEZjgACCCgUUBEcUtWdo3BF+qe6MLuO/mIHKwkOJvEJDiYBGY4AAggoFFARHFLX/1jhivRPdX5miP5iBysJDibxCQ4mARmOAAIIKBRQERyeen+uwhXpn+rcjNr6ix2sJDiYxCc4mARkOAIIIKBQQEVwSNPwE4Ur0j/V2Wm19Bc7WElwMIlPcDAJyHAEEEBAoYCK4JC20TyFK9I/1R8f1YhVvPbbHyVi4jw5c+6iZMuSUfp1aSQvZnpGrl2/IX2GT5Ovv9suSRInktaNqkj14OLa2KPH/5DuQybLvoO/SYb0aaR/WCPJkzOr/kXoqCQ46ED6XyUEB5OADEcAAQQUCqgIDulC5ytckf6pTk+pHlN8+swFqdigu0wc1kly58giYz5aKD/tOSjTRnaTqKkLZN/BoxLRp4X46t5vN0SmRobJS5mfk3ptBknh/LmkcUh5Wb/pJxkcNVtWzh0hCeIH6l/IIyoJDiYpCQ4mARmOAAIIKBRQERzSN/1M4Yr0T3VqUrVYwWHnvsPy7ttvaP+Z7wShVfdRsnb+SKlQ/wMZ2C1UXnsli/Znwz6cK8mTJZGalUpImZAusmnJOIkfeC8oVGvSR7q2qi3582TXvxCCgzKrh05EcLDWl9kRQAABIwIqgsMzzRYYaams9uTE9/5zrqlzl8mBQ0dlWK/m8lpQY9mwKEpSPpFMq5+3+GvZuuOA1KocJP0jZ8jn0wbGzNO5/3gpkDdHzKMMFYvlxMGkIsHBJCDDEUAAAYUCKoJDhhaLFK5I/1THx1d5aPG3m3fJgJEzZdaYHpLqyRSSp2Rj2bpikiRJnFCr/3zFt7JmwzYJqRokoycvkE8n9omZp8fQKfJylozyfvXS+hfCiYMyK04crKVkdgQQQMC0gIrg8FzLz02vIy4T/D6u8gPDlqzZJONnfCHjh3aQTBnSaX/uO3H4an6kpEmdUvv/Zy9YLTv3HpbaVYKkZ/hUWTpraMw8bXtFSdECuTlxiMuGWDWGEwerZJkXAQQQMC6gIjhkbP2F8cYKRhwbWynWLL7fqoiaulCmRHSJCQm+gkoNe0iPtvXkzdfvvbfQL2K6pHs6tdSs9I6UrNFJNi4eK4kT3TuNKF+vmwwIayR5c72sYIX3puBRhUlKgoNJQIYjgAACCgVUBIdMbRcrXJH+qY5GVYwpvnT5L6nSqKfMjOouzz3zdKxJJs76Urbv/lki+7aW30+ekYYdhsrsMT0kc6ZnpHGnYZIvdzZpUidYVq7bLKOnLJDlc8JjXpbUv5r/riQ4mFQkOJgEZDgCCCCgUEBFcHih3RKFK9I/1ZHRwTHFi5Z/oz12SJAgfqwJ1n02SpIlTSx9I6bL6g1bJWmSxNKhaXWpVLqwVnfy9DnpOmii7DlwRDI+m1YGdQuVnNle0L8IHZUEBx1I/6uE4GASkOEIIICAQgEVwSFzh6UKV6R/ql9Hltdf7GAlwcEkPsHBJCDDEUAAAYUCKoLDix2XKVyR/ql+iSynv9jBSoKDSXyCg0lAhiOAAAIKBVQEh6ydlytckf6pDo0oq7/YwUqCgw78w0eOS9+IGXLg8FHtzdbOLWpJicKvayMJDjoAKUEAAQRsElARHF4OW2HTamO3+XlYGUf6Gm1KcNAh5vvVl2rli0mdqu/Kxi27pWPfsbJh0RjtwzcIDjoAKUEAAQRsElARHLJ1dSY4HAgnONh0m1jb5tbt2+J7u7VK2aIxv85SoHwLmT+pn2TKkJbgYC0/syOAAAKGBFQEhxwfrDTUU1XxviHqPt1R1ZoeNg8nDgZ1d+37Rdr1HiNrPo2UePECCA4G/ShHAAEErBRQERxy9lhl5RL/c+49g0o50tdoU4KDATHfB2007TJCerWvL4XeyKmN5FGFAUBKEUAAAYsFVASHV3uutniVD59+98B3HelrtCnBQafYgcPHpF2vMdKtdYgUfytPzCiCg05AyhBAAAEbBFQEh9y919iw0gdb7Oxf0pG+RpsSHHSIHTvxhzTpPEIGf9BE8uZ6KdYIgoMOQEoQQAABmwRUBIc8fb+yabWx2/zUN8iRvkabEhx0iDVoP1RqVnxHypYo8EA1wUEHICUIIICATQIqgsPr/dbatNrYbbb3KeFIX6NNCQ6PEPO911C6dpcHPi98RO8WUrJoPt5xMHrHUY8AAghYKKAiOOQb4Exw2NaL4GDhreGeqTlxcM9esBIEEEBARXB4Y+DXjkBu7fmOI32NNuXEwajYffUEB5OADEcAAQQUCqgIDm8OXqdwRfqn2ty9uP5iBysJDibxCQ4mARmOAAIIKBRQERwKDl2vcEX6p/q+WzH9xQ5WEhxM4hMcTAIyHAEEEFAooCI4FArfoHBF+qfa1PVt/cUOVhIcTOITHEwCMhwBBBBQKKAiOBQe/o3CFemfamOXovqLHawkOJjEJziYBGQ4AgggoFBARXAoGvGtwhXpn+qbTkX0FztYSXAwiU9wMAnIcAQQQEChgIrg8HakM8FhQ0eCg8Jbwb1TERzcuzesDAEE/E9ARXAoPmqjI3Dr2hd2pK/Rppw4GBW7r57gYBKQ4QgggIBCARXBoUTUdwpXpH+qtW3f0l/sYCXBwSQ+wcEkIMMRQAABhQIqgkPJMZsUrkj/VGvaFNJf7GAlwcEkPsHBJCDDEUAAAYUCKoJDqQ+/V7gi/VOtalVQf7GDla4ODnfu3JWffzkmWZ5/9oHvinDQLFZrgoNbdoJ1IIAAAiIqgkOZcT84Qrmi5YNfpOjIQh7R1NXB4e7du5K3dFNZPidc0j+d2o1+fMmVK3eFRSGAgL8KqAgO5cY7ExyWtSA4KLlvp85dJr5vqGxaJ1ieSfeUkjlVTsKJg0pN5kIAAQTMCagIDsETN5tbRBxHL2n2ZhxH2jvM1ScOPooyIWFy8c8rcvnKVYkfGCgJEgTGEtq6YpK9Yvd1Izg4yk9zBBBAIJaAiuBQcdIWR1QXN83vSF+jTV0fHNZu3C4J4vvCQsBDr61ogVxGr1lpPcFBKSeTIYAAAqYEVASHylO2mlpDXAd/HvpGXIfaOs71weFvjVu3b8vpMxckQ/o0tgI9qhnB4VFC/DkCCCBgn4CK4FB16jb7FvyvTgsb53Okr9Gmrg8OvkcUg6PmyNKvNsnt23dkz7rpcv7iZekyYLwM69lcnkr1hNFrVlpPcFDKyWQIIICAKQEVwaHaR84Eh88aERxMbf7fg3uGT5Uz5y5KywaVJaTlAC04XI2+Lv1HzpBr127IqP6tlfSJ6yQEh7jKMQ4BBBBQL6AiONSY/qP6hemYcV6DvDqqnC9x/YlDsart5PNpAyVVyhSSs3gDLTj4fv68clVK1+osm5aMc1SR4OAoP80RQACBWAIqgkOtGdsdUf3k/dcd6Wu0qeuDQ77STeXbL8ZKksQJYwWHi5euSMmaHYXfqjC65dQjgAACj6+AiuAQMusnR4A+rpfHkb5Gm7o+ODQLi9A+ObJD0+qS591Q7cTh5OlzMjhqtty6fUfGD+1g9JqV1nPioJSTyRBAAAFTAiqCQ93ZO0ytIa6DZ9d9La5DbR3n+uDg+/Cnjn0/lJ8PH5Obt25L8mRJ5Mpf0ZIrx4sS2aelPOvwb1kQHGy9X2mGAAII/E8BFcGh/hxngsPMOgQHpbf3rv2/ytHjpyVeQIBkypBOcmZ7Qen8cZ2M4BBXOcYhgAAC6gVUBIcGc3eqX5iOGafXzq2jyvkS15841GreTyqUekvKligoqZ9M4bzYfSsgOLhuS1gQAgj4sYCK4NDok12OCH5Uy9kPNNR70a4PDuNmfCGr12+Rw7+dkML5c0mFd9+SEkVel8SJEuq9RkvrCA6W8jI5AgggYEhARXAInbfbUE9VxVNqvKpqKkvncX1w+PvqfY8pVq3fKqvWb5Ejx05JyaL5pGKpwlIgbw4JCHj4x1FbKvf/kxMc7FCmBwIIIKBPQEVwaDp/j75miqsmVc+peEZrpvNMcPj78n0vSC5atkEiJs7TXpL0fQR1aJ1gqR5czJEAQXCw5sZkVgQQQCAuAiqCQ/PPnAkOE6oRHOKy5w8dc/fuXdm282f5cvV3smrdFkmYMIH23kOl0oXl2IkzEj72Yyn+Vh75oE0dZT31TuTF4PDttoN6L89QXZF8LxmqpxgBBPxXwKp/h0oWMP/vUMuFex3ZmHFVX3Gkr9Gmrj9xGDX5M1myZpOcPX9JShTOK5XLFNbedQgMjBdzrQd//V1qNe8v21ba/xXbXgwOqfJb8zHdF7aMNXr/UY8AAn4qYNW/Q9Hbzf871GbRPkd2ZUyVHI70NdrU9cGhVov+UrlMESkXVFCeSJ401vXdvHlLEiSILzdu3JTxMxdLu9D3jF6/6XqCwz+EBAfTtxMTIOA3Am4ODu2+cCY4jK5EcLD0L4Dv/Yby9brJ+oWjLe3zqMkJDgSHR90j/DkCCDwo4Obg0HHxfke2LLJidkf6Gm3q+hOHYyf+kEGjZ8ven4/I9Rs3Y64v+tp17aOoF3000Og1K60nOBAclN5QTIaAnwi4OTh0/tKZ4DCiAsFBye0f2nm4JEuSWMqXLCQDRs6Qfl0ayZ4Dv8rGLbu176nwfWumkz8EB4KDk/cfvRHwqoCbg0PXpQccYQ0vn82Rvkabuv7EIX/ZZrJ2/khJkTyplKzZSdZ8GqFd48p1W2T9pp9k8AdNjF6z0nqCA8FB6Q3FZAj4iYCbg0P3ZT87sguDy73sSF+jTV0fHAoGt5Tlc8K1k4VStTrLkplDtF/H9P2KZqEKreT7JeOMXnOc631f5V2uXldp1/g9qVmphDYPwYHgEOcbioEI+LGAm4NDzxXOBIeBZQgOSv5KtOs1Ri5fuSpRA9tK2MAJkj7tU1KnSpBs23VQPpy2yNaXI7sPmSybf9ovTULKExwesrv8VoWSW55JEPALATcHhz4rrfmsm0dtbL/S5j+D4lE9VPy5608czl+8rH3AU68O9eX4qbPS6oORcvKP85IoYQLp3fF97Vc17fjZvH2/jJvxuWR9IYO8lDkDwYHgYMdtRw8EHlsBNweHfqucCQ59ShEcLLnhfY8oTp25IKlSJrfti658nxdRo1lfiejbSj5euIbg8B87y4mDJbc8kyLwWAq4OTgMXHPIEfOeJbM60tdoU9efOPzXBf1x9qL4fuNi8fRBRq/ZcP246Z9r71S0alhFBo6aRXAgOBi+hxiAAAKxBdwcHAZ/5Uxw6B5EcLD074nvsYXvZck966Zb2sf3TZyd+o2TueN6aS9lEhz+m5sTB0tvRSZH4LEScHNwCP/6sCPWXd/J4khfo009e+JgV3CYPm+FTJy5WPtoa9/PX1evad+TEVKlpLRvUo3fqvjXHUdwMPrXj3oE/FfAzcFh+DpngkOX4rGDw63bt8X3fU3TPlku334xJuZzi65dvyF9hk+Tr7/bLkkSJ5LWjapI9eDi2s109Pgf4nuRf9/B37Rvj+4f1kjy5FR7kkFwMPj3lhMHThwM3jKUI4DAQwTcHBwi1//iyJ51LPZirL5teoyW7FkzyYRZi2XDoqiY4BA1dYHsO3hUIvq0kNNnLsj77YbI1MgweSnzc1KvzSDtiyAbh5S/91lHUbNl5dwRkiB+oLJrIjgYpCQ4EBwM3jKUI4CAx4LDqG+cCQ7ti8YODvsPHdWCQ64SDWMFhwr1P5CB3ULltVfunVAM+3CuJE+WRPttvzIhXWTTknESP/BeUKjWpI90bVVb8udR93HWrg0Ob5RRo4aqAAAYWklEQVRp+j//st29K+I7rrH6HYdH/Y3nA6D+EeJRxaPuFv4cAQT+FnDzicOYb391ZKPaFMn80L73B4fXghprQSLlE8m0+nmLv5atOw5IrcpB0j9yhnw+7Z/vcOrcf7wUyJsj5lGGigtzbXBY9tUPuq6vXFABXXVWFREcCA5W3VvMi8DjLODm4DDuuyOO0Ld864VHBoebt25LnpKNZeuKSZIkcUKt/vMV38qaDdskpGqQjJ68QD6d2Cdmnh5Dp8jLWTLK+9VLK7sm1wYHZVdo8UQEB4KDxbcY0yPwWAq4OThM2ORMcGhe6NHBwXcz+E4cvpofKWlSp9TujdkLVsvOvYeldpUg6Rk+VZbOGhpzz7TtFSVFC+T2jxMHr/xNITgQHLxyr7JOBNwk4ObgMPn73xyhalLw+UeeOPgKKjXsIT3a1pM3X7/33kK/iOmS7unUUrPSO1KyRifZuHhszAcklq/XTQaENZK8udR9DwYnDiZvD4IDwcHkLcRwBPxSwM3BYepmZ4JD4zf1BYeJs76U7bt/lsi+reX3k2ekYYehMntMD8mc6Rlp3GmY5MudTZrUCZaV6zbL6CkLtC+K/PtlSRU3G8HBpCLBgeBg8hZiOAJ+KeDm4DB9y1FH9qRB/kwxfX3fxly8Wnvt//d97cHfnyW05tMISZkimfSNmC6rN2yVpEkSS4em1aVS6cJa7cnT56TroImy58ARyfhsWhnULVRyZnv4I5C4XiTBIa5y/z+O4EBwMHkLMRwBvxRwc3CYufWYI3tS/42MjvQ12pTgYFTsvnovBofgZqNNXvXDhy+Z2M6SeZn0H4Fvt1n3rX1F8nnjm/m4Hx4PAav+HVoz1fy/Q7O3ORMc6uYjODwed/cjrsKLwcEvNuYxvUir/leaj4vP4XhMbxo/u6zE974dwNTPx9t+NzU+roND8j0X16G2juPEwSQ3wcEkIMMNCRAcDHFR7IcCKoLDJz8ed0SuVt4MjvQ12pTgYFTsvnqCg0lAhhsSIDgY4qLYDwVUBId5250JDjVeJzj4xS1LcPCLbXbNRRIcXLMVLMSlAiqCw2c/nXDk6qrledaRvkabcuJgVIwTB5NiDDcjQHAwo8dYfxBQERwW7jjpCFXV155xpK/RpgQHo2IEB5NiDDcjQHAwo8dYfxBQERw+3+lMcKicm+DgD/eo8KjCL7bZNRdJcHDNVrAQlwqoCA6Ld51y5Ooq5krvSF+jTTlxMCrGiYNJMYabESA4mNFjrD8IqAgOS3addoQqOFc6R/oabUpwMCpGcDApxnAzAgQHM3qM9QcBFcFh2e4/HKEq92paR/oabUpwMCpGcDApxnAzAgQHM3qM9QcBFcFhxR5ngkOZnAQHf7hHecfBL3bZPRdJcHDPXrASdwqoCA6r9p5x5OJKvfK0I32NNuXEwagYJw4mxRhuRoDgYEaPsf4goCI4rNl31hGqkjnSONLXaFOCg1ExgoNJMYabESA4mNFjrD8IqAgOa/c7ExxKZCc4+MM9yqMKv9hl91wkwcE9e8FK3CmgIjisO3DOkYsrnu0pR/oabcqJg1ExThxMijHcjADBwYweY/1BQEVw2HDgvCNUb2dL7Uhfo00JDkbFCA4mxRhuRoDgYEaPsf4goCI4fPuzM8GhyMsEB3+4R3lU4Re77J6LJDi4Zy9YiTsFVASH7w5ecOTi3noplSN9jTblxMGoGCcOJsUYbkaA4GBGj7H+IKAiOGw6dNERqkJZn3Skr9GmBAejYgQHk2IMNyNAcDCjx1h/EFARHL4/7ExwKJiF4OAP9yiPKvxil91zkQQH9+wFK3GngIrgsPmXS45c3JsvpnSkr9GmnDgYFePEwaQYw80IEBzM6DHWHwRUBIctDgWH/AQHf7hFhRMH/9hm11xlcLPRlq1lycR2ls3NxAjYJaAiOGz91ZkThzcyc+Jg133iaJ9rtxxtT3MEEEAAgX8JqAgO24786YhpvheecKSv0aY8qjAqxqMKk2IMRwABBKwTUBEcfvzNmeCQ93mCg3V3hotm5sTBRZvBUhBAwO8FVASHn45edsQxT6YUjvQ12pQTB6NinDiYFGM4AgggYJ2AiuCww6Hg8BrBwbobw00zc+Lgpt1gLQgg4O8CKoLDzmPOnDjkzsiJg1/cvwQHv9hmLhIBBDwioCI47Pr9iiNXm+u55I70NdqURxU6xG7evCX9ImfIqvVbJHmyJNIutJpUKl1YG0lw0AFICQIIIGCTgIrgsPu4M8Hh1QwEB5tuE+vbjP1okRw6clyGdG+q/d8+wz+Sj8f1ksSJEhIcrOenAwIIIKBbQEVw2HviL939VBa+8mwyldNZNhcnDjpog6p3lKmRYfJCxvQPVHPioAOQEgQQQMAmARXBYZ9DwSEHwcGmu8TiNn9euSpvV2krnZvXlDkLV0uihAmlbeOqUqJIXh5VWGzP9AgggIBRARXBYf9JZ04csj/DiYPR/XZl/fFTZ6VMSBdp06iqhIYEy679v0jTLiPkyxlDJG2aJ3lU4cpdY1EIIOCvAiqCw4FTVx3hy5Y+qSN9jTblUcUjxHwnDoWCW8oPS8drL0b6fhp3HCY1Kr4jpYvnJzgYveOoRwABBCwUUBEcfj7tTHB4OR3BwcJbw96pfcFh/uR+8twzT2uNG3UIl7rvvas9ruAdB3v3gm4IIIDA/xJQERwO/RHtCHLWtPf+x6nbfzhx0LFDg6Nmy9Xo69K3cwPZe+CINA2LkCUzh0ia1CkJDjr8KEEAAQTsElARHA47FByyEBzsuk2s73P5ylXpPnSKbN6+T1I/+YR0aVGTlyOtZ6cDAgggYFhARXD45YwzJw4vPs2Jg+EN9+IAHlV4cddYMwIIPK4CKoLDr2evOcKTOU1iR/oabcqjCqNi99UTHEwCMhwBBBBQKKAiOBw550xweOEpgoPCW8G9UxEc3Ls3rAwBBPxPQEVwOHr+uiNwmVIncqSv0aacOBgV48TBpBjDEUAAAesEVASHYw4Fh4wEB+tuDDfNzImDm3aDtSCAgL8LqAgOv19w5sThuVScOPjF/Utw8Itt5iIRQMAjAiqCw/GLNxy52gxPJnSkr9GmPKowKsajCpNiDEcAAQSsE1ARHE5eciY4PJOS4GDdneGimTlxcNFmsBQEEPB7ARXB4dSfNx1xTP9EAkf6Gm3KiYNRMU4cTIoxHAEEELBOQEVwOO1QcEhHcLDuxnDTzJw4uGk3WAsCCPi7gIrg8MdlZ04c0qbgxMEv7l+Cg19sMxeJAAIeEVARHM5eueXI1aZJHt+Rvkab8qjCqBiPKkyKMRwBBBCwTkBFcDj3lzPB4alkBAfr7gwXzcyJg4s2g6UggIDfC6gIDuev3nbEMXXSQEf6Gm3KiYNRMU4cTIoxHAEEELBOQEVwuOBQcEhFcLDuxnDTzJw4uGk3WAsCCPi7gIrgcCnamROHlEk4cfCL+5fg4BfbzEUigIBHBFQEhz+v3XHkap9IHC9W329+2CmDo2bLmXMX5bWcWSW8RzNJkzqlI2v7d1MeVZjcAoKDSUCGI4AAAgoFVASHy9edCQ4pEv0THP68clXK1O4iI/q0kPx5csioSfPl5B/nJLJvK4VacZuK4BA3t5hRBAeTgAxHAAEEFAqoCA5XbtxVuCL9UyVPGBBTvOLrzbJw2QaZNLyz9p9dvnJVilVtJ98vGScJEzr7eQ8EB/17+tBKgoNJQIYjgAACCgVUBIerDgWHpP8KDhNnfSnnLlyS7m3rxuj4gsPMqO7y/HPpFIoZn4rgYNws1giCg0lAhiOAAAIKBVQEh+ibzpw4JEnwz4nDqMmfya3bt6Vz85oxOu/W6ixRA9pIjpeeVyhmfCqCg3EzgoNJM4YjgAACVgmoCA5Wrc3IvJNmfyknT5+TPp0axAwrFNxSPpnQhxMHI5BurOXEwY27wpoQQMBfBR6X4LBq/VaZs3C1zBj9gbaVvt+sKBMSpr3jkCCBs58wyYmDyb9dBAeTgAxHAAEEFAo8LsHhr6vXpExIFwnv2Uzyv5Zdho79WK5cjdZ+JdPpH4KDyR0gOJgEZDgCCCCgUOBxCQ4+ku9/3Cv9ImbImXMX5A1feOjeVJ5MmVyhVtymIjjEzS1mFMHBJCDDEUAAAYUCj1NwUMiidCqCg0lOgoNJQIYjgAACCgUIDgox/2MqgoNJY4KDSUCGI4AAAgoFCA4KMQkO1mASHKxxZVYEEEAgLgIEh7ioGRvDiYMxrweqCQ4mARmOAAIIKBQgOCjE5MTBGszrN/V9GUpAQID4PhPszl1nPpHMmqv3n1kDAkTYOu/uN/vn4b0TESP/aiZKEPsbJr175e5dOScO7t0bVoYAAggggIDrBAgOrtsSFoQAAggggIB7BQgO7t0bVoYAAggggIDrBAgONmzJ5DlLZMa8ldo3nZULKig92taVwECew9lAb7pF5MR5Mn3eCokX75/9+nRCH8mWJaPpuZnAGoF9B3+TDn0+lCJv5pKe7evFNDl6/A/pPmSy+P48Q/o00j+skeTJmdWaRTBrnAWWrNkk/SKmy8CuoVK6eH5tnm9+2CUtukVK/PiBMfN2aVFL6lQtGec+DIy7AMEh7na6Rn6/ba/0HDZV+6KSlCmSSYtuI6VcUAGpXTlI13iKnBXw/QP20osZJaQK++XsTujrvn33QRk4apZkzZxBUiRLGis41GszSArnzyWNQ8rL+k0/yeCo2bJy7ghJ8K//MtLXhSqrBHwhfduOA9oXOjWsVS4mOCz76gdZvWGLjOzX2qrWzGtAgOBgACsupf1HzpRn0qaWJnWCteFff7ddO32YPqpbXKZjjM0CnfuPl2IFX5MKpd6yuTPt4iJw9PhpSZM6pcycv0rOnr8UExzOXfhT+8KgTUvGSfzAe/+rtVqTPtK1VW3Jnyd7XFoxxgKB/YeOaqd5oZ2GS42K78QEh3mLv5Zd+3+VAWGNLOjKlEYFCA5GxQzWN+40TGpVKiHvvv2GNvLXoyelYYdwWbdglMGZKHdCoFlYhNy9e1cOHzkhAfECpEaF4tK0bgUnlkJPAwITZi6OFRx+3HVQ+kfOkM+nDYyZxRcKC+TNIdWDixuYmVI7BBp3HBYrOEz5eKmsXr9Voq/fkIuXLmuPoXq0qyfJkia2Yzn0uE+A4GDxLVGn1UBpVq+CvF3wNa3TiVNnpXKjnrJ52QSLOzO9CoFJs7+UJIkTSbXg4nLi9Flp2nmEdG0dIqWK3QuC/LhT4P7g8N3W3TJ68gL5dGKfmAX3GDpFXs6SUd6vXtqdF+HHq7o/OKzesFV27D0sDWuW1U6MfKHvhYzptPDAj/0CBAeLzUM7D5eqZd/W3mvw/Rw4fEx8/yuWEweL4S2afvzML+Tk6XPSvwtHphYRK5n2/uDge/ehZ/hUWTpraMz8bXtFSdECuTlxUCKudpL7g8P9s2/b+bP0DJ8iy+cMU9uY2XQJEBx0McW9aNDoWfLkE8mlVcMq2iS+l3wWLF0vUyPD4j4pI20T+HHXz5IzW2ZJlDCB1nPsR4vk4p9XYr10Z9tiaKRb4P7gcOHSZSlZo5NsXDxWEidKqM1Tvl437Zl53lwv656XQnsE7g8Oh387ob3smjbNk9oCfti+TwaNni2Lpw+yZ0F0iSVAcLD4hvD9F0/YgAkyM6q7JEuWRDvq9r308175ty3uzPQqBHyPmgrme0VaNagiv588Iw3bD5W+nRtK0QK5VEzPHBYJ3B8cfG187xvly51Ne1F55brNMnrKAlk+JzzmZUmLlsK0cRC4PziMmvyZ7P35iIzq31r76PeOfT/UXqLs2KxGHGZniFkBgoNZQR3jZ8xfKVPmLJGbt25L5TJFtDe5fd9dwY/7BXy/+993xDTZe/A3SZE8qfY8vO5777p/4X66wqFjP5ZPvlgrd+7c0V5qDQwMlOrBxbRn4b5HTF0HTZQ9B45IxmfTyqBuoZIz2wt+KuXOy/b9psuhI8fl1q3bEhgvnvZCcniPplKsUB4ZMHKmrNv0kxb0ShTJK2Eta0uSxPdOj/ixV4DgYK833RBAAAEEEPC0AMHB09vH4hFAAAEEELBXgOBgrzfdEEAAAQQQ8LQAwcHT28fiEUAAAQQQsFeA4GCvN90QQAABBBDwtADBwdPbx+IRQAABBBCwV4DgYK833RBAAAEEEPC0AMHB09vH4hFAAAEEELBXgOBgrzfdEEAAAQQQ8LQAwcHT28fiEUAAAQQQsFeA4GCvN90QQAABBBDwtADBwdPbx+IRQAABBBCwV4DgYK833RBAAAEEEPC0AMHB09vH4hFAAAEEELBXgOBgrzfdEEAAAQQQ8LQAwcHT28fiEUAAAQQQsFeA4GCvN90QQAABBBDwtADBwdPbx+IRcF7gnWrtpUuL2lIuqIDzi2EFCCBguQDBwXJiGiBgvcDZ85ekWNV28unEPvJqtsyxGvaLmC7HT52VScM7W7IQgoMlrEyKgGsFCA6u3RoWhoB+AYKDfisqEUDAnADBwZwfoxFwhYCR4HDi1FnpP3KmbN2xX5InSypvF8wtYS1rS/JkSbRrWbxqo0yavUQ7pXgq1RPSoEYZqfveu9qf3bx1W4aMmSMr1v4gCRMmkJYNKsuEmV9I5+a1eFThijuBRSBgvQDBwXpjOiBguYCR4FCreT95LWdWaRdaTa7fuCHdBk3SAsLgD5rIkWOnpHy9bhI1oK0ULZBLduw9LKGdhsvsD3tKruyZZc7C1TLl46XyUWRXSZsmlURMnCeLln8jg7qGEhws32UaIOAOAYKDO/aBVSBgSuDv4BAvXsAD89y5c1cK539Ve8dh1/5fpV7rgbJl+URJkCC+Vrtz72Gp22aQ/LhqsgRIgJy/+Kc8/dSTMfNUathDQioHSc1KJaRB+6GSO8eL0rFZDe3PL166IoUrtZbhvVoQHEztIIMR8I4AwcE7e8VKEfhPgb+Dw4jeLSRr5gyx6sbPWCxX/rqqBYelX30vYQMmPHSeVZ+MkGfTPSVT5y6TZV99L39e/kskIEDOnruoBYX61UtL2Tph2qMLX4j4++etCq2kZ/v6BAfuTwT8RIDg4CcbzWU+3gJ6H1UsX/uD9I2YLj8sHf9QkIXLNkjkxPkyPryj9mjC9/NeaG+pVLqwFhyCqneUJnWDpda/gsOb5ZpL304NCQ6P9y3G1SEQI0Bw4GZA4DEQ0Bsc9hw4IjWa9ZU18yLlmbSptSu/Gn1drl2/IamfTCE9w6fKzZu3JLxnM+3PrvwVLb5ft2zTqKoWHGq16C9v5ske86jizLmLUvy99jyqeAzuIS4BAb0CBAe9UtQh4GIBvcHBdwk1m/WTdGlTyYAujcX3ToTvtyROn7kgUyPD5MNpi2TF15tl7vjecuvWbek94iM5fOSEvFP4dencvKaMm/65zF+yTnvskSZ1Shk+7hNZtX6L9O/SmBMHF98fLA0BlQIEB5WazIWAQwJGgsPvJ8/IwFGztF/HDAwMlIJ5X5FeHeprQcD3smOnfuNk577Dkv7p1BLWqracOnNewsd+LK0bVZXalYNkwMiZ8tU327Rfx2zdqIrMmr9KQuuUl4qlCjt09bRFAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwUIDnZq0wsBBBBAAAGPCxAcPL6BLB8BBBBAAAE7BQgOdmrTCwEEEEAAAY8LEBw8voEsHwEEEEAAATsFCA52atMLAQQQQAABjwsQHDy+gSwfAQQQQAABOwX+D3HKYuYwvQZ/AAAAAElFTkSuQmCC",
      "text/html": [
       "<div>                            <div id=\"840aa1ff-d516-4340-8712-0c7be276f510\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"840aa1ff-d516-4340-8712-0c7be276f510\")) {                    Plotly.newPlot(                        \"840aa1ff-d516-4340-8712-0c7be276f510\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,255],[0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}<br>Layer: %{y}<br>color: %{z}<extra></extra>\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]]},\"title\":{\"text\":\"Mask\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('840aa1ff-d516-4340-8712-0c7be276f510');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ablated_logits = model.run_with_hooks(\n",
    "    rand_tokens_repeat, fwd_hooks=[(filter_value_hooks, prune_attn_heads)]\n",
    ")\n",
    "ablated_log_probs = F.log_softmax(ablated_logits, dim=-1)\n",
    "ablated_pred_log_probs = torch.gather(\n",
    "    ablated_log_probs[:, :-1], -1, rand_tokens_repeat[:, 1:, None]\n",
    ")[..., 0]\n",
    "print(\n",
    "    \"Original loss on repeated sequence:\", induction_pred_log_probs[:, seq_len:].mean().item()\n",
    ")\n",
    "print(\n",
    "    \"Loss on repeated sequence without induction heads:\",\n",
    "    ablated_pred_log_probs[:, seq_len:].mean().item(),\n",
    ")\n",
    "px.imshow(\n",
    "    attn_head_mask,\n",
    "    labels={\"y\": \"Layer\", \"x\": \"Head\"},\n",
    "    color_continuous_scale=\"Blues\",\n",
    "    title=\"Mask\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62fd62e5-d6c0-44de-b67f-9bee49319bf2",
   "metadata": {},
   "source": [
    "A threshold of 0.5 gives us 3 heads and drops performance in the following way:\n",
    "\n",
    ">Original loss on repeated sequence: -6.096261024475098\n",
    "\n",
    ">Loss on repeated sequence without induction heads: -9.745206832885742"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b6dc153-5de7-4ab1-85d2-2606004c8663",
   "metadata": {},
   "source": [
    "A threshold of 0.6 gives us two heads, with the following loss:\n",
    ">Original loss on repeated sequence: -6.096261024475098\n",
    "\n",
    ">Loss on repeated sequence without induction heads: -8.345271110534668"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "368b0b06-d800-4e5c-9401-57d6f1201939",
   "metadata": {},
   "source": [
    "### Ablating heads that mostly attend to the previous token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "850e64d7-dcec-4baa-ad49-adc0e9d6ca96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Long text: we have two runs with two different prompts and different answers, eg 'Steve Jobs founded' -> ' Apple' and 'Bill Gates founded' -> 'Microsoft'. We patch parts of the layer outputs or residual stream from specific tokens and positions and see which patches significantly shift the answer from 'Apple' to 'Microsoft'\n"
     ]
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "coloraxis": "coloraxis",
         "hovertemplate": "x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>",
         "name": "0",
         "type": "heatmap",
         "x": [
          "Head 0",
          "Head 1",
          "Head 2",
          "Head 3",
          "Head 4",
          "Head 5",
          "Head 6",
          "Head 7",
          "Head 8",
          "Head 9",
          "Head 10",
          "Head 11",
          "Head 12",
          "Head 13",
          "Head 14",
          "Head 15"
         ],
         "xaxis": "x",
         "y": [
          "Layer 0",
          "Layer 1",
          "Layer 2",
          "Layer 3",
          "Layer 4",
          "Layer 5",
          "Layer 6",
          "Layer 7"
         ],
         "yaxis": "y",
         "z": [
          [
           0.027988387271761894,
           0.050817787647247314,
           0.06861968338489532,
           0.031243784353137016,
           0.040589071810245514,
           0.042291149497032166,
           0.022635014727711678,
           0.04909639433026314,
           0.05521165952086449,
           0.018792012706398964,
           0.051243629306554794,
           0.034362636506557465,
           0.1176576018333435,
           0.04433203861117363,
           0.03281690552830696,
           0.019391782581806183
          ],
          [
           0.051099956035614014,
           0.05374652147293091,
           0.01564853824675083,
           0.18012017011642456,
           0.045214418321847916,
           0.2715858221054077,
           0.10926873236894608,
           0.044607922434806824,
           0.05784713104367256,
           0.044993799179792404,
           0.16187720000743866,
           0.25243663787841797,
           0.41557085514068604,
           0.19385139644145966,
           0.07484206557273865,
           0.02463134564459324
          ],
          [
           0.38289040327072144,
           0.1295502781867981,
           0.05824587121605873,
           0.46146976947784424,
           0.37013423442840576,
           0.36953499913215637,
           0.08449036628007889,
           0.03721775487065315,
           0.1587517261505127,
           0.1831357777118683,
           0.04390658438205719,
           0.2249511480331421,
           0.049546193331480026,
           0.04181525483727455,
           0.31112954020500183,
           0.06048464775085449
          ],
          [
           0.07017214596271515,
           0.07232293486595154,
           0.03803824633359909,
           0.23560552299022675,
           0.03785311058163643,
           0.07164649665355682,
           0.03152455762028694,
           0.2580982446670532,
           0.9985824227333069,
           0.3006895184516907,
           0.2997439503669739,
           0.0989137589931488,
           0.3864877223968506,
           0.11750023066997528,
           0.17233213782310486,
           0.013575251214206219
          ],
          [
           0.08191242069005966,
           0.07029113173484802,
           0.024787385016679764,
           0.2186519205570221,
           0.1651458591222763,
           0.1930815726518631,
           0.19477494060993195,
           0.07729773223400116,
           0.03770160675048828,
           0.04388655349612236,
           0.08513171970844269,
           0.029208874329924583,
           0.13372521102428436,
           0.18646501004695892,
           0.054016366600990295,
           0.02008497528731823
          ],
          [
           0.10474904626607895,
           0.07148798555135727,
           0.08303850144147873,
           0.022116560488939285,
           0.0245988592505455,
           0.021530311554670334,
           0.03135489672422409,
           0.019414279609918594,
           0.027183979749679565,
           0.1106434240937233,
           0.10631193965673447,
           0.07634454220533371,
           0.05989508330821991,
           0.040975600481033325,
           0.39580607414245605,
           0.02888302505016327
          ],
          [
           0.031818609684705734,
           0.1340329498052597,
           0.044984325766563416,
           0.037765853106975555,
           0.06614495068788528,
           0.08521831780672073,
           0.164445698261261,
           0.04341691732406616,
           0.07583362609148026,
           0.27800294756889343,
           0.06184188649058342,
           0.08890586346387863,
           0.03657130151987076,
           0.0444006472826004,
           0.06741028279066086,
           0.16647715866565704
          ],
          [
           0.04968482628464699,
           0.04948631301522255,
           0.03500191494822502,
           0.041399430483579636,
           0.06689193844795227,
           0.03444244712591171,
           0.06597341597080231,
           0.04243674874305725,
           0.08141452819108963,
           0.037084221839904785,
           0.03423188626766205,
           0.03423410281538963,
           0.04130541533231735,
           0.06264433264732361,
           0.06538373976945877,
           0.03236440569162369
          ]
         ]
        }
       ],
       "layout": {
        "autosize": true,
        "coloraxis": {
         "colorscale": [
          [
           0,
           "rgb(247,251,255)"
          ],
          [
           0.125,
           "rgb(222,235,247)"
          ],
          [
           0.25,
           "rgb(198,219,239)"
          ],
          [
           0.375,
           "rgb(158,202,225)"
          ],
          [
           0.5,
           "rgb(107,174,214)"
          ],
          [
           0.625,
           "rgb(66,146,198)"
          ],
          [
           0.75,
           "rgb(33,113,181)"
          ],
          [
           0.875,
           "rgb(8,81,156)"
          ],
          [
           1,
           "rgb(8,48,107)"
          ]
         ]
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Prev Token Scores"
        },
        "xaxis": {
         "anchor": "y",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0,
          1
         ],
         "range": [
          -0.5,
          15.5
         ],
         "scaleanchor": "y",
         "type": "category"
        },
        "yaxis": {
         "anchor": "x",
         "autorange": true,
         "constrain": "domain",
         "domain": [
          0,
          1
         ],
         "range": [
          7.5,
          -0.5
         ],
         "type": "category"
        }
       }
      },
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg4AAAFoCAYAAAA//nPjAAAAAXNSR0IArs4c6QAAIABJREFUeF7snXl8Ddcfht8rsSSU1r51obRU7bXUj5ZQS221RSwllghqF0osSbSlIYi91qJa2lDVlqKUVluqqosudEGlIYhEEBGR3N/nnORuWWTkzrnJtO/8U03OfGfmmXNmnnzPMiaz2WwGNxIgARIgARIgARLQQMBEcdBAiUVIgARIgARIgAQkAYoDKwIJkAAJkAAJkIBmAhQHzahYkARIgARIgARIgOLAOkACJEACJEACJKCZAMVBMyoWJAESIAESIAESoDiwDpAACZAACZAACWgmQHHQjIoFSYAESIAESIAEKA6sAyRAAiRAAiRAApoJUBw0o2JBEiABEiABEiABigPrAAmQAAmQAAmQgGYCFAfNqFiQBEiABEiABEiA4sA6QAIkQAIkQAIkoJkAxUEzKhYkARIgARIgARKgOLAOkAAJkAAJkAAJaCZAcdCMigVJgARIgARIgAQoDqwDJEACJEACJEACmglQHDSjYkESIAESIAESIAGKA+sACZAACZAACZCAZgIUB82oWJAESIAESIAESIDiwDpAAiRAAiRAAiSgmQDFQTMqFiQBEiABEiABEqA4sA6QAAmQAAmQAAloJkBx0IyKBUmABEiABEiABCgOrAMkQAIkQAIkQAKaCVAcNKNiQRIgARIgARIgAYoD6wAJkAAJkAAJkIBmAhQHzahYkARIgARIgARIgOLAOkACJEACJEACJKCZAMVBMyoWJAESIAESIAESoDiwDpAACZAACZAACWgmQHHQjIoFSYAESIAESIAEKA6sAyRAAiRAAiRAApoJUBw0o2JBEiABEiABEiABigPrAAmQAAmQAAmQgGYCFAfNqFiQBEiABEiABEiA4sA6QAIkQAIkQAIkoJkAxUEzKhYkARIgARIgARJwqTh8tPdrTJm9yoF6gQImlC9bCg2erI6XBnXDQ5XK5sldadN7Ii5cvHLXYz9V93FsWDT1rmUs1/jJ26F4qFK5PLkWcdBD3/yEt9//FL+f/gexcddwXzFPVH24Iry7tELH1k3z7Lx4YBIgARIgAWMTyBNxCBzTD9WrPCjJJd+5g7ORF/Dmu7tx/cZNbFszC5UrlHE51W9/OIlbScnW4waFrcMDJe7DOL9e1p/dX7woatesmu/FYfMH+/Fq+Fvo1qEF2rdqjJL334crcdex/ZMvsOfgtwgc0x/9urdxOWMekARIgARIwPgE8kQc3loSiAa1H3Og98eZf/DCoOkY7PM8Jg73zkQ2JSUVJpMJIkPhiq3ji1NQvkxJrF0w+Z4Olx8yDh36vYxyZR7A+vApmc59zIzFMMGERa+MvqfrutfCyXdSUNDd7V53Y3kSIAESIIF8TiDfiIPg1KiDP1o0qYsFwSNlt4HoPng9cBg+2P0lvv3xJD55ey4qlS+NI8d/xbI3P8Cvv5+FyQTUqfkoxvn1RJ0nHsXNxFto8cIY9Oj4jPzL2n6bvfhtvL/rc3yxfQk8PQrf9dZkJw4Hvv4eK9/6CL//FSlFpnqVSvDr1xmtWzSQ8bISh+2fHMKMueswe+pQdGn7P1lOZAU2b9+Pc1EX4elZBC0a10HAiN4oU+p++fud+49g8itv4P21r2DRmq049uMpuBUogKefqoWgCb4oUbxotuffrs8k2U2yOiwgx+pnNpvx5ruf4L0PDyL60hWULf0Auj//DIb172yVtD/PRCF89VYc++kUbt1KQsXypWU2Y0ifjtYybX0C0Op/9VGgQAG8u+MzzBg/QJa5mZiExWu3Ye/n3+JK7DWUKVUCnZ5rJrulLGIRF38dC1ZG4MujPyHu6nWUKF4MLZrUwaQRPne9zhwvjgVIgARIgAR0J5BvxOFK3DU8020MXuzZFlNG9UVMbDye7T4Wjz/6INo88xSaPVULNas/jJ9+/QuDJ4TiuWeewoiBXSWQpeu248ujJ7B1dQiqPFRBvnC/+f43HNgabn2xpaaa0arnOPyv0ZOYPdUvR5BZicOhb05g+Mvz0atTS/Tr0Ub+5f7Wtr3Y+vHnWDl3Ipo3rp1JHD4//CNGT1+EKaP6oW+31vK4QjzEy3T04O54vnVTXIqJw6yFG2FOTZVdNYUKFZRdChOCl+HJx6tg0kgf1H+yOn4/HYkXR78G786tMPmlPtlew7zlW7D+vd3o4NVEdkkIsXJzK5BlecFu7ZZdmDzSR2aBhIyFLNgAX+/2UsYuX7mKrr7T8MiD5eUxSz1QHOKa5i7fLLNDoozYBK9CBd1R5aGK8pgPVy6H0iVLYMjEufjl1FnMHD8QdWs9ih9/+QshC9bL+/fqy0PkvmNnLJHXFhIwGBXKlUTUhRi8tugtVKpQGm+ETszxXrEACZAACZCA6wjkiTiIFHr92tXlVd65k4KzkdEQLzvxV/V7q4KlLIi/Qpt3HS1fxuKlbNnEi+j03+ex55158gUrtsRbt9Gm9wS0feYpBE30heUFL47TqF4NWebId7/Kl9i6hS+jSf2aORLOShwGjp2Di5fjIAY+imyD2EQXivhr+9FHKmLVvAAHcYiLv4HB40Ph/2Jn+Re82JJuJ8vratGkNhYEv2Q9jxO/nYbPiFmYN2MEnm/dxCoO4i93n65e1nLDJoXJv+I3LZ2W7TUkJ9/BglUR8i9/cTyRXalbqxqeblgLndo8LbsxxHYr6TZavDBaipC9iAjpEBmfqaP7Ydmb27Fi44fYs3mezPZYNiFnB77+AV9/tExmDrr4TpMC9Pn7i1A4/b4cP/GHFJ3p415EnxfSpElsazfvwsJVEdj/3gJ5Lm28J6BJgyfw2pSh1jLi+Fev3ZCyyI0ESIAESCD/EMgTccjq8qs+VAHTxr6Ipg2fkL+2iINIaY9MzyyInzdsNwytmzfA3BnDHcKMClwk0/4fbpgtX+ZevcajTYuGMmUuNtFVcPjYz/j03fnWl/7dbkNW4iCOLQYb2r/gRIzxQUul9Bz6YIlVHJbNHofA11fjhXbNHV7KJ06egc/wEAQH+MoXtv3WpOMI+WIX52zJOGx5Iwi1a1SxFpsYshyn/orExxvn5FiLRLeNECZxbqKbQfzlL17yM8YPlF05P586g97+IZgfNFJeV1bb8JcX4K+/z+PTLWEOv35n+36ZFRC8H324ohSHUg/chzcX2sZVrNuyC/PfeA+7NoXKDIRl++2Pv9HTL0h2SbVr2Rhzl23Gxq170bXd/2R3R+P6NVG8mGeO18cCJEACJEACrieQJ+Iwa9JgPF4tbVZFAVPadEwx8t9+s4iD/QyAOykpqNt6iOx+cHNzHHiXkpKC+4p6yr+AxRa6bDN27juMg9sWQfxOdIP06dYaY4b00EQ5ozhYjj2wV7tM3QRBYW/iw71f4/u9q63iUNSziMwMiBdy2MwR1mMePvYLhgbMg7ubG0wZBnqKTIGQHTFw0SIOH7z5KqpXqewgDif/PIedb72u6TrsCwmxGh+0DH+djcK+9xbgj9P/yHMRktOyWb0s4/Uf9ZrMTIhuIPvt432H8fKrK7F5+Qw5tkSIg5C/8FmjrMUWrdmGVZs+QsGC7plii2sVs2v6dX9O/u7DvV9hx56v8N1Pv8v71bxxHdllZS8c93zB3IEESIAESEB3AnkiDlnNqsh4ZVmJgyhjGUA5anC3TDCEhIi+eLFZ/qoVfwHfuJmI0dMWZfrL9240s8o4PNV+GNo+2yjTGAnRR//DL3/KNL1lcKRfv06o9fgjGDdzKYImDJTrJ4hN/NXv7R8sB/4983TdTKcguhXEbA5nxCHy/CWUK/2AtSvH/iD7Dx2HmFkhBk7eV6yozH6IsQZiIGNW24gpCyFmvOx7d77Dr8UaEWKwqch8iHElWYnD+nd3Y96KLdi4OBAPZBBDEazU/cUzDX4UknL4u18QtuJd2c0iMh2WbiHdaz8DkgAJkAAJ3DMBw4mDX0AYrsTFy0GE9i+Uc1GXUL6M48tSvMyaNngCV+OvIyo6Bm8vm64ZUFbiIMZIiOOI8RWWaaEiE9G61wTUrlEVS2ePzTQ4UqTzt+78Qv5lXqPaQ7h9Oxn/6zoa3To0zzTrQ3QJiL/axXXlVhws4zvEmAWRHcm4iTELyzfswI43X0PlimXQvOsotG7REKHT/K1FV7/9Mb4+9rPsdrAM5BQzWuwX5xLdM98c/012z4iBl1mJg5Cpfi+9KrMQYjCkZRNdKPHXElChXCncSEjE50d+RPNGtR0kQmQfAuesxhfbF8sBmdxIgARIgATyBwHDiYPorx80/nX0eP5Z2fXgUaSwnFEh/rIVMwPsB+GteWcnIj46iPjrCZjg7w3vzo5jCu414yDGC4jUfvfnW2Cgd3s5sFNMZRRdIhsWibUpqmcSByEKfV96VU4TjVgVAtGFIV7GKzZ8IM/p2afr4XZysjzPLTs+w3srg6Vg5FYcxPTKkVPD5cqRvTo9K7MaYiGrq/E38NW3J+QxnnumkRxfIDYxSFGMRRBdOM2eehKiG0TIzoBe7eSMCTHbpYtvoJzeOXVUX9xfohj2HfpO7jd2aE8M7dtRxslKHMTPBS/RNSK6nJ547BE5gFLM5Pg76qLMVpjNQFufiXIQ5PABXVG29P24FHMVC1a+h2vXE+QYCm4kQAIkQAL5h4DhxEGgEy/wZevT1nEQm+ie8HnBK9Ngw+jLsWjjPVH2sYtuhHsZcJfdOg5iKqJ46Z86/Y8cnyFehi8NekFmNsSW1ToOYmxBj6FBePbputbxDvbrOBQuXEhOuxTTS8Wy1mLLrTiIfUUWREwR/fjTw3LAqJidIARLjJXo9NzTkpNleqaYprp2805Z/uLlWJQrUxI9Oz3rsEbD6XMXsOCN93D0h9+QlJQsxx0IabOXtOzEQYzzWLLufew9+C0ux17F/cWLSVbjh/WSGQexCbEIX7MNP/z8h1w9VHRrNG1QC2OGdLeWyT9NhmdCAiRAAv9tAi4Vh/82al49CZAACZAACRifAMXB+PeQV0ACJEACJEACLiNAcXAZah6IBEiABEiABIxPgOJg/HvIKyABEiABEiABlxGgOLgMNQ9EAiRAAiRAAsYnQHEw/j3kFZAACZAACZCAywhQHFyGmgciARIgARIgAeMToDgY/x7yCkiABEiABEjAZQQoDi5DzQORAAmQAAmQgPEJUByMfw95BSRAAiRAAiTgMgIUB5eh5oFIgARIgARIwPgEKA7Gv4e8AhIgARIgARJwGQGKg8tQ80AkQAIkQAIkYHwCFAfj30NeAQmQAAmQAAm4jADFwWWoeSASIAESIAESMD4BioPx7yGvgARIgARIgARcRoDi4DLUPBAJkAAJkAAJGJ8AxcH495BXQAIkQAIkQAIuI0BxcBlqHogESIAESIAEjE+A4mD8e8grIAESIAESIAGXEaA4uAw1D0QCJEACJEACxidAcTD+PeQVkAAJkAAJkIDLCFAcXIaaByIBEiABEiAB4xOgOBj/HvIKSIAESIAESMBlBCgOLkPNA5EACZAACZCA8QlQHIx/D3kFJEACJEACJOAyAhQHl6HmgUiABEiABEjA+AQoDsa/h7wCEiABEiABEnAZAYqDy1DzQCRAAiRAAiRgfAIUB+PfQ14BCZAACZAACbiMAMXBZah5IBIgARIgARIwPgGKg/HvIa+ABEiABEiABFxGgOLgMtQ8EAmQAAmQAAkYnwDFwfj3kFdAAiRAAiRAAi4jQHFwGWoeiARIgARIgASMT4DiYPx7yCsgARIgARIgAZcRoDi4DDUPRAIkQAIkQALGJ0BxMP495BWQAAmQAAmQgMsIUBxchpoHIgESIAESIAHjE6A4GP8e8gpIgARIgARIwGUEKA4uQ80DkQAJkAAJkIDxCVAcjH8PeQUkQAIkQAIk4DICFAeXoeaBSIAESIAESMD4BCgOxr+HvAISIAESIAEScBkBioPLUPNAJEACJEACJGB8AhQH499DXgEJkAAJkAAJuIwAxcFlqHkgEiABEiABEjA+AYqD8e8hr4AESIAESIAEXEaA4uAy1DwQCZAACZAACRifAMXB+PeQV0ACJEACJEACLiNAcXASdcJts5MRst89OdUMKAifkpqq5pxNQCF3N5j0ji4wCBa6B047UbNgrCC2u7tJyf0T55ycoqZumFVUOMEYQBH3AkpAJ6WkKuPsXkBBxQAQl3Bb71aSXpmBEkULKoldwKSGhThZNzf924o42yJqUCjha6SgFAcn75YRxSE1NVXR6wEoVFCBOIgXj3hRqnhuifevirgA3MXDUMlmQvIdNfKnShwEhsLubrrTEIQT76QqqRriZOU9VCDvcTduq6nPAEp4qnlbmkwmZZzdhFPqHJ3ioHtzswakODjJVpU4iEqflGLWuSmltU2ZcVDwMJQvh4L6vxxkVkBkHBRt4hWs9ytenG1BVeJgApKTVfAwQwULy22TdUPBad8ynDiYEJeQpKg2qxOHAiL7ouD+yYyDosyOhxqHUnbvjBKY4uDknTKcOMCMFIUvYSOKg5pnoRnuIjWvJLi6jIPIRektUVZxUJBxELGTFGVfZMZB0QvtasJtNVVDZBzE21LvmygSfgUUZhwUZXYoDk6+4LLZneLgJFdV4iBa/u0UBSlYkXFISVHwN3ZayMLu7uk92k6Ctd9ddCeY1f0trOTdrrKrwgTcSVbRqaAy42BCYTnGQf9NlThYs0YKKkhcQrL+7SS9Vd/nWVB3bxB3TekYBwVdFeKcKQ76tzfZqWQ2y6Fh3HJJQJ04pA+Ay+V5ZbebuNmpqSrFQVVXhSJxMOIYB1XiIDLRqh4HUioVdFWYgFvJCgQ7vQEpyTiYgLjrCsc4qMg4wIwCJjXiJyzHTdHAS4qDzi+Q9HD5WhzupKSgbush2B+xAOXLlFRDQEPUc1GXEDhnNX77429UKl8asyYPRr1a1eSeRhMHcc7KZlVYBsDpnCaVZquwe0WVOSubVaFKHKAii5HWwOSsCkVjHFRlHMR5i753nauzZCG6KlRs4lyLKxAHcc4i46A3C8nABLibTEq6bigOKmpZPs845IU4pKSkwi1tiK91e3H0a/hfo9oY0rcjPj/8A2Yv3oQ9m8NQ0N1NmTiIBnpbzCRQsKmaVSHOWcyq0H1TPB1TTS7DDPcM9UhPLsl31NSNtKhqYquYVSFeOknJajJosqtCxRgHkXG4oaarQty94oq6KtTOqlChJGZ4FFQRV8+WbMxYhs04iJRq6LLN+OzL40g1m9G4Xg2ZCVi4MgJJt5MxfdyL8o7EX09Aqx7jZNYi/loCgsLeRExsPDw9iiBwTD/Uf7I6fv39LKaHrkX1KpVxMSYO68OnWO/mlbhraN93Eg5/vBzubmkvxZ5+QXj5pT5oVK+GMnEQGp4s5qcr2FLEeAEF7wZl4qByVoXSrgohoGpAS3FQEFrV4Mi0dRwUSKUQBzE4UgEL0fTcC6i5h8rWcYAJJTzFOCO9N4VdFZxVoffNUh7PsOJw4OvvpSRErA6R6bPew0Pg168zHnmwHPwnz8eBbeHyRb9jz1fYue8wVs0LQI+hM+Hzghd6dWqJEyfPYPS0Rfh0SxjOREaj78hZeGXyEHTwauIA/fiJPzBrwQZ88Oar1p8HzFqBJg1qyjjKuirMwO1UNSPcVXZVqFjHQa7PlKomjS7qjpq/V8UaAGpeOlIqVcwkUDnGAQoHR6paACo9ha73U1h2VRhwHQe1gyMVZAZMgIcKh9K7QhgwnmHFQWQcEm8lycyB2ILD1qNi+VIY1r8zOg2Yiimj+qJ549pSDryaN0DTBk+g88CpOLprJeR8ZADe/sGYNMIH95coht7+ITi2e5X1d5Z7+fWxn7Fo9Ta8uzLIenunvb4Gjz36IAb2aqdOHMTqgIpWjlTVVSEAKZuOqWgBKFETlOR1zIAc46BoU9JVoVQc0uuG3pkB2VWh5A6m9b0rWrtAXcZB0ToOZqCAoimTaZmd/DvGwaP+aEWt+O5hE79fkifH1XJQw4pD7NXrCFuxBaf/Pg+YTDgfHYO+3dpg+IAuWL5hByKjLmHmhIFo2WMs9m4JQ+T5S+gzYhbK2Q2yFOIxc/xAVH24AvwCwnBwW3gmZt///Ifsxtj51uvW342ZsRgtmtRRmnFQNsbBBKSmKFo5UtGS0yozDuKmqhnjoHLlSDVLTtskSu+3u6BskktOq4iscnCkklkVMOgCUIpmPojaoWLlSBFXj8GRHg3HanmX6l4m8btFusfUK6BhxUFkGG4nJ8vuBTGYccbcdXLGgxCHc1EX5TiEoIm+spti+ZzxiL4ci66+0/DNzhWZ2P1x5h8MmxSGA1szi0Nc/HW08Z6Irz5ciiKFC8l9O744Ba9MHowGtR9Tm3FQMDhSPLjNCpecVpFxSJtVoer1rqx7PC3joOBNKULeUTU40qRm7IRc4yN9jJBeD680HQFuqeqqULgAlMqMAwdHptcwnboqPBpN0LPKao6V+O0CzWVdXdCw4jBu5lI5JdK3d3uc/PMcxkxfjPatGmOCv7dkKLohYuOuYezQnujctpn8mZCJQT4d0LF1U4iMxZwlmxASMAhR0THZioPYb8jEuWhY53H49euEPQePYtGabfjk7VA5hkLZGAfLh4wU1AiVYxxUiIN8+ZrVjHEQeBW822XMgorEQZyzmo9cqWWsZHAkzEhSJFFpfwmrmYJoNHEQ9dlNruOgoLXk83UcPBoHKHgK5xwy8WhYzoXyqIQhxMEyJsHC6I3QifD0KIyps1fLKZG1a1ZF6+YNMS10DeYE+qFVs/rYELEH4au34tD2xShW1EPuejYyGsHz1yP6UizE1CJf73bo3dULd8s4iP0uXLyCl19biV9OncWDFcvitSlDUevxR2RMZeJgTh/joHfFkCtHKuoTVjTGQWVXhcoxDm7ual464s/sZNmvr+8YijQWCl4M6akBJQtAyW+6qJpVYU6fVaFvI1Q6ONIsvo4pRgTqWzeEMBQQM0yUVA9TeleFvpxFNF26Kpq+rP+JaYiYeCRUQ6m8KZKvxcEZJDv3H8H+Q99hQfBLzoTJcV9l4qBqcKTCJaeNOB1TviylmeR4q++pgAgpvqyoc1jrOajJOKSJg5pzNqFwQQUvHqUrR6oRB3ETjfZ1TKULQCmajqnX1zE9mk29p7avV+HEr+foFUr3OP9KcbiRkIj+o17DjPED0LDOY7pDsw+oThyMt46D4KJiOqaIq+zrmKrWcVA5q0LR1zGVDo404BgHKX+KuioM95Er4daKVo60LLSlezJDrzEO/5um9B2SXfDEr17Lk+NqOei/Thz2HzqOGfPWwqerF8YM6aGFgVNllImD6KoQ3w3QuTWJcKmKFoASIJWMcRDioGg6pgAsvnSg4q9sVYMjBWcl0zFlD7bOFc7auhRlHAz5dUyVsypMKO7prn99FtMxVayimV4/VEzH1C3j0GKmU++I3O6ceGhWbndVvt+/ThyUE8twAGXioGhwpBQHOUNBwWb5kJHOodNmVah6oenuZtarF10VqjZVXRVmVbMq0leOVHEXb90x2EeuRFeFom9ViPpWwrOg/tVOioOiwZGKuioEBF3GODwbrD9PDRETP8+b42o4NX4dUwuku5VRJQ5cx8FGXak4WJI6ur/jFX6rQlFXhfgzVdnXMVWt46ByAShVsypUfx1ThTj8hz+r7dHyFWdfE7naP/HgjFzt54qdmHFwkrJacdB/5Ly4XFXfqhCxlXVVqMo4mIFUk+5jI2UaQ+XKkWIdBxV/vauJmpbV8XB30/+cFYuDugWgxNcxVdxBkXFQMKtCZVdFfp+O6ZU3Yw0SP8ubsRVaXokUBy2U7lJGlTiIQ6paclrZOg6KuioEC5WDI40oDsq6KhS9zJSJg1zHQY1gi3qnQhwEi/iE24pIq/k6pupZFSrGOOjWVdHatmqwk6+Le9o9cb/tY4v3tKMLClMcnIR887btbzSR7U6bzGaZ35f5LwrLb61F5FsxbTqgZX+xarv4kXw52IUTYyWtq76m7yN/JmKkl7Mc0f4s0oa82QYAOoiD9YRsf3U7nLX8vf18xbRIZnG+Ms3vOJfRmnGwnJ88NdvZWM7EQs3SQyCLpx9Yxs54X+w+cuVI2J51+l7pnGy/ST9ny8/tfp+G33ZW1iPbhbVcq+2kHHk6ErCduDXjkN4dYrlPEmeGGaCOhB2vP6t7KuqGZTiC3S3MpjbboqeVtd1DeR52M0ssXRVZ1iO5syWC5Rzt/t/uRDIyEf8vF4Cyv7EZmkmW12HfYETlt1ZO2xGS7qR/psyuDciilrZhuWLrzxzvnyWofRuxRJfiYF9Jrc01rZVa66IdG4dHgKV9O9yZrAZHOrZYeU7yJLKiIlugjGg7A9ujQqwcabmvlnN3uB9299uh/VmuLasKnZ5xyOq+WlnnVBHtGdk9JcW5iiWnLc+VtEuz3Q3blaZddYYwNrJZHF+XMQ7PzXXyLZG73RM/nZy7HV2wF8XBSchXE1PTa3KGJuXw4LK92O08IXOFt2vFDtGsT/aMr2lHRcnyGZN+FPsXQZGCmT9vbP/7LM9RI6czlxKsJdPasf1DzvZwsxbK4r1v+Z1lfxGlUkkPm6JYLcP2FnII4yAgab/J9Pv0d5BA+2NkvPUGOT57bDfRUVbsXvv2FpDhQV+1rGda3AxWYJE/eTczWFJGYbLIovV9mX5z7vNwS3+J2xlI+nEyI7Vn4MjD/tzEb1LtuoTkNacHs78XmRg5vL6yuMfpp5hpVL795VsZZfFDu+vKKFzi/y/E33J4t9vutu3da5MIW+3ORMVePNLBxIqvWGa4gbY6nX637EE51jS71pd+Vum7fBMVl/Zyz1g9MjbE9AiZmdu3UgugNFNqXaWMXWtND2irdLZzchCx7DzFRqlCibQPClo2+3qWdl/sXvHWNmqnAPaVWP6VYLtTnoXTurEcRMxyppZLyPQMSi+d3kwtdcO+uXkWzNiiND7I7Ip5tJt/7zvpsEfinok6RFETguLgJFcpDko2h1eFfkcwpf/lp19Eh0hnLiboPmBOUW6dAAAgAElEQVRANP2KD3joHleeuFUcdAZiBqqWK6pzUNszv5iHmil3KRYR0vnMxT1UMZ1PtJKL8Yn6r+CVfv1XbtzWfWqjOOdvL8Qp6qowoXWV0jrfvTSpKF+iiO4s0psgihYW4zL0f+bpIg7tF+rOU0vAxN3jtRTLkzIUByexqxMHe6938iTtdhdNU2QcnPfwrM/JPuOg21mbgYqlPJSd80/n4vV/ZJmBR8sV1T9uOtT7xINWwU1MUTgI1U3R9FT7jINudS79NSZXeFSwfXMhLeOg/2aCV5XSKqoGyikSB9FIPIvkY3F4Pm++Upm4K2++yqmlTlIctFC6Sxm14qC/gYtLEeMQFLxzJCUl4iAyDiXVicOP50RXhc6bzDikd1XoHFqEu08+aPXezPIjpGpqXdoHo1Rs0fG3VISVMdO6KvTdBN+jysQhY1eFfueuVhwyd5/qcea6ZBw6LdXjVO45RuLHo+55H1ftQHFwkrQ6ccg8nsHJU03bXXRViM8bq3mG4+ylBCUvnkolRf+qmpNWIg4AHi2rLuNQrIiCrgoxVVdVxiG9q0LFHTSaOIhm+M2Fq0pS8yK24xgHXZ4asumVK66mq0Koqkdh/euzELSieoxx6LxcJ4j3Fibxo5H3toMLS1McnIStThzEiSn420/lGAczcOaybXCkk2gddheDI1VtasTBhKplPFW5jqKMg1pxUJJxMAPR11RlHMy4ciNZf101AUfPqxrjIMRBjHHQV9HEkyjj4Ei92qN8wRdWkHEwAZ7i0/ZObh5dVzoZIXe7J+7wz92OLtgrX4vDnZQU1G09BPsjFqB8mZIuwJH9IT7edxgh89fj1ZeHol3LRtaCysTBOlJb58tWnHFQ1VVRSdXgSABqxAGoWlbd4Mj7PPTvqsg4q0Lnmqemq0KhOAgecTeSdH8Jp2Uc1IxxEI8Nr6pllPzNUb54Eb19JK2KyTEOCsQBgC5dFd1W690UNMVL3O6nqVxeFKI4ZKCekpIKNzGp2G5b/95ufPfjKVy+chWDfJ53jThY59vrWy3Ew1Cs4KfzHySW9o+zCmZViAdLpVKqMg5m/Pj3NSU8VImDeDmIrgq9N+XiIKb/Of8HoONlm4EL12zTMfVloijjoLSrQs2sCnHfpDgo2sR0TN03vTIO3dfqfmpaAia+P0RLsTwpY1hxEAvVhC7bjM++PI5UsxmN69XArMmDsXBlBJJuJ2P6uBcl0PjrCWjVY5zMWsRfS0BQ2JuIiY2Hp0cRBI7ph/pPVsevv5/F9NC1qF6lMi7GxGF9uOOKXSf/PIfHH30QQyfOg3eXVi4TB+sKP7pWDROKyH4/vZ/gaSd55tJN/btYVIqDmI4pBkcqwFG1TFElcQVnNbMqzEhJX0tJ1yqX/lelklkVCjMO4k/h2BvJuqMQAZWNcTADrUXGQedN5XRMwdlTTsfUezNBj892ePZcp/eJaYp3c+tgTeXyopBhxeHA199LSYhYHSKf+b2Hh8CvX2c88mA5+E+ejwPbwuHu5oYde77Czn2HsWpeAHoMnQmfF7zQq1NLnDh5BqOnLcKnW8JwJjIafUfOwiuTh6CDV5Ns78OQCXNdJw6quirEx4ZUiYPKMQ6quipUioOqrgplsyqAFGWfLwfcVGQcMiwApfdDVMU6DlIczsepkUpzWleFAg9OyzioCCyyoIUVzPTSKeNQzHu93tVKU7wb7/lqKpcXhQwrDiLjkHgrSWYOxBYcth4Vy5fCsP6d0WnAVEwZ1RfNG9eWcuDVvAGaNngCnQdOxdFdK60L0Xj7B2PSCB/cX6IYevuH4NjuVXddpMal4iCvSsHgyPR1HFRUNnG2yroqxOBIBQ8tcc4//a0o46BIHFR1VYj6pnQ6phJxMONCfJKKqiGbiBJxMAFy5UgF9Vk8MlRkHAQLsQCUqk1FV4Vesyru671B1WXfNe71dwfmyXG1HNSw4hB79TrCVmzB6b/Py3Vbz0fHoG+3Nhg+oAuWb9iByKhLmDlhIFr2GIu9W8IQef4S+oyYhXJ2gyyFeMwcPxBVH64Av4AwHNwWfldmLhcHy8cItNzJeyiT1ZLT97D7XYuqWDlSHNBosyrEO6GKZclpveDaxVE1HVMsOa1GVw24joMpfR0HBUBUDY4UVUTFdEzVsyqEOOjtUXqJQ/E+GxW04JxDXts8IOdCeVTCsOIgMgy3k5Nl94IYzDhj7jpUKl9aisO5qIvo6ReEoIm+spti+ZzxiL4ci66+0/DNzhWZUP9x5h8MmxSGA1vzkTgo6qqQgyOz+FaFXvVPiTjIMQ4GW8fBiEtOK17HQd10TGMtOS0yO9+cv6om46BIHNSu46Coq0KnWRUl+r6l1+PxnuLEv5M2Ti8/boYVh3Ezl6JerWrw7d0eYvDimOmL0b5VY0zw95acRTdEbNw1jB3aE53bNpM/EzIxyKcDOrZuCpGxmLNkE0ICBiEqOib/iYM4YVUZB0WzKsQpK1kASvXgSPmRK/03VbMqxJmqyDjIWRWqxjiY0zMOev9ZCUDpAlAJt3XvMZQrR6oa4yDEQcV0TJULQJkBjyL5N+PwQP+39X84aIgYt6mfhlJ5U8QQ4pDx4zhvhE6Ep0dhTJ29GgXd3VC7ZlW0bt4Q00LXYE6gH1o1q48NEXsQvnorDm1fjGJF06bynY2MRvD89Yi+FAuTyQRf73bo3dULOWUchHD8eTYKd+6kwK1AAZgKmBA6bRjatWwMZes4WOuDznlSE1DYXf9GKk/XiIMjFa3jkNZVoWgdB1WDI82QM5R0rnGyaij9yJVYAErFSStaclrAkIMjFW1eVVQMjjSjfAlFU6LFOg7ya6/6A9FjHYeSA97R/8Q0RIzd2Neh1KFvfsLsxZvksgB1a1VD6DR/lC5ZIlOk3QeOYtmb25F8JwXly5aUfxw/XLmchiNqL5KvxUH7ZWQuuXP/Eew/9B0WBL/kTJgc91UrDgpakkpxENMxVazjIMc4qOmqkIMjVXyrQvGS02q+VWHQlSOvG0wcVM6qgKKPXClex0HJrAqduipKDdyc43tARYErG/pYw167cRPt+0xCWNAINKpXE+GrInDh0pVM77dLMVfRxTcQEauC8WDFsti07VN8+sUxbFg0VddT/FeKw42ERPQf9RpmjB+AhnUe0xVYxmDKxEHRGAdx/kUUdlUYbh0HRRkHwVnZtyrMgBFXjhSZQwU9Feq6KsTgyOv6f+RK1A3xkSsFfxbIx5OKwZEirtqPXOXfr2OWHrRF6Tsku+Axb/pYfyWyCO/v+kIuKyC26zdu4tnuY3Hk4+UoVKigtdyxH09h1sKN+HD9a/JnIps+eHwoDn2wRNdr+NeJw/5DxzFj3lr4dPXCmCE9dIWVVTBl4iAPpubRompWhbLpmCLjoGgdB3UZBxOqylkV+m9yOqbRPqsNdes4RMerGxwZq2rJaWVjHBRlHBSKg2iDKmZViJanR1dF2cHv6d+INUS8tC5tvJ7YVr71Ea7ExSNwTH/rz4Q4bFwc6NANIf5o7vjiFKycOxE1qj0k9/vrbBTmzhiu4Yjai/zrxEH7petTUp04iOYk0w76nKglivzIlVhSW/+//aQ4XFLwkSszULGUqs9qm/HTuWt6U5a0lWUcFA2OFOes6uuYSsc4qPqstqqMg+qPXKkYHKlQHOQLXnyrQudHnXjG6bFyZLmhEfo+gzVGu7iml7WkGK8nvt0UMLy39WfP+QRg8SujUbP6ww4RP9z7FWaErkPRokVQpHAhuRLyQ5U4xkEjdtcUUycOqmZViJUjHb/FoScpZV0VihaAEtcuxjjo/syiOGSqVkqmY8KM6GtJCl46aacfe0NNV4WqdRzkR66q6L/ktPpvVeTfroryw7bq+YjUHCt6VU9r2VWbPsKFi1fkEgOW7elOI7HljSCHjIOYYTh6+mIpC2J5gj0HjyJ89TZ8vHFOpm8waT6RLAoy4+AMPUDdrAqZEDDr/0BUODgyLeOg/7cqBIqKigZHCsA/nrvmZC3IvLvqWRUqpmOqzjiI2Ugq8lwqV46MVTAdU3BWJQ4itqoxDsqWnM7nX8es4L9N9+eDloAXVtq62vd+fgxvv/+pdZCjmFnRvu9kOcahYEHbdz7EbMKfT57GvBkjrIeo12YIPnlnHiqU1e8L0xQHLXfwLmXiElMdHoaygyHbHgb7v2sdH6GW/cxmWyeCmBgnY6U7hLXrwhI/w38tnRuW4va/tr+EtDEOQkrSvlaYVaeIXEJC/Fr+0j6y7XSyegnYf1bbAUPGP+nTd86IyjomNMPvK4qMQ/pmOTfxvybL2VvPM+2nmbb0S3C4nPSf/ZhhyemM52CNZcWQftZ2WGz7CKBpF1u1nBjjYH8D0/+dBdPMVcburphNsh7Y36e0j1zZ7qH1l/YXnkWFSPuRPYz03rD0E0hJP/cs70vGrrMM1cKhqmZqCCa4FXCoVI63yO6U5B20u7RMl5ThuPbrOGRXUzP/3BbVgYl9c5Of1b5t6zHMeNvTb3XGdmZ/n6y/ywA04+DIbOucte3b9Vymf2RU3qosqroQB/vnieTpcHPse0GzuvqMbNJOwn46pqU5p9Ul8WuT5XGSublYfmJ3vtZ/pv/Dfslph8vKCq5dhUgrm/agsj47LfdFp29VVBqxPfPzxAU/iVrRzXqUhJu30L7vJIRO90ejujXw+tJ3cONmopySefrcBURduIwWTergq29/lh9yFLMqHihxHw4f+wUTZy3HF9sXy2836bVRHJwk+ea355yMkP3uraqWVRK7hIdtFK7eB1h46LTeIWW86qXVrZOvYryHOOd2j+nbr2gP1qOQfg8B+7jRV5OU3D8RNDklVUnsynZSqfcB/rx4Q++QMt6jZYspiSuCWiVB5yN8+VeMzhFt4ZpWKaUkdklP59tJ5ZEfKDm3nIL+s/wFhyJHjv+KkPkbcPlKHJ4S8hA4TH5n6d0dn0FkJNYumCzLr377YzkDQ4jUfcU88fJLffBU3cdzOtw9/Z7icE+4MhemODgyoTjYeFAcHOsGxcHGg+LgWDfyszg8OGqHk2+J3O0eubRr7nZ0wV4UBychUxwoDtlVIYoDxSG7ukFxMI44PDTmQyffErnb/dziLrnb0QV7URychExxoDhQHLQ1ImYcmHHIrqbk54zDI2M/1lbBdS51dlEnnSPqF47i4CRLigPFgeKgrRFRHCgORhSHKuN3aqvgOpc6s7CjzhH1C0dxcJIlxYHiQHHQ1ogoDhQHI4pD1Qm7tFVwnUudXvC8zhH1C5evxUGslFW39RDsj1iA8mX0m4N6r/jEkp3B8zfg1F/n5NfIAkb4wOt/9WUYigPFgeKgrUVRHCgORhSHagGfaKvgOpf6M6yDzhH1C0dxyMAyJSU10wpbXQdNQ8+Oz6Jf9+fkPNkJwUvxxfYl8ChSiOKQgR9nVdiAcHCkY+WgOFAcjCgOj03erd8b9x4i/T63/T2Udm1Rw4qD2WxG6LLN+OzL40g1m9G4Xg3MmjwYC1dGIOl2MqaPe1GSjL+egFY9xsmsRfy1BLk4RkxsPDw9iiBwTD/Uf7I6fv39LKaHrkX1KpVxMSZOLtdp2UTWY/snh9CtQwvrAhpNOo5AxKoQPFSpLMWB4pBti6U4UByyqxycVeFIJj8Pjnz85bwRh1OhFIdc6dDduioOfP29lISI1SFy8bTew0Pg168zHnmwHPwnz8eBbeHyRb9jz1fYue+w/Bxpj6Ez4fOCF3p1aokTJ89g9LRF+HRLGM5ERqPvyFl4ZfIQdPBqctdzPfHbaYyduQT73l0A8ZlgdlWwq4JdFdqaNzMOzDgYMeNQc+oebRVc51K/zWmnc0T9whk645B4K0lmDsQWHLYeFcuXwrD+ndFpwFRMGdUXzRvXlnLg1bwBmjZ4Ap0HTsXRXSvlC19s3v7BmDTCR66+1ds/BMd2r7L+LivE/1y4jGGTwjBj3AA8/VQtWYTiQHGgOGh7IFEcKA5GFIda0/Zqq+A6l/rltbY6R9QvnGHFIfbqdYSt2ILTf5+Xa6yej45B325tMHxAFyzfsAORUZcwc8JAtOwxFnu3hCHy/CX0GTEL5ewGWQrxmDl+IKo+XAF+AWE4uC08W7Kn/orE2BlLpJC0bFbPWo7iQHGgOGh7IFEcKA5GFIcnp3+qrYLrXOrnV5/TOaJ+4QwrDiLDcDs5WXYvuLkVwIy56+RnRIU4nIu6iJ5+QfITpKKbYvmc8Yi+HIuuvtPwzc4Vmej9ceYfmUk4sDVrcRDSIcRi9lQ/NKhd3WF/igPFgeKg7YFEcaA4GFEc6szcp62C61zqp1ltdI6oXzjDisO4mUtRr1Y1+PZuD/EN8jHTF6N9q8aY4O8t6YhuiNi4axg7tCc6t20mfyZkYpBPB3Rs3RQiYzFnySaEBAxCVHTMXcXBd9zr6N2lVZbjHygOFAeKg7YHEsWB4mBEcagXvF9bBde51A/BrXWOqF84Q4iDZUyC5bLfCJ0IT4/CmDp7NQq6u6F2zapo3bwhpoWuwZxAP7RqVh/iu+Thq7fi0PbFKFY07ZPMZyOjETx/PaIvxcJkMsHXux16d/XC3TIOYlxDuz6THL55LmKFzRyBNi0acoxDhrrI6Zg2IJxV4Vg5KA4UByOKQ/2Qz/R7495DpO+DvO6htGuL5mtxcAbFzv1HsP/Qd1gQ/JIzYXLclxkHZhyYccixmcgCFAeKgxHFoeEreSMO382gOGh7suhU6kZCIvqPeg0zxg9AwzqP6RQ16zAUB4oDxUFbE6M4UByMKA5PvXpAWwXXudSx6a10jqhfuH9dxmH/oeOYMW8tfLp6YcyQHvqRyiYSxYHiQHHQ1swoDhQHI4pD49kHtVVwnUsdDWypc0T9wv3rxEE/NNoiURwoDhQHbW2F4kBxMKI4NH39c20VXOdSR6Y8q3NE/cJRHJxkSXGgOFActDUiigPFwYji8HToF9oquM6lDr/8jM4R9QtHcXCSJcWB4kBx0NaIKA4UByOKw//mHdJWwXUu9dWkFjpH1C8cxcFJlhQHigPFQVsjojhQHIwoDi3mf6mtgutc6tDE5jpH1C8cxcFJlhQHigPFQVsjojhQHIwoDs8syBtx+GICxUHbk8WApa4k3FF21mazmtCFCxZQExjAqfM3lMR+vGIxJXFF0KTkVCWxixRyUxJXBD1xLl5JbK9e05XEFUF3vB2kLLaqwLUrlVAS+pNT0UriiqDPVSunJHZCUoqSuCLoI2U8lcQu4u582JbhXzkfJBcRDo77Xy72cs0uzDg4yZni4AiQ4mDjQXFwrBsUBxsPioNj3cjP4uC1+Gsn3xK52/2zMWmfSsiPG8XBybtCcaA4ZFeFKA4Uh+zqBsXBOOLQZslhJ98Sudt93+inHXY89M1PmL14Ey5fuYq6taohdJo/SpfMnBG7cPEKpsxehV9OncGDFcvi1ZeHotbjj+TuJLLZi+LgJE6KA8WB4qCtETHjwIxDdjUlP2cc2i47oq2C61xq70tNrRGv3biJ9n0mISxoBBrVq4nwVRG4cOlKlp9UGDBmtvxeU/+ebbFt5+f4/uc/pGTouVEcnKRJcaA4UBy0NSKKA8XBiOLQfvk32iq4zqV2j2xijbj7wFG8v+sLrJoXIH92/cZNPNt9LI58vByFChW0los8fwlCHPa/txAZPw6p5+lRHJykSXGgOFActDUiigPFwYji8PyKvBGHXSNs4rDyrY9wJS4egWP6WxEKcdi4OBAPV7YNhhWfXNj0/l7ZRfH1tz+jcsWy8ptNjz5cUVsj1VgqX4vDnZQU1G09BPsjFqB8mZIaL0n/YidOnsGrCzfi9LkLKF/mAUwc3hstm9WTB6I4UBwoDtraHMWB4mBEcei08qi2Cq5zqY/9G1sjhq/eCvE+DBje2/qz53wCsPiV0ahZ/WHrz7Z/cgizFm7EG6ET0LheDWza9qnMVGxf96quZ0dxyIAzJSUVbm626YpmsxmtvSdgvF8vdHruaRw8/AMmzVqBrz5chsKFClIcMvDjrAobEA6OdKwcFAeKgxHFocuqb3V96WoN9uGwRtaiqzZ9BDHoMWiir/VnT3caiS1vBGXKOCzf8AG2rZkly6WmmtGg7VAc2BaOB0rcp/XQOZYzrDiIF3ross347MvjSDWbpV3NmjwYC1dGIOl2MqaPe1FefPz1BLTqMU5mLeKvJSAo7E3ExMbD06MIAsf0Q/0nq+PX389ieuhaVK9SGRdj4rA+fIoV3K2k29hz8Ft0bWebU9ugrR8+3DAblSuUoThQHLJtZBQHikN2lYOzKhzJ5OfBkS+sOZbji1RFgQ+GPmUNu/fzY3j7/U+xYdFU+TMxs6J938lyjEPBgrbFKk79FYmXpi7EvvcWyHLiD2HxvvpyxxLcV0y/tTIMKw4Hvv5eSkLE6hCYAPQeHgK/fp3xyIPl4D95vjQsdzc37NjzFXbuOywHlfQYOhM+L3ihV6eWEN0Po6ctwqdbwnAmMhp9R87CK5OHoIOXrV8pY2VITr4j0z6bP/hMGp3ITLCrwpESMw7MOGT3EGXGgRkHI2Ycuq/9ToUX5Bjz/SENrWUSbt5C+76TEDrdH43q1sDrS9/BjZuJcraE6EKPunAZLZrUkeW7DZ6OF3u2RbcOLfDW1r346NPDiFgVnOPx7qWAYcVBZBwSbyXJzIHYgsPWo2L5UhjWvzM6DZiKKaP6onnj2lIOvJo3QNMGT6DzwKk4umuldbSpt38wJo3wwf0liqG3fwiO7V6V7UhUISqjpy1GudIPIPyV0ahdo4o8LsWB4pBdg2PGgRkHZhy0vY7yc8ah57q8EYetg23iICgeOf4rQuZvwOUrcXhKyEPgMPnuenfHZxAZibULJkvYv5/+B4FzVuOfC5floEiRiefgyPR6GHv1OsJWbMHpv88DJhPOR8egb7c2GD6gC5Zv2IHIqEuYOWEgWvYYi71bwiCmqfQZMQvl7AZZCvGYOX4gqj5cAX4BYTi4LfyutVwMTvn2+5N4+bWV2LJiJiqWL01xyECMGQdmHJhxyPllya4KR0b5WRy81x/P+YYqKPGebwMFUfUJadiMg8gw3E5Olt0Lostgxtx1qFS+tBSHc1EX0dMvSA4kEd0Uy+eMR/TlWHT1nYZvdq7IRO6PM/9g2KQwHNiaWRyuxF3D4WO/yIGRls133Ovw7twKz7duQnGgOGTbEplxYMaBGQdtL6r8LA4+G77XdhE6l9oysL7OEfULZ1hxGDdzKerVqgbf3u1x8s9zGDN9Mdq3aowJ/t6SjuiGiI27hrFDe6Jz27Q1v4VMDPLpgI6tm0JkLOYs2YSQgEGIio7JVhzE4Mo23hPkCl2iD0kMPhELbGxaOk0OpmRXhWNlZMaBGQdmHHJ+QDPjYJyMQ9+3fsj5hioo8c6LaVP+8+NmCHHIuALWG6ET4elRGFNnr0ZBdzfUrlkVrZs3xLTQNZgT6CeX29wQsQdi7uuh7YtRrKiHZH82MhrB89cj+lIsTCYTfL3boXdXL9wt4yD2E2uEL1j5Hs5fvIL7ixeT4yh6dHxGxqQ4UByya9jMODDjwIyDttdefs449N/0o7aL0LnUpv51dY6oX7h8LQ7OXObO/Uew/9B3Wa7l7UzcjPtSHCgOFAdtLYqzKmycmHEwTsZhwNt5Iw4b+1EctD1ZdCp1IyER/Ue9JpfabFjnMZ2iZh2G4kBxoDhoa2IUB4pDdjUlP2ccfDf/pK2C61xqfZ+06ZX5cfvXZRzEWt0z5q2FT1cvjBnSQzlzigPFgeKgrZlRHCgORhSHwVtOaKvgOpda51Nb54j6hfvXiYN+aLRFojhQHCgO2toKxYHiYERxGPrez9oquM6l1ng/qXNE/cJRHJxkSXGgOFActDUiigPFwYjiMCziF20VXOdSq3rV0jmifuEoDk6ypDhQHCgO2hoRxYHiYERxGL41b8ThjZ4UB21PFgOWojhQHCgO2houxYHiYERxGPn+r9oquM6llnd/QueI+oVjxsFJlpeu33EyQva7F7B93VvXY3gWctM1nn0wVQtAeRZWd87JKalqeJjVhBVRy92f9o0WvbciBRVVOgBmRTwSk1P0xmCNV6yw7cuDeh7k8rUkPcM5xEq4rYZHpQfU1Dlx8oXd1bRvj4LOYx69/Tfng+QiwpJuNXOxl2t2oTg4yZni4AiQ4mDHQ9GLkuLgWOcoDo48KA42HnqIw9gdeSMOi7pSHJx8Peff3SkOFIdsayfFwQENMw42HMw4OLaa/JxxmPDhyTx5AS3oUiNPjqvloMw4aKF0lzIUB4oDxUFbI6I4UByyqyn5WRwCPsobcQjrTHHQ9mQxYCmKA8WB4qCt4VIcKA5GFIeXd57SVsF1LhXa8XGdI+oXjhkHJ1lSHCgOFAdtjYjiQHEwojgE7vpdWwXXudTs59V+LsGZ083X4nAnJQV1Ww/B/ogFKF+mpDPXqcu+V+Nv4PkXX8bYIT3kVzXFRnGgOFActDUvigPFwYjiMH133ojDq+0pDtqeLBlK5YU4pKSkws0t6ylpgXNW4+gPJ+HXtyPFIZs7ylkVdmA4ONKhllAcKA5GFIegPX/k6v3l7E4h7ao7G0LZ/obNOJjNZoQu24zPvjyOVLMZjevVwKzJg7FwZQSSbidj+rgXJbT46wlo1WOczFrEX0tAUNibiImNh6dHEQSO6Yf6T1bHr7+fxfTQtahepTIuxsRhffiUTMCPfn8Syzd8gGqPVEL1KpUoDhSHnBslxYHikE0t4awKRzD5eXBkyN68EYegthSHnB+yWZS4W8bhwNffS0mIWB0CE4Dew0Pg168zHnmwHPwnz8eBbeFwd3PDjj1fYee+w1g1LwA9hs6Ezwte6NWpJU6cPIPR0xbh0y1hOBMZjb4jZ+GVyUPQwatJpjNJTr4Db/9gzA9+Ce+8v4/icJe7yYwDMw7ZVQ9mHJhxMGLG4dV9f+bq/eXsTtPbVHM2hLL9DZ1xSLyVJDMHYgsOW4+K5UthWP/O6DRgKlRc8aMAACAASURBVKaM6ovmjWtLOfBq3gBNGzyBzgOn4uiulShQQKgGpAxMGuGD+0sUQ2//EBzbvcr6O3viy9d/AJHheGlQN7wa/hbFgeKgrUEy48CMAzMOmtpKfs44zN6fN+IQ2JrioKnyZCx0t4xD7NXrCFuxBaf/Pg+YTDgfHYO+3dpg+IAuWL5hByKjLmHmhIFo2WMs9m4JQ+T5S+gzYhbK2Q2yFOIxc/xAVH24AvwCwnBwW3im8zwbGY2JIcuxefkMFCpUkOKQw51kxoEZB2Yccn7csavCOF0VoQf+yvmGKijxcqtHFUTVJ6RhMw4iw3A7OVl2L4jBjDPmrkOl8qWlOJyLuoiefkEImugruymWzxmP6Mux6Oo7Dd/sXJGJ3B9n/sGwSWE4sDWzOKx/bzdWbvwQBQumrVmfcPOWPJ6QlHF+PTmrIgNNigPFgeKQ88OZ4mAccZh3MG/EYVJLR3E49M1PmL14Ey5fuYq6taohdJo/SpcskW1lE+PyBo1/HR9tnIOqD1XIuVLeQwnDisO4mUtRr1Y1+PZuj5N/nsOY6YvRvlVjTPD3tnZDxMZdw9ihPdG5bTP5MyETg3w6oGPrphAZizlLNiEkYBCiomOyFYeMLNlVcffaRXGgOFAccn4CUxyMIw4LPj+d8w1VUGLCs1WtUa/duIn2fSYhLGgEGtWrifBVEbhw6QoWBL+U5ZFv305Gn5GvSMlYv2jqf1McLGMSLITeCJ0IT4/CmDp7NQq6u6F2zapo3bwhpoWuwZxAP7RqVh8bIvYgfPVWHNq+GMWKeshdRbdD8Pz1iL4UC5PJBF/vdnJ2xN0yDhSHe2sRFAeKA8Uh5zZDcTCOOIQfyhtxGNfCJg67DxzF+7u+kIP8xXb9xk08230sjny8XHahZ9yWvbldfo127xfHED5r1H9LHHJuftmX2Ln/CPYf+i5bI3Mmtv2+XADKkSTFgeJAccj56UJxMI44LPnyTM43VEGJ0c2rWKOufOsjXImLR+CY/tafCXHYuDgQD1cu53B08QeyyMi/tzIIPYcFUxy03psbCYnoP+o1zBg/AA3rqF19i+JAcci2XnJWhQMaTse04aA4GEccln99VuurR9dyI5s9Yo0nsudiskDA8N7Wnz3nE4DFr4xGzeoPOxx38PhQ+A/ogib1a6KL7zSKg5a7sv/QccyYtxY+Xb0wZkgPLbs4VYbiQHGgOGhrQhQHikN2NSU/T8d843DeiMPwp23isGrTR7hw8Yoc8G/Znu40ElveCHLIOHyw+0sc+/EUXn15iCxGcdD2bHJ5KYoDxYHioK3ZURwoDkYUh9VH/tZWwXUu5dfUlknY+/kxvP3+p9iwaKo8ihj02L7vZDnGwTLjT/x89PTFOH7id7gVSPtswtVrN3BfMU+89vJQtGxWT7czzNezKnS7SoWBKA4UB4qDtgZGcaA4GFEc1h7NG3EY0tgmDmIZgPZ9JyF0uj8a1a2B15e+gxs3E+WUzNPnLiDqwmW0aFInE15mHLQ9m1xeiuJAcaA4aGt2FAeKgxHFYf2357RVcJ1L+TZ6yCHikeO/ImT+Bly+EoenhDwEDpOrHr+74zOIjMTaBZMpDjrfA2XhKA4UB4qDtuZFcaA4GFEcNh6L1FbBdS414KkHdY6oXzh2VTjJkuJAcaA4aGtEFAeKgxHFYdN3eSMO/RtSHLQ9WQxYKurqbWVnXdg9bYCL3pv4DLmqrViRtKW5uQGp6jDDzZT2oTa9N5V1Iy4hWe/TlfEUoZCxCylqg+7pH9pTAcRNUewtP6h7gfZr4JiW14tL0ULOt5N3vvtHr9O5pzh9G1a+p/KuLMyMg5O0KQ6OACkONh4UB8e6QXGw8aA4ONaN/CwOW45HOfmWyN3uPg0q5W5HF+xFcXASMsWB4pBdFaI4UByyqxsUB+OIw3vf5404eNenODj5es6/u1McKA4UB23tkxkHZhyyqyn5OeOw9Yfz2iq4zqV61quoc0T9wjHj4CRLigPFgeKgrRFRHCgORhSH93+8oK2C61yqe119P4Wt5+lRHJykSXGgOFActDUiigPFwYji8MFPeSMOL9ShOGh7smQoJT7qUbf1EOyPWIDyZUrmKoYeO/mMmIWTf/xtHb5dvJgnvti+WIamOFAcKA7aWhnFgeJgRHH48ES0tgquc6kutcvrHFG/cPk645AX4pCSkgo3N8dpkB1fnIJFs0ajWpXMg1UoDhQHioO2BxLFgeJgRHH4+MRFbRVc51Kdajt+Llvn8E6FM6w4mM1mhC7bjM++PA4x97xxvRqYNXkwFq6MQNLtZEwf96IEE389Aa16jJNZi/hrCQgKexMxsfHw9CiCwDH9UP/J6vj197OYHroW1atUxsWYOKwPn+IAVXz3/N2VQVlmPSgOFAeKg7ZnEMWB4mBEcdj18yVtFVznUs8/WVbniPqFM6w4HPj6eykJEatDIJb46D08BH79OuORB8vBf/J8HNgWDnc3N+zY8xV27juMVfMC0GPoTPi84IVenVrixMkzGD1tET7dEoYzkdHoO3IWXpk8BB28mmSiW7+tH55pUkd+dazkA8UxYZg3nn26rixHcaA4UBy0PZAoDhQHI4rD7l/yRhza16I4aHuyZCh1t64KkXFIvJUkMwdiCw5bj4rlS2FY/87oNGAqpozqi+aNa0s58GreAE0bPIHOA6fi6K6VKJC+spq3fzAmjfCRHwrp7R+CY7tXWX9nOZXUVDNmzF2LNs80RPPGdfDl0Z8w+ZU38OGGOahQtiTFIcM94wJQNiBcx8GxclAcKA5GFIe9v17O1fvL2Z3aPlHG2RDK9jdsxiH26nWErdiC03+fl4MWz0fHoG+3Nhg+oAuWb9iByKhLmDlhIFr2GIu9W8IQef4S+oyYhXJ2gyyFeMwcPxBVH64Av4AwHNwWrgn04PGh6P78M+j03NMUB4pDtnWG4kBxyK5ycAEoRzL5eR2Hfb/FaHov6F2oTc3SeofULZ5hxUFkGG4nJ8vuBTGYccbcdahUvrQUh3NRF9HTLwhBE31lN8XyOeMRfTkWXX2n4ZudKzLB++PMPxg2KQwHtmYWh5uJSfj9dCTq1apm3W/AmNno1/05tGvZiOJAcaA4aHwcMePAjIMRMw6fncwbcfCqQXHQ+GhxLHa3ropxM5fKl7lv7/Y4+ec5jJm+GO1bNcYEf28ZRHRDxMZdw9ihPdG5bTP5MyETg3w6oGPrphAZizlLNiEkYBCiomOyFYdrN26ida/xWBgySnZ9fHn0BCbNWoGP33odpR4oTnGgOFAcNLZuigPFwYjicPDUFY01XN9iLR8vpW9AHaMZIuNgGZNgue43QifC06Mwps5ejYLubqhdsypaN2+IaaFrMCfQD62a1ceGiD0IX70Vh7YvRrGiHnLXs5HRCJ6/HtGXYmEymeDr3Q69u3rhbhkHsd+hb05g3ootuHg5FpUrlMHkl/qgSf2aMiYHRzrWRo5xsPFgV4Vj3aA4UByMKA5fnIrV8ZWrPdQzj+fd2kU5nWW+FoecTv5uv9+5/wj2H/oOC4JfciZMjvtSHCgO2VUSigPFIbu6wTEOjmTy8xiHL3/PG3Fo/hjFIccXsJ4FbiQkov+o1zBj/AA0rPOYnqEzxaI4UBwoDtqaGDMOzDgYMePw9R9x2iq4zqWaVX9A54j6hfvXZRz2HzqOGfPWwqerF8YM6aEfqWwiURwoDhQHbc2M4kBxMKI4HP7zqrYKrnOpp6vdr3NE/cL968RBPzTaIlEcKA4UB21theJAcTCiOBz5K2/EoemjFAdtTxYDlqI4UBwoDtoaLsWB4mBEcTh6Ol5bBde5VOOqJXSOqF84ZhycZElxoDhQHLQ1IooDxcGI4vBtHolDI4qDtgeLEUtRHCgOFAdtLZfiQHEwojgcO5M3GYenqjDjoO3JYsBSN26blZ11zLUkJbFLFiukJK4IahJfHFOw3b6jjvOd1FQFZwzEJyQriSuClr8/7Rstem9Jd9SwEOf5a9Q1vU9Xxqv9oLoHbLIiHp6F3ZWwEEHFd3xUbMkpauKKc425ruZZ93h5T6dRfHdWTb3N6cQaPlLcocihb37C7MWbcPnKVdStVQ2h0/xRumTmui++GD1/5Xuy3OOPPoiQSYNR9aEKOR3unn7Prop7wpW5MMXBkQnFwcaD4uBYNygONh4UB8e6kZ/F4fjfeSMODR62iYNYwbh9n0kICxqBRvVqInxVBC5cupJpnaKLl+PQxTcQK+dORJ2aj2LJuvfxwy9/4M2FU5x802V4zptV6amup5l/g1EcKA7Z1U6KA8Uhu7pBcTCOOPxw7nqevIDqPXSf9bi7DxzF+7u+wKp5AfJn12/cxLPdx+LIx8tRqFBBazkhDj/99heee+Yp+bPf/vgbLwWG47OIhbpeAzMOTuKkOFAcKA7aGhEzDsw4ZFdT8nPG4cc8Eoe6duKw8q2PcCUuHoFj+lsRCnHYuDgQD1cul20DXLt5F079eQ5zZwzX1kg1lqI4aASVXTGKA8WB4qCtEVEcKA5GFIefIvMm41DnQVvGQXx3SXz0MWB4byvC53wCsPiV0ahZ/eEssYoPMr6ycCPeWjINZUvruyYExUHbMy/bUhQHigPFQVsjojhQHIwoDif+uaGtgutcqnblYtaIqzZ9hAsXryBooq/1Z093GoktbwRlmXH4eN9hrNiwAyteH4+HKmWfkcjtKedrcbjbZ7Vze8G52S85+Q5CFmzA3s+/lV/aFJ/q7trufzIUxYHiQHHQ1qooDhQHI4rDz1F5Iw5PVrKJw97Pj+Ht9z/FhkVTJUIxY6J938lyjEPBgo4zdMSsisVr38ea+ZOynHWhrbXevRTFIQOflJRUuLkVcPjp0nXb8efZKMwJHCb/GzRvHd5ZPgNFCheiOGTgx1kVNiAcHOlYOSgOFAcjisOv5xP0eNfec4wnKha17pNw8xba952E0On+aFS3Bl5f+g5u3EyUUzJPn7uAqAuX0aJJHcRfT0C3wdPl2IfKFcrc8zG17mBYcRCTQUKXbYawq1SzGY3r1cCsyYOxcGUEkm4nY/q4FyUDAbJVj3HYH7EA8dcSEBT2JmJi4+HpUQSBY/qh/pPV8evvZzE9dC2qV6mMizFxWB/uOHWlda8JWLtgMh55sHwmrsw4MOPAjIO2xw3FgeJgRHH4LY/EoaadOAhuR47/ipD5G3D5ShyeEvIQOAz3lyiGd3d8BpGREO+o7Z8cku+yjFmIg1vDZVm9NsOKw4Gvv5eSELE6BGLNod7DQ+DXrzMeebAc/CfPx4Ft4XB3c8OOPV9h577DchpLj6Ez4fOCF3p1aokTJ89g9LRF+HRLGM5ERqPvyFl4ZfIQdPBq4sBWzJ99ptsYOShFpIoKFyqEMUO6w6t5A1mO4kBxoDhoexxRHCgORhSHkxfyJuNQo4It46CthbmulGHFQWQcEm8lycyB2ILD1qNi+VIY1r8zOg2Yiimj+qJ549pSDsRLvmmDJ9B54FQc3bUSBQqkLW/o7R+MSSN8pIn19g/Bsd2rrL+z3IKo6BiZIho9uDuG9u2EEydPY9ikMHy0YY4cqUpxoDhQHLQ9sCgOFAcjisOp6JvaKrjOpfRY9VLnU7KGM6w4xF69jrAVW3D67/NynePz0THo260Nhg/oguUbdiAy6hJmThiIlj3GYu+WMESev4Q+I2ahXJmS1osX4jFz/EBUfbgC/ALCcHBbeCbOIuMgRq9+s3OFHBgptiET5sK7Syu0a9mI4pCBGMc42IBwjINj5aA4UByMKA6/X8wbcXisnPPLZVMc7F74lgzD7eRk2b0gBjPOmLsOlcqXluJwLuoievoFyakropti+ZzxiL4ci66+06QAZNz+OPOPzCIc2JpZHERZIQ6iS8Qy2GTw+FD07/GczGQw48CMAzMO2h5PFAeKgxHF4c9LidoquM6lqpVN+0M1P26GzTiMm7kU9WpVg2/v9jj55zmMmb4Y7Vs1xgR/b8lZdEPExl2TUyc7t20mfyZkYpBPB3Rs3RQiYzFnySaEBAyC6I64mziID4vcTExCcIAvfj11FsMmz8fHG+fIqS4UB4oDxUHbo43iQHEwojj8lUfi8CjFQduDJWMpyzoOljEJlt+/EToRnh6FMXX2ahR0d0PtmlXRunlDTAtdgzmBfmjVrD42ROyBWG3r0PbF1i6Gs5HRCJ6/HtGXYmEymeDr3Q69u3ohp4yDWBc88PU1OPr9byh5f3FMGtGbgyOzuaXsqrCBYVeFYyWhOFAcjCgOpy/nTcahahlmHHJnDk7stXP/Eew/9F2mr4c5ETLLXZlxYMaBGQdtrYriQHEwojicibmlrYLrXKpK6bSB//lxy9ddFbkFdiMhEf1HvYYZ4wegYZ3HchtG034UB4oDxUFTUwHFgeJgRHE4eyVvxOGRUhQHbU8WHUrtP3QcM+athU9XL4wZ0kOHiHcPQXGgOFActDUzigPFwYjicC42SVsF17nUQyUL6xxRv3D/yoyDfnhyjkRxoDhQHHJuJ6IExYHiYERxiMwjcXiQ4qDtwWLEUhQHigPFQVvLpThQHIwoDv/E5U3GofIDzDhoe7IYsBTFgeJAcdDWcCkOFAcjikPU1dvaKrjOpSrdX0jniPqFY1eFkywpDhQHioO2RkRxoDgYURwuxOeNOFQoQXHQ9mQxYCmKA8WB4qCt4VIcKA5GFIfoa8naKrjOpcoXL6hzRP3CMePgJMv4xFQnI7h+d7f0j3ypOLL4xLmKLTlFTVxxroXdC6g4ZSTdUVc3CrqlfahN7+1Wsrpz9iikhrNJfh/XWJuqdiIo3ElV01YUNW154woXVFM3ihVyvm5czCNxKEdxMFajvpezpTg40lL1QKQ4OHKmONh4UBwc6wbFwcZDD3G4dD1vMg5l72PG4V7exYYqS3GgOGRXYZlxcCTDjIONhyrBZsbBsc7pIQ4xN+7kyTupdDH3PDmuloOyq0ILpbuUoThQHCgO2hoRxYHikF1Nyc9dFVcS8kYcShWlOGh7shiwFMWB4kBx0NZwKQ4UByOKQ+zNFG0VXOdSJT3ddI6oXzhmHJxkSXGgOFActDUiigPFwYjiEJdH4vAAxUHbgyVjKctntfdHLED5MiVzF8TJvSLPX0LngYEOUe7cSZFf3Wz77FOgOFAcKA7aGhnFgeJgRHGIT8ybjEMJD2YctD1ZMpTKC3FISUmFm1v2U4Ouxt9An5GzsOWNIJS4ryjFIcM9UzXoi7MqHEFzVoWNB2dVONYNzqqw8dBjcOS1W+qmKN/txVi8iJopqrl6GWfYKV93VdxNHMxmM0KXbcZnXx6HeFk1rlcDsyYPxsKVEUi6nYzp416Ulxp/PQGteoyDyFrEX0tAUNibiImNh6dHEQSO6Yf6T1bHr7+fxfTQtahepTIuxsRhffiUbNm+snAjHn2kEvp2a50Wn+s4OLCiONhwcFaFYzNixoEZByNmHK4n5Y043FeY4pArybmbOBz4+nspCRGrQ+TyL72Hh8CvX2c88mA5+E+ejwPbwuHu5oYde77Czn2HsWpeAHoMnQmfF7zQq1NLnDh5BqOnLcKnW8JwJjIafUfOwiuTh6CDV5Nsz1V0W/gFhOGjjXNQ0D0tjURxYFcFuyq0NW+KA8XBiOKgcnXgu7UcPbIl2lrmvZcydMYh8VaSzByILThsPSqWL4Vh/Tuj04CpmDKqL5o3ri3lwKt5AzRt8AQ6D5yKo7tWokD6yone/sGYNMIH95coht7+ITi2e5X1d1mhnLPkbVQoVwq+3u2tv6Y4UBwoDtoePBQHioMRxeHmbTUrcebUajx1WPUyp2Pk9veGFYfYq9cRtmILTv99HjCZcD46Bn27tcHwAV2wfMMOREZdwswJA9Gyx1js3RIGkS3oM2IWytkNshTiMXP8QFR9uILMJBzcFp4tx+Q7KWjedRQ+WPeqlAfLRnGgOFActD1+KA4UByOKQ2Jy3oiDR0Hnl8vW1jLvvZRhxUFkGG4nJ8vuBTGYccbcdahUvrQUh3NRF9HTLwhBE31lN8XyOeMRfTkWXX2n4ZudKzJR+uPMPxg2KQwHtmYvDt/+cBJzl29BxKpgh/0pDhQHioO2Bw/FgeJgRHHQVrv/W6UMKw7jZi5FvVrV4Nu7PU7+eQ5jpi9G+1aNMcHfW95B0Q0RG3cNY4f2ROe2zeTPhEwM8umAjq2bQmQs5izZhJCAQYiKjslRHNZu3iWzGMEBvhSHu7QRDo60weHgSMeKQnGgOFAc/h2CYQhxsIxJsCB/I3QiPD0KY+rs1XKQYu2aVdG6eUNMC12DOYF+aNWsPjZE7EH46q04tH0xihX1kLuejYxG8Pz1iL4UC5PJBF/vdujd1QtaMg5ifIOIM3pwd4oDxUFT66c4UByyqyiqBFscj9MxbdTz8wBDTQ+RfFooX4uDM8x27j+C/Ye+kws1qdzYVeFIV9UDkes4OHLmOg42HlzHwbFuUBwoDirfeSL2v1IcbiQkov+o1zBj/AA0rPOYUoYUB4pDdhWMGQdmHJhx0Pb4zc8fudJ2Bf+tUv86cdh/6DhmzFsLn65eGDOkh/K7SXGgOFActDUzjnGwcVKVmWNXhWNdZFeFtrZ5r6X+deJwrwCcLU9xoDhQHLS1IooDxSG7msKMg7Y2lF9KURycvBMUB4oDxUFbI6I4UBwoDtraSn4vRXFw8g5RHCgOFAdtjYjiQHGgOGhrK/m9FMXByTtEcaA4UBy0NSKKA8WB4qCtreT3UhSH/H6HeH4kQAIkQAIkkI8IUBzy0c3gqZAACZAACZBAfidAccjvd4jnRwIkQAIkQAL5iADFIR/dDJ4KCZAACZAACeR3AhSH/H6HeH4kQAIkQAIkkI8IUBzy0c3gqZAACZAACZBAfidAccjvd4jnRwIkQAIkQAL5iADFQcHNuJOSkmNUdze3HMtkVUBVbFVxxTWoiq0qLs/ZseaRs2t4kLNrOOfqwcudHAhQHBRUiFotfXOM+svB9TmWyaqAqtiq4oprUBVbVVyes2PNI2fX8CBn13DO1YOXO1EcVNeB6Mux1kPEXb0Ov4AwbF0T4nDY8mVK5uo0VMVWFVdcpKrYquLynB2rJjm7hgc5u4Zzrh683Ini4Mo6EBMbj+5DZuCL7Yt1P6yq2KriCgCqYquKy3N2rLbk7Boe5Owazro/lP8jAdlVofhG8wHgmgcAOZNzdk2ZdYN1Q/Fj/j8XnuKg+JbzocWHFl9o2hqZqraiKi6zUa5p2yo5a6uZLJWRAMVBQZ3YGLHHGjUh8RbWbf4Eowd3czjSgF7tcnVkVbFVxRUXqSq2qrg8Z8eqSc6u4UHOruGcqwcvd3IgQHFQUCFGT1uUY9Qlr43NsUxWBVTFVhVXXIOq2Kri8pwdax45u4YHObuGc64evNyJ4sA6QAIkQAIkQAIkkDsCzDjkjluOex38+gfs//I4rsTFo+T9xeHVvAG8/lc/x/20FFAVW1VccU2qYquKy3N2rInk7Boe5OwazlqesyyTPQGKg4LasWrTR9hz8Ft0btsM4t+DenfAB7u/RNd2/8Ow/p2dOqKq2KriiotVFVtVXJ6zYxUlZ9fwIGfXcHbqAcydJQGKg4KK8Hz/lxGxKgRFPYugTe+J2PfufFy7cRN9RszCzrded+qIqmKriisuVlVsVXF5zo5VlJxdw4OcXcPZqQcwd6Y4qKoDbX0CsHdLmAxvEYeUlFR0HTQNH2+c49RhVcVWFVdcrKrYquLynB2rKDm7hgc5u4azUw9g7kxxUFUHhk0KQ9tnG6Fnp2fh1Ws8gib4YuvOz1Gu9AOYPu5Fpw6rKraquOJiVcVWFZfn7FhFydk1PMjZNZydegBzZ4qDqjpwKeYqvvr2BLp1aIGefkGyy6J18wbo80JrFCzo7tRhVcVWFVdcrKrYquLynB2rKDm7hgc5u4azUw9g7kxxUFkH/jwTheL3FUXZ0vfrfhhVsVXFFQBUxVYVl+fsWG3J2TU8yNk1nHV/KP/HAnJwpIIbPu31NTh4+AfcTEzCnKl+aNmsHvqPeg3BAb548vEqTh1RVWxVccXFqoqtKi7P2bGKkrNreJCzazg79QDmzsw4qKoDYkDkrrdeR+T5Swievx5vhk/Bzn1HsGnbp4hYFezUYVXFVhVXXKyq2Kri8pwdqyg5u4YHObuGs1MPYO5McVBVB3oMnYn3VgbDza0AfEbMwpYVM+Wh2vWZhD2b5zl1WFWxVcUVF6sqtqq4PGfHKkrOruFBzq7h7NQDmDtTHFTVgSPf/Yovvz0Bn65eGB+0DMvnjMPRH05i5cYP8eGG2U4dVlVsVXHFxaqKrSouz9mxipKza3iQs2s4O/UA5s4UB1V1YODYOTj24ykUKGCShyhgKoBKFUpj5viBaNrwCacOqyq2qrjiYlXFVhWX5+xYRcnZNTzI2TWcnXoAc2eKg6o60O+lV7F2wWQUKVxI90Ooiq0qrgCgKraquDxnx2pLzq7hQc6u4az7Q/k/GJCzKhTd9N9P/4PvT/wul5ouUbwYnqr7OKo+VEGXo6mKrSquuGhVsVXF5Tk7VlVydg0PcnYNZ10exP/hIBQHBTd/+foPsCFij5SF4sWK4vqNmzj20ykM6fM8/Pp1cuqIqmKriisuVlVsVXF5zo5VlJxdw4OcXcPZqQcwd2ZXhao68Gz3sfIjV/aLP8XExqO3fwj2Ryxw6rCqYquKKy5WVWxVcXnOjlWUnF3Dg5xdw9mpBzB3pjioqgMdX5yCD9fPltMxLZvZbEb3ITOwfd2rTh1WVWxVccXFqoqtKi7P2bGKkrNreJCzazg79QDmzhQHVXVALPQkZlW80L45SpUsgWvXE/Dhnq9Q5aEKchVJsdWo9lCuDq8qtqq44iJVxVYVl+fsWDXJ2TU8yNk1nHP14OVODgQ4xkFBhWjjPQHu7nf/mNXud+bm6siqYquKKy5SVWxVcXnOjlWTnF3Dg5xdwzlXD17uRHFgHSABEiABEiABEsgdAWYczojzwQAADVlJREFUcseNe5EACZAACZDAf5IAxeE/edt50SRAAiRAAiSQOwIUh9xx414kQAIkQAIk8J8kQHFQdNsTbt7C18d+xtnIaIh/F/UsgocqlcP/Gj2JYkU9cn3Uq/E3cH+JYrneP7sdt39yCK1bNETxYp4ORS5ejoPJZHJYkyI3B//y6AmULlnCYTZJVHQMkm4nO72i5ujpi9Hm/+2deVRW1RrGn263Qc3rjCgq5nQ10/JmppSaJM7iFJFaoGaoIHVFEyHE4eaASoSI4jxiaEo5JJbeMLEUcyDHnAfK2ZxFQLT17tb3LSgEzvnOcbU6z/4nW5z9nHf/9j7ffs/e735P8xfQpe3Lekx7YJ2tO/aj1tMuqFCuNI6dOoPbt++g5FPFVf8Jp8cff0z3/SRD4M9nLqjMoqJVp2ZVVKlUQbdeUSpKH3dr37wol/7pGhnLR0/8giaN6qk+vHkrA4eOpaOai5Pi40iRHCcyNqScOXcJO/ccxr3791UCNRfn8rqkzXpOxBiztM3Spc26hhArFUCAjoMJw2PbrgMYNmYGnq7mjBquldVEk5mZjRPpZ3HwyClMChuIV5o00HXn5z36I23DnD/VFV3JPBc1djD++eijmrXrv9oHzhXKYuQQH/uRURHZsHkHRk6ahwmh76KVWyPNulJhfkKSyqQ5MdQvz0e+UlL3IvijOIwI7AXPNvon/Ze7DFa2nT1/WdlfvaqzLjtzV1qwfD0WffYVZk0appyHUVPmY9P3aer7Izdu3ca167ewf9MCzfeRyTd43Excu35TpSI/fvosnnumJk79fA5VKzshImwgKjmV1axblAoPGjuF1Z0dvxbLViej0bO1kLb/GAL7dUNk3HKUKlkCFy5fxeihfdCuVZPCZPL9++IVX2Pp5xuRFD8J65O3q7H2QsPaAB7Brr2H8VFwf7Rp2ViztlnPiRhilrZZurRZ8/BhhUII0HEwYYi07x2MD99/K1/nQN68x0UvVj+UekpBP/6x8z9HRmYWhg301iwtupK06r2wqRjo45lnIti19wjCJ8/D2kUTNOtKBTlmNi9qhHo7lTJ5RgI+GPSm+re8tco9v/p0si5tqdSi23vY/PlUiMM2MWapmmgktfdjjxV8JLagG7p7DcHimA/tb7w/HT2NidOWYsEnI1Q1vZOwl99o9Ozqju4dWiid1N0H8eXGrRg9tC8WLl+PTVvTsDA6RBeLHv3DUaFcKTR8phaaN2mABvVq5NHRa3P73sOxdPpIlClVErJK1KF3MBLiwlGvtivSz1yA/4gorNE5NiRbYkLcKOUsdfYJwfhQPzSo+7Sye+9PJzD8f3FIio/QzMOs56Swvnf0GczvpUDu6YgubdY8fFiBjsPDHwON2/lh65rp+U5c2Xdz0KzTIOxYP0uXYbIyYPtcd34C9+7d1/UmbPuhvXj5KvoOiUBIYG+1rSJFNJt2GoTt6+J02dy0kz+2rZ2u6mbcyYLwkf+XZf+cnHto1tlft3Zux0H+nZWVjVlL1iL5+90IDuiFJo3q6rLZrXMAUr6IsWf/FN1WXkPw3appDjkOwmLrmli1/SNFMop6eA/FxuUfw9GxsW3nAVy6cg3i5GxJ3ats9/Fqi84ebmrM6HUc2vUajtx5R2SFx8ZB2vDHv2sB3qyTP/7/WRSKF3sCnXxCkDhnrH0LKDv7Ltw8B+OHJO3jzqznRNpmlrZZurRZy4jktUUhwBWHolDSeM3A4Ei4OFfAIN8u9r1bkZBYh7hFq3HgyEnMjRyuUfX3y+XHP3lFVIF15c1Qa8k9qZz+5Tz6D52MN7u6w63xs2q7Yvvug+oNXE95Y8Bo9OrWWm1HLFrxFTZu3qm2E7w9WynttP1HsWhqqGZpzz6/23My/SyqV8375dGr127g8pXrupwo0fT7YAoa1quJAT6ekDk+dv4XSNt/BPOjHFtxkE8nt331RfTu7qGch/jEDdiYslOtMiSs+gbLVn3jcFpyG8gDh09iztJ1ahtkeEBPDBgeme82V2HgZZUkyM8LzRrXx9ff7sDEafHw9nSH+yuN1LjYsHmnfSWmMK0//n1s1CL8cvYiAt/pjgOHTuJE+jn4vtEWGRmZmLlkDW7ezMC08e9rlTXtOTH7GTTj2abNmocPKxRCgI6DCUPk16s3MH7qEnyzZZfaE1cxDlnZKhCuxUsNEfbft3UHlOl9ayysmX/UPXfxV0TGLVOBahLUGR7kqzuIUfaq3x8Zg6vXb6Kmq4uaZKLnrkTKth+VtsQlSDpurUXiA6S8HTjugU6N3k+ZyxK8xKkcPHoK/3jkEVSr4oxPxg62MwgaHYuPRwdoNVkFWY4YNwtHjqeruq5VnREZ7q/iKCT2wef1tqj/7+qadQuqICsQEbFLsX33T7ocKdlOGRI+Dbcz7qBSxfKIiwjC1LkrkbrrIGq4VsKYYX119Z/YfDcnR8XAJK5LUdseUmQVRuInPFo2xtCB3n8K2C0KHLOeE9sk/KAthaLY9qBraHNeMmbycKSfWBeg42DiKJClVtkTzriTqRwIiRB3JBJfTM0dgW6k6WZGdIudst1x+co1lCtTqsCtFj1tkqBF2zdA9NQvqI44gVnZ2Spw1MgiTmROTo6KG3hYxZE+lgn+4qWrcCpfJs/H24y0Xbaxbt66rZ4RcRwcKWY9J2JTYlIKuudzOkUcq/v3oU5Q6Slm2myWtlm6Zv7W6ekb1slLgI4DRwQJkMDfkoAE9a5PTsWJ0+fUaknxYk/CtUpFtGn5ou7Yl4JWHE79fB6BH0ZjQXQIypbW7hCGTpiNHh1b4oWGdfL0x75DJ9SJIY8W2k+X2ITuZGZhxsJV6ohr85ca2vV/PHAMh4+no0eHlg459G6eAXB/+T8qMNuo4+KySitbu508mqGma2W1pSeMJTaqZIli6r96T6f9LQf8Q2wUHYeHCJu3IgESeDgE5CitHKmVL9TKkeiSJYpDJk/Jq7J8dTJ6dW+Nd3p20GXM863fwbyo4HzrJn+XhkPHTmPW5GGatRu491W2tmz6HIb4edkDaHfvO4KwiLkqRmjA250160oFqS/bjxL0LJOwrcg2lpyYevG5uvjA//eTTnqKnGySbSU5uisnmozIqSI2X7h0RR3JdSpfGlPilqmYmjo1qqo8IuJI7Euer8dc1nGQAB0HBwGyOgmQwF+PQPOugeqYZ34JpGT70HvAGGxZFaPLcDn9ULO6S4F1Vy8Yp1lb9vR/SJqJ4I9momKFMggO6GnXkERsEmT8bWK0Zl2p8EqXQHUyxpZ8Ttq/bOYopXXl2g14+oaqU0R6i+1ItGhFxH6KCxevqLgoR3KqyOmdDQlT1EqRFNkW6Rc0CTa2jIHQ21uO16Pj4DhDKpAACfzFCMixV8kNkl+shLytvvZGEFK/nKHLarMmLJuuHMsNCInC88/Whr9vF2Xj7YxMlQ/l+zWxumyWrYRvV0arI+Jy2kgmepmUKzuXV6e95EiwHm1xFKSI47F64Xi7bRI4GzMvER1fawr/Pl112Sw5PtYtibDHjEjStc6+ISpnixSz+kGXsRarRMfBYh3O5pKAFQhMiIlH2r6j6OPdTq0OyJ647JnLHvm8hHVqS2BUkK8uFN36hRl2ZDa3AbknQnEU/EOiVFC1W+P6kCyrpf5VAlPCB+myWbScypVBx9ZNsXxNMko+VQJ7Dx5Xid62bN+j/hYRNkCztjggUiSI+EFxHbaJXqv4mMgFOHP+Mvr17KCy4c5cvFoFV0sWWzoOWmkaez0dB2N5Uo0ESOAvQEBO8axYuwlJyak4fuqsOtlU7MknVHCk5NF4s8trpp0O0dv8198dhRWzx9irS3K0les2Y+eeQ3B1qaicINuyvdZ7yFaH5N+QTK2SmyX0vbdUiu+U1D2oVsUJvl6i/YRWWfv1tq0K3QL5VBTnafrCL7B564/Iyr6rEtJJ7IdtuyU+cSN6d29t5C2pVUQCdByKCIqXkQAJkAAJ5E9AthFkRYTFGgToOFijn9lKEiABEiABEjCEAB0HQzBShARIgARIgASsQYCOgzX6ma0kARIgARIgAUMI0HEwBCNFSIAESIAESMAaBOg4WKOf2UoSIAESIAESMIQAHQdDMFKEBEiABEiABKxBgI6DNfqZrSQBEiABEiABQwjQcTAEI0VIgARIgARIwBoE6DhYo5/ZShIgARIgARIwhAAdB0MwUoQESIAESIAErEGAjoM1+pmtJAESIAESIAFDCNBxMAQjRUiABEiABEjAGgToOFijn9lKEiABEiABEjCEAB0HQzBShARIgARIgASsQYCOgzX6ma0kARIgARIgAUMI0HEwBCNFSIAESIAESMAaBOg4WKOf2UoSIAESIAESMIQAHQdDMFKEBEiABEiABKxBgI6DNfqZrSQBEiABEiABQwjQcTAEI0VIgARIgARIwBoE6DhYo5/ZShIgARIgARIwhAAdB0MwUoQESIAESIAErEGAjoM1+pmtJAESIAESIAFDCNBxMAQjRUiABEiABEjAGgToOFijn9lKEiABEiABEjCEAB0HQzBShARIgARIgASsQYCOgzX6ma0kARIgARIgAUMI/Aazv/OJKm1qkQAAAABJRU5ErkJggg==",
      "text/html": [
       "<div>                            <div id=\"8cd5dbc7-3339-45a4-be3d-30fa98dcfe2f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8cd5dbc7-3339-45a4-be3d-30fa98dcfe2f\")) {                    Plotly.newPlot(                        \"8cd5dbc7-3339-45a4-be3d-30fa98dcfe2f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"Head 0\",\"Head 1\",\"Head 2\",\"Head 3\",\"Head 4\",\"Head 5\",\"Head 6\",\"Head 7\",\"Head 8\",\"Head 9\",\"Head 10\",\"Head 11\",\"Head 12\",\"Head 13\",\"Head 14\",\"Head 15\"],\"y\":[\"Layer 0\",\"Layer 1\",\"Layer 2\",\"Layer 3\",\"Layer 4\",\"Layer 5\",\"Layer 6\",\"Layer 7\"],\"z\":[[0.027988387271761894,0.050817787647247314,0.06861968338489532,0.031243784353137016,0.040589071810245514,0.042291149497032166,0.022635014727711678,0.04909639433026314,0.05521165952086449,0.018792012706398964,0.051243629306554794,0.034362636506557465,0.1176576018333435,0.04433203861117363,0.03281690552830696,0.019391782581806183],[0.051099956035614014,0.05374652147293091,0.01564853824675083,0.18012017011642456,0.045214418321847916,0.2715858221054077,0.10926873236894608,0.044607922434806824,0.05784713104367256,0.044993799179792404,0.16187720000743866,0.25243663787841797,0.41557085514068604,0.19385139644145966,0.07484206557273865,0.02463134564459324],[0.38289040327072144,0.1295502781867981,0.05824587121605873,0.46146976947784424,0.37013423442840576,0.36953499913215637,0.08449036628007889,0.03721775487065315,0.1587517261505127,0.1831357777118683,0.04390658438205719,0.2249511480331421,0.049546193331480026,0.04181525483727455,0.31112954020500183,0.06048464775085449],[0.07017214596271515,0.07232293486595154,0.03803824633359909,0.23560552299022675,0.03785311058163643,0.07164649665355682,0.03152455762028694,0.2580982446670532,0.9985824227333069,0.3006895184516907,0.2997439503669739,0.0989137589931488,0.3864877223968506,0.11750023066997528,0.17233213782310486,0.013575251214206219],[0.08191242069005966,0.07029113173484802,0.024787385016679764,0.2186519205570221,0.1651458591222763,0.1930815726518631,0.19477494060993195,0.07729773223400116,0.03770160675048828,0.04388655349612236,0.08513171970844269,0.029208874329924583,0.13372521102428436,0.18646501004695892,0.054016366600990295,0.02008497528731823],[0.10474904626607895,0.07148798555135727,0.08303850144147873,0.022116560488939285,0.0245988592505455,0.021530311554670334,0.03135489672422409,0.019414279609918594,0.027183979749679565,0.1106434240937233,0.10631193965673447,0.07634454220533371,0.05989508330821991,0.040975600481033325,0.39580607414245605,0.02888302505016327],[0.031818609684705734,0.1340329498052597,0.044984325766563416,0.037765853106975555,0.06614495068788528,0.08521831780672073,0.164445698261261,0.04341691732406616,0.07583362609148026,0.27800294756889343,0.06184188649058342,0.08890586346387863,0.03657130151987076,0.0444006472826004,0.06741028279066086,0.16647715866565704],[0.04968482628464699,0.04948631301522255,0.03500191494822502,0.041399430483579636,0.06689193844795227,0.03444244712591171,0.06597341597080231,0.04243674874305725,0.08141452819108963,0.037084221839904785,0.03423188626766205,0.03423410281538963,0.04130541533231735,0.06264433264732361,0.06538373976945877,0.03236440569162369]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\"},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]]},\"title\":{\"text\":\"Prev Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('8cd5dbc7-3339-45a4-be3d-30fa98dcfe2f');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "long_text = \"we have two runs with two different prompts and different answers, eg 'Steve Jobs founded' -> ' Apple' and 'Bill Gates founded' -> 'Microsoft'. We patch parts of the layer outputs or residual stream from specific tokens and positions and see which patches significantly shift the answer from 'Apple' to 'Microsoft'\"\n",
    "print(\"Long text:\", long_text)\n",
    "attn_cache = {}\n",
    "\n",
    "def filter_hook_attn(name):\n",
    "    split_name = name.split(\".\")\n",
    "    return split_name[-1] == \"hook_attn\"\n",
    "\n",
    "def cache_attn(attn, hook):\n",
    "    attn_cache[hook.name] = attn\n",
    "\n",
    "logits = model.run_with_hooks(long_text, fwd_hooks=[(filter_hook_attn, cache_attn)])\n",
    "\n",
    "# we go through the cache and find the avg attention paid to previous tokens\n",
    "prev_token_scores = np.zeros((model.cfg.n_layers, model.cfg.n_heads))\n",
    "for layer in range(model.cfg.n_layers):\n",
    "    for head in range(model.cfg.n_heads):\n",
    "        attn = attn_cache[f\"blocks.{layer}.attn.hook_attn\"][0, head]\n",
    "        prev_token_scores[layer, head] = attn.diag(-1).mean().item()\n",
    "\n",
    "px.imshow(\n",
    "    prev_token_scores,\n",
    "    x=[f\"Head {hi}\" for hi in range(model.cfg.n_heads)],\n",
    "    y=[f\"Layer {i}\" for i in range(model.cfg.n_layers)],\n",
    "    title=\"Prev Token Scores\",\n",
    "    color_continuous_scale=\"Blues\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d74825cb-e29b-45f0-9c8c-ca2e9a172c73",
   "metadata": {},
   "source": [
    "We identify Head 8, Layer 3 as mostly responsible for identifying the previous token.\n",
    "Future work: Find the effect of pruning that head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "63be69a5-a42b-465b-a296-2e92a982489f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pruning the \"previous token head\"\n",
    "layer = 3\n",
    "head_indices = torch.tensor([8])\n",
    "\n",
    "def prune_attn_heads(value, hook):\n",
    "    \"\"\"Value has shape [batch, pos, index, d_head]\n",
    "    \"\"\"\n",
    "    value[:, :, head_indices, :] = 0.0\n",
    "    return value\n",
    "\n",
    "def filter_value_hooks(name):\n",
    "    return name.split(\".\")[-1] == \"hook_v\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "3cb71c0b-9709-4f80-8591-8e223fc86891",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<>:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "<>:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "/tmp/ipykernel_137162/3350977072.py:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "/tmp/ipykernel_137162/3350977072.py:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "/tmp/ipykernel_137162/3350977072.py:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "/tmp/ipykernel_137162/3350977072.py:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n",
      "/tmp/ipykernel_137162/3350977072.py:5: SyntaxWarning:\n",
      "\n",
      "list indices must be integers or slices, not tuple; perhaps you missed a comma?\n",
      "\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "list indices must be integers or slices, not tuple",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [77], line 5\u001b[0m\n\u001b[1;32m      2\u001b[0m ablation_log_probs \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mlog_softmax(ablation_logits, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m      3\u001b[0m text_token_ids \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mtokenizer\u001b[38;5;241m.\u001b[39mencode(long_text)\n\u001b[1;32m      4\u001b[0m ablated_pred_log_probs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mgather(\n\u001b[0;32m----> 5\u001b[0m     ablated_log_probs[:, :\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m], \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtext_token_ids\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m]\u001b[49m\n\u001b[1;32m      6\u001b[0m )[\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m, \u001b[38;5;241m0\u001b[39m]\n",
      "\u001b[0;31mTypeError\u001b[0m: list indices must be integers or slices, not tuple"
     ]
    }
   ],
   "source": [
    "ablation_logits = model.run_with_hooks(long_text, fwd_hooks=[(filter_value_hooks, prune_attn_heads)])\n",
    "ablation_log_probs = F.log_softmax(ablation_logits, dim=-1)\n",
    "text_token_ids = model.tokenizer.encode(long_text)\n",
    "ablated_pred_log_probs = torch.gather(\n",
    "    ablated_log_probs[:, :-1], -1, [[text_token_ids]][:, 1:, None]\n",
    ")[..., 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f0776be-ac92-4fac-8081-c850ca06f89e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    \"Original loss on repeated sequence:\", induction_pred_log_probs[:, seq_len:].mean().item()\n",
    ")\n",
    "print(\n",
    "    \"Loss on repeated sequence without induction heads:\",\n",
    "    ablated_pred_log_probs[:, seq_len:].mean().item(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "a4592faf-f361-41e5-a7d4-e3ac6cb94a15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[664,\n",
       " 452,\n",
       " 767,\n",
       " 6613,\n",
       " 342,\n",
       " 767,\n",
       " 1027,\n",
       " 49887,\n",
       " 285,\n",
       " 1027,\n",
       " 9172,\n",
       " 13,\n",
       " 24088,\n",
       " 686,\n",
       " 31969,\n",
       " 32254,\n",
       " 11420,\n",
       " 8,\n",
       " 5204,\n",
       " 686,\n",
       " 8217,\n",
       " 8,\n",
       " 285,\n",
       " 686,\n",
       " 25310,\n",
       " 27982,\n",
       " 11420,\n",
       " 8,\n",
       " 5204,\n",
       " 686,\n",
       " 16880,\n",
       " 5983,\n",
       " 844,\n",
       " 12097,\n",
       " 4243,\n",
       " 273,\n",
       " 253,\n",
       " 3828,\n",
       " 18012,\n",
       " 390,\n",
       " 12541,\n",
       " 5542,\n",
       " 432,\n",
       " 2173,\n",
       " 21761,\n",
       " 285,\n",
       " 6887,\n",
       " 285,\n",
       " 923,\n",
       " 534,\n",
       " 20412,\n",
       " 3012,\n",
       " 5333,\n",
       " 253,\n",
       " 3662,\n",
       " 432,\n",
       " 686,\n",
       " 25989,\n",
       " 8,\n",
       " 281,\n",
       " 686,\n",
       " 16880,\n",
       " 8]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5619db46-59ed-45d3-80d4-b76040c7cbc9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}