notebooks/Text Normalization Demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Normalization using Memory Augmented Neural Networks\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook and the accompanying paper <a href=\"http://arxiv.org/abs/1806.00044\">Text Normalization using Memory Augmented Neural Networks</a>  demonstrates an accuracy of 99.4% (English) and 99.3% (Russian) on the Text Normalization Challenge by Richard Sproat and Navdeep Jaitly."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table of Contents\n",
    "---\n",
    "1. [Import Dependencies](#import)\n",
    "2. [Global Config](#config)\n",
    "3. [Load Dataset](#load)\n",
    "4. [XGBoost Classification](#xgb)\n",
    "5. [Encode Data](#encode)\n",
    "6. [DNC Normalization](#dnc)\n",
    "7. [Data Postprocessing](#post)\n",
    "8. [Results Analysis](#result)\n",
    "9. [Comparison](#comparison)  \n",
    "10. [Conclusion](#conclusion)\n",
    "\n",
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Import Dependencies\n",
    "<a id=\"import\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aman/anaconda3/envs/deep-tf/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import gc\n",
    "import sys\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append(\"../src\")\n",
    "\n",
    "from utils import Encoder\n",
    "from utils import Normalized2String\n",
    "from XGBclassify import XGB\n",
    "import classification_report\n",
    "import DNCnormalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**System Information**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Oct 09 2018 \n",
      "\n",
      "CPython 3.6.3\n",
      "IPython 6.2.1\n",
      "\n",
      "numpy 1.13.3\n",
      "pandas 0.21.0\n",
      "matplotlib 2.1.0\n",
      "seaborn 0.8.1\n",
      "sklearn 0.19.1\n",
      "xgboost 0.6\n",
      "tensorflow 1.11.0\n",
      "\n",
      "compiler   : GCC 7.2.0\n",
      "system     : Linux\n",
      "release    : 4.15.0-20-generic\n",
      "machine    : x86_64\n",
      "processor  : x86_64\n",
      "CPU cores  : 8\n",
      "interpreter: 64bit\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -n -m -p numpy,pandas,matplotlib,seaborn,sklearn,xgboost,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Global Config\n",
    "<a id=\"config\"></a>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Language : English or Russian?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = 'english'\n",
    "use_memory=True\n",
    "#lang = 'russian'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if lang == 'english':\n",
    "    # input data\n",
    "    data_directory = '../data/english/'\n",
    "    data = data_directory+'output-00099-of-00100_processed.csv'\n",
    "    vocab = data_directory+'en_vocab.data'\n",
    "    # interim data\n",
    "    encoded_file = data_directory+'en_encoded.npy'\n",
    "    encoded_len_file = data_directory+'en_encoded_len.npy'\n",
    "    normalized_file = data_directory+'en_normalized.npy'\n",
    "    # model\n",
    "    model_directory = '../models/english/'\n",
    "    xgb_path = model_directory+'en_xgb_tuned-trained.pk'\n",
    "    dnc_path = model_directory+'dnc_translator/ckpt'\n",
    "    end_token = -1\n",
    "    # results\n",
    "    result_dir = '../results/english/'\n",
    "    result_csv = 'normalized.csv'\n",
    "\n",
    "elif lang == 'russian':\n",
    "    # input data\n",
    "    data_directory = '../data/russian/'\n",
    "    data = data_directory+'output-00099-of-00100_processed.csv'\n",
    "    vocab = data_directory+'ru_vocab.data'\n",
    "    # interim data\n",
    "    encoded_file = data_directory+'ru_encoded.npy'\n",
    "    encoded_len_file = data_directory+'ru_encoded_len.npy'\n",
    "    normalized_file = data_directory+'ru_normalized.npy'\n",
    "    # model\n",
    "    model_directory = '../models/russian/'\n",
    "    xgb_path = model_directory+'ru_xgb_tuned-trained.pk'\n",
    "    dnc_path = model_directory+'dnc_translator/ckpt'\n",
    "    end_token = -1\n",
    "    # results\n",
    "    result_dir = '../results/russian/'\n",
    "    result_csv = 'normalized.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Load DNC Configurations **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(vocab,'rb') as vf:        \n",
    "        vocab_load=pickle.loads(vf.read())\n",
    "start_token = vocab_load['output']['<GO>']\n",
    "input_vocab_len=len(vocab_load['input'])+1\n",
    "output_vocab_len=len(vocab_load['output'])+1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 3. Load Dataset\n",
    "<a id=\"load\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Dataset by Sproat and Jaitly (2016) - An RNN Model of Text Normalization**  \n",
    "- English Source: https://storage.googleapis.com/text-normalization/en_with_types.tgz\n",
    "- Russian Source: https://storage.googleapis.com/text-normalization/ru_with_types.tgz  \n",
    "*The data is preprocessed for achieving results comparable to the ones presented in the above mentioned paper.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Read CSV as DataFrame**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 92451 entries, 0 to 92450\n",
      "Data columns (total 5 columns):\n",
      "sentence_id    92451 non-null int64\n",
      "token_id       92451 non-null int64\n",
      "semiotic       92451 non-null object\n",
      "before         92451 non-null object\n",
      "after          92451 non-null object\n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 3.5+ MB\n"
     ]
    }
   ],
   "source": [
    "raw_data = pd.read_csv(data)\n",
    "raw_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>semiotic</th>\n",
       "      <th>before</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>It</td>\n",
       "      <td>It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>can</td>\n",
       "      <td>can</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>be</td>\n",
       "      <td>be</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>summarized</td>\n",
       "      <td>summarized</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>as</td>\n",
       "      <td>as</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentence_id  token_id semiotic      before       after\n",
       "0            0         0    PLAIN          It          It\n",
       "1            0         1    PLAIN         can         can\n",
       "2            0         2    PLAIN          be          be\n",
       "3            0         3    PLAIN  summarized  summarized\n",
       "4            0         4    PLAIN          as          as"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Dropping the ground truth labels**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "raw_data.drop(['after'], axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 4. XGBoost Classification  \n",
    "<a id=\"xgb\"></a>\n",
    "**ToBeNormalized or RemainSame**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# instantiate and load trained\n",
    "# XGBoost Model for classification\n",
    "xgb = XGB(xgb_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 100%        \r"
     ]
    }
   ],
   "source": [
    "# Class of tokens in the data\n",
    "raw_data['class'] = xgb.predict(data=raw_data)\n",
    "# Raw to Classified Data\n",
    "classified_data = raw_data.copy(deep=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>semiotic</th>\n",
       "      <th>before</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72327</th>\n",
       "      <td>5947</td>\n",
       "      <td>4</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>he</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92408</th>\n",
       "      <td>7546</td>\n",
       "      <td>10</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Britain</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>306</td>\n",
       "      <td>11</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>attracted</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>51</td>\n",
       "      <td>20</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>atmospheric</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55792</th>\n",
       "      <td>4581</td>\n",
       "      <td>2</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>a</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55875</th>\n",
       "      <td>4586</td>\n",
       "      <td>12</td>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>2.98</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50709</th>\n",
       "      <td>4162</td>\n",
       "      <td>11</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Register</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23681</th>\n",
       "      <td>1905</td>\n",
       "      <td>0</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Some</td>\n",
       "      <td>RemainSelf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25183</th>\n",
       "      <td>2040</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>November 14, 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57024</th>\n",
       "      <td>4688</td>\n",
       "      <td>4</td>\n",
       "      <td>DATE</td>\n",
       "      <td>2005</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentence_id  token_id semiotic             before           class\n",
       "72327         5947         4    PLAIN                 he      RemainSelf\n",
       "92408         7546        10    PLAIN            Britain      RemainSelf\n",
       "3858           306        11    PLAIN          attracted      RemainSelf\n",
       "671             51        20    PLAIN        atmospheric      RemainSelf\n",
       "55792         4581         2    PLAIN                  a      RemainSelf\n",
       "55875         4586        12  DECIMAL               2.98  ToBeNormalized\n",
       "50709         4162        11    PLAIN           Register      RemainSelf\n",
       "23681         1905         0    PLAIN               Some      RemainSelf\n",
       "25183         2040         1     DATE  November 14, 2011  ToBeNormalized\n",
       "57024         4688         4     DATE               2005  ToBeNormalized"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classified_data.sample(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "id_tobenormalized = classified_data.index[classified_data['class']=='ToBeNormalized'].tolist()\n",
    "id_remainself = classified_data.index[classified_data['class']=='RemainSelf'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Sanity Check..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens to be normalized : 6778\n",
      "Tokens to remain self : 85673\n"
     ]
    }
   ],
   "source": [
    "print('Tokens to be normalized : {}'.format(len(id_tobenormalized)))\n",
    "print('Tokens to remain self : {}'.format(len(id_remainself)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 5. Encode Data\n",
    "<a id=\"encode\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Instatiate encoder with the vocabulary\n",
    "encoder = Encoder(vocab_file=vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# use existing Encoder Decoder parameters\n",
    "# to perform the decoding of the test data\n",
    "enc_data, enc_len = encoder.encode(classified_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# encode 'tobenormalized' tokens only\n",
    "tobenormalized_enc_data = enc_data[id_tobenormalized]\n",
    "tobenormalized_enc_len = enc_len[id_tobenormalized]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Sanity check..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens to be normalized : 6778\n"
     ]
    }
   ],
   "source": [
    "print('Tokens to be normalized : {}'.format(len(tobenormalized_enc_data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Saving encoded data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "np.save(encoded_file, tobenormalized_enc_data)\n",
    "np.save(encoded_len_file, tobenormalized_enc_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 6. DNC Normalization\n",
    "<a id=\"dnc\"></a>\n",
    "**Generate Normalized Form**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tobenormalized_enc_data = np.load(encoded_file)\n",
    "tobenormalized_enc_len = np.load(encoded_len_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "DNCnormalize.config['num_encoder_symbols'] = input_vocab_len\n",
    "DNCnormalize.config['num_decoder_symbols'] = output_vocab_len \n",
    "DNCnormalize.config['start_token']= start_token\n",
    "DNCnormalize.config['end_token']= end_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using DNC model at ../models/english/dnc_translator/ckpt\n",
      "building model..\n",
      "building encoder..\n",
      "building decoder and attention..\n",
      "building greedy decoder..\n",
      "Reloading model parameters...\n",
      "INFO:tensorflow:Restoring parameters from ../models/english/dnc_translator/ckpt\n",
      "model restored from ../models/english/dnc_translator/ckpt\n",
      "Number of batches: 33\n",
      "Normalized 200 out of 6600\n",
      "Normalized 400 out of 6600\n",
      "Normalized 600 out of 6600\n",
      "Normalized 800 out of 6600\n",
      "Normalized 1000 out of 6600\n",
      "Normalized 1200 out of 6600\n",
      "Normalized 1400 out of 6600\n",
      "Normalized 1600 out of 6600\n",
      "Normalized 1800 out of 6600\n",
      "Normalized 2000 out of 6600\n",
      "Normalized 2200 out of 6600\n",
      "Normalized 2400 out of 6600\n",
      "Normalized 2600 out of 6600\n",
      "Normalized 2800 out of 6600\n",
      "Normalized 3000 out of 6600\n",
      "Normalized 3200 out of 6600\n",
      "Normalized 3400 out of 6600\n",
      "Normalized 3600 out of 6600\n",
      "Normalized 3800 out of 6600\n",
      "Normalized 4000 out of 6600\n",
      "Normalized 4200 out of 6600\n",
      "Normalized 4400 out of 6600\n",
      "Normalized 4600 out of 6600\n",
      "Normalized 4800 out of 6600\n",
      "Normalized 5000 out of 6600\n",
      "Normalized 5200 out of 6600\n",
      "Normalized 5400 out of 6600\n",
      "Normalized 5600 out of 6600\n",
      "Normalized 5800 out of 6600\n",
      "Normalized 6000 out of 6600\n",
      "Normalized 6200 out of 6600\n",
      "Normalized 6400 out of 6600\n",
      "Normalized 6600 out of 6600\n"
     ]
    }
   ],
   "source": [
    "normalized_data = DNCnormalize.normalize(tobenormalized_enc_data, tobenormalized_enc_len,dnc_path,\n",
    "                                         use_memory=use_memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Sanity Check..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6778"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tobenormalized_enc_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6800"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(normalized_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Saving the normalized form**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "np.save(normalized_file, normalized_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 7. Data Postprocessing\n",
    "<a id=\"post\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Load Normalized Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "normalized_data = np.load(normalized_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Encoded from to String form**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Converting the numpy array to a list form\n",
    "normalized_data = normalized_data[0:tobenormalized_enc_len.shape[0]]\n",
    "normalized_data = np.split(normalized_data, normalized_data.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Sanity check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total instances : 6778\n",
      "Shape of each instance : (1, 1, 150, 1)\n"
     ]
    }
   ],
   "source": [
    "print('Total instances : {}'.format(len(normalized_data)))\n",
    "print('Shape of each instance : {}'.format(normalized_data[0].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Reshaping the nested numpy arrays\n",
    "for i in range(len(normalized_data)):\n",
    "    normalized_data[i] = np.reshape(normalized_data[i],\n",
    "                                    normalized_data[i].shape[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Converting encoded to string format\n",
    "str_converter = Normalized2String(vocab)\n",
    "for i in range(len(normalized_data)):\n",
    "    normalized_data[i]=str_converter.to_str(normalized_data[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "A sneak peek..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nineteen eighty four',\n",
       " 'nineteen',\n",
       " 'august nineteen ninety one',\n",
       " 'january twenty second two thousand one',\n",
       " 'thirty five',\n",
       " 'june twenty fifteen',\n",
       " 'the thirtieth of september two thousand eight',\n",
       " 'two thousand five',\n",
       " 'may twelfth nineteen eighty one',\n",
       " 'nineteen eighty seven']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_data[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "**Merging Normalized with Remain Self**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "classified_data['after'] = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "classified_data.loc[id_tobenormalized, 'after'] = normalized_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "RemainSelf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "classified_data.loc[id_remainself, 'after'] = classified_data.loc[id_remainself, 'before'] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "A sneak peek into the final results..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>semiotic</th>\n",
       "      <th>before</th>\n",
       "      <th>class</th>\n",
       "      <th>after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1984</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen eighty four</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>19</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>DATE</td>\n",
       "      <td>August 1991</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>august nineteen ninety one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>DATE</td>\n",
       "      <td>January 22, 2001</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>january twenty second two thousand one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>35</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>thirty five</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>DATE</td>\n",
       "      <td>June 2015</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>june twenty fifteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>DATE</td>\n",
       "      <td>2008-09-30</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the thirtieth of september two thousand eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>DATE</td>\n",
       "      <td>2005</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two thousand five</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>DATE</td>\n",
       "      <td>May 12, 1981</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>may twelfth nineteen eighty one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1987</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen eighty seven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>Jun 29, 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>june twenty ninth twenty eleven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>CHEK</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>c h e k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>1</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>23</td>\n",
       "      <td>8</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>ATM</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>a t m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>ATR</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>a t r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>26</td>\n",
       "      <td>10</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Theatres</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>theaters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>DATE</td>\n",
       "      <td>2003</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two thousand three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>A.</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1986</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen eighty six</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>IARC</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>i a r c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>DC</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>d c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>36</td>\n",
       "      <td>5</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>organisation</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>organization</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>37</td>\n",
       "      <td>7</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>FD</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>f d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>37</td>\n",
       "      <td>8</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>&amp;</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>38</td>\n",
       "      <td>9</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1970s</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen seventies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>38</td>\n",
       "      <td>11</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1980s</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen eighties</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>40</td>\n",
       "      <td>10</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1700</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>seventeen hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>41</td>\n",
       "      <td>4</td>\n",
       "      <td>DATE</td>\n",
       "      <td>August 31, 1987</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>august thirty first nineteen eighty seven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>PbS</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>p b s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>42</td>\n",
       "      <td>13</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>1</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92018</th>\n",
       "      <td>7510</td>\n",
       "      <td>2</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>W. D.</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>w d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92022</th>\n",
       "      <td>7510</td>\n",
       "      <td>6</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>G.</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92026</th>\n",
       "      <td>7510</td>\n",
       "      <td>10</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>B. H.</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>b h</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92028</th>\n",
       "      <td>7510</td>\n",
       "      <td>12</td>\n",
       "      <td>DATE</td>\n",
       "      <td>8 April 2008</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the eighth of april two thousand eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92037</th>\n",
       "      <td>7512</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>5 March 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the fifth of march twenty eleven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92055</th>\n",
       "      <td>7514</td>\n",
       "      <td>3</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1973</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen seventy three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92076</th>\n",
       "      <td>7517</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>27 April 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the twenty seventh of april twenty eleven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92095</th>\n",
       "      <td>7519</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>17 October 2013</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the seventeenth of october twenty thirteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92097</th>\n",
       "      <td>7520</td>\n",
       "      <td>0</td>\n",
       "      <td>DATE</td>\n",
       "      <td>23 July 2010</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the twenty third of july twenty ten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92198</th>\n",
       "      <td>7528</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>May 13, 2013</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>may thirteenth twenty thirteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92208</th>\n",
       "      <td>7529</td>\n",
       "      <td>8</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1999's</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen ninety nines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92228</th>\n",
       "      <td>7531</td>\n",
       "      <td>2</td>\n",
       "      <td>DATE</td>\n",
       "      <td>3 January 2016</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the third of january twenty sixteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92230</th>\n",
       "      <td>7531</td>\n",
       "      <td>4</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>yahoo!</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>yahoo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92254</th>\n",
       "      <td>7533</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>4 September 2008</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the fourth of september two thousand eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92260</th>\n",
       "      <td>7534</td>\n",
       "      <td>4</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>&amp;</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92263</th>\n",
       "      <td>7534</td>\n",
       "      <td>7</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1971</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen seventy one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92280</th>\n",
       "      <td>7536</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>21 June 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the twenty first of june twenty eleven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92287</th>\n",
       "      <td>7537</td>\n",
       "      <td>5</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>2</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92289</th>\n",
       "      <td>7537</td>\n",
       "      <td>7</td>\n",
       "      <td>DATE</td>\n",
       "      <td>July 1963</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>july nineteen sixty three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92303</th>\n",
       "      <td>7538</td>\n",
       "      <td>5</td>\n",
       "      <td>DATE</td>\n",
       "      <td>November 22, 2008</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>november twenty second two thousand eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92306</th>\n",
       "      <td>7539</td>\n",
       "      <td>1</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>PPIHC</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>p p i h c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92311</th>\n",
       "      <td>7539</td>\n",
       "      <td>6</td>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>12.42</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>twelve point four two</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92314</th>\n",
       "      <td>7539</td>\n",
       "      <td>9</td>\n",
       "      <td>MEASURE</td>\n",
       "      <td>19.99 km</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>nineteen point nine nine kilometers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92320</th>\n",
       "      <td>7539</td>\n",
       "      <td>15</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>156</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one hundred fifty six</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92326</th>\n",
       "      <td>7539</td>\n",
       "      <td>21</td>\n",
       "      <td>MEASURE</td>\n",
       "      <td>4,720 ft</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>four thousand seven hundred twenty feet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92329</th>\n",
       "      <td>7540</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>20 June 2011</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the twentieth of june twenty eleven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92349</th>\n",
       "      <td>7542</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>4 October 2015</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the fourth of october twenty fifteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92375</th>\n",
       "      <td>7544</td>\n",
       "      <td>2</td>\n",
       "      <td>DATE</td>\n",
       "      <td>2012</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>twenty twelve</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92426</th>\n",
       "      <td>7549</td>\n",
       "      <td>1</td>\n",
       "      <td>DATE</td>\n",
       "      <td>9 May</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>the ninth of may</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92445</th>\n",
       "      <td>7549</td>\n",
       "      <td>20</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>USSB</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>u s s b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6778 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentence_id  token_id  semiotic             before           class  \\\n",
       "20               1         6      DATE               1984  ToBeNormalized   \n",
       "27               1        13  CARDINAL                 19  ToBeNormalized   \n",
       "68               4        20      DATE        August 1991  ToBeNormalized   \n",
       "74               6         0      DATE   January 22, 2001  ToBeNormalized   \n",
       "84               8         2  CARDINAL                 35  ToBeNormalized   \n",
       "99               9         5      DATE          June 2015  ToBeNormalized   \n",
       "151             16         5      DATE         2008-09-30  ToBeNormalized   \n",
       "157             17         4      DATE               2005  ToBeNormalized   \n",
       "177             19         4      DATE       May 12, 1981  ToBeNormalized   \n",
       "215             21         1      DATE               1987  ToBeNormalized   \n",
       "236             22         1      DATE       Jun 29, 2011  ToBeNormalized   \n",
       "238             23         0   LETTERS               CHEK  ToBeNormalized   \n",
       "239             23         1  CARDINAL                  1  ToBeNormalized   \n",
       "246             23         8   LETTERS                ATM  ToBeNormalized   \n",
       "248             23        10   LETTERS                ATR  ToBeNormalized   \n",
       "298             26        10     PLAIN           Theatres  ToBeNormalized   \n",
       "308             27         4      DATE               2003  ToBeNormalized   \n",
       "328             29         3   LETTERS                 A.  ToBeNormalized   \n",
       "330             29         5      DATE               1986  ToBeNormalized   \n",
       "364             32         0   LETTERS               IARC  ToBeNormalized   \n",
       "392             34         2   LETTERS                 DC  ToBeNormalized   \n",
       "417             36         5     PLAIN       organisation  ToBeNormalized   \n",
       "430             37         7   LETTERS                 FD  ToBeNormalized   \n",
       "431             37         8  VERBATIM                  &  ToBeNormalized   \n",
       "444             38         9      DATE              1970s  ToBeNormalized   \n",
       "446             38        11      DATE              1980s  ToBeNormalized   \n",
       "468             40        10      DATE               1700  ToBeNormalized   \n",
       "477             41         4      DATE    August 31, 1987  ToBeNormalized   \n",
       "484             42         4  VERBATIM                PbS  ToBeNormalized   \n",
       "493             42        13  CARDINAL                  1  ToBeNormalized   \n",
       "...            ...       ...       ...                ...             ...   \n",
       "92018         7510         2   LETTERS              W. D.  ToBeNormalized   \n",
       "92022         7510         6   LETTERS                 G.  ToBeNormalized   \n",
       "92026         7510        10   LETTERS              B. H.  ToBeNormalized   \n",
       "92028         7510        12      DATE       8 April 2008  ToBeNormalized   \n",
       "92037         7512         1      DATE       5 March 2011  ToBeNormalized   \n",
       "92055         7514         3      DATE               1973  ToBeNormalized   \n",
       "92076         7517         1      DATE      27 April 2011  ToBeNormalized   \n",
       "92095         7519         1      DATE    17 October 2013  ToBeNormalized   \n",
       "92097         7520         0      DATE       23 July 2010  ToBeNormalized   \n",
       "92198         7528         1      DATE       May 13, 2013  ToBeNormalized   \n",
       "92208         7529         8      DATE             1999's  ToBeNormalized   \n",
       "92228         7531         2      DATE     3 January 2016  ToBeNormalized   \n",
       "92230         7531         4     PLAIN             yahoo!  ToBeNormalized   \n",
       "92254         7533         1      DATE   4 September 2008  ToBeNormalized   \n",
       "92260         7534         4  VERBATIM                  &  ToBeNormalized   \n",
       "92263         7534         7      DATE               1971  ToBeNormalized   \n",
       "92280         7536         1      DATE       21 June 2011  ToBeNormalized   \n",
       "92287         7537         5  CARDINAL                  2  ToBeNormalized   \n",
       "92289         7537         7      DATE          July 1963  ToBeNormalized   \n",
       "92303         7538         5      DATE  November 22, 2008  ToBeNormalized   \n",
       "92306         7539         1   LETTERS              PPIHC  ToBeNormalized   \n",
       "92311         7539         6   DECIMAL              12.42  ToBeNormalized   \n",
       "92314         7539         9   MEASURE           19.99 km  ToBeNormalized   \n",
       "92320         7539        15  CARDINAL                156  ToBeNormalized   \n",
       "92326         7539        21   MEASURE           4,720 ft  ToBeNormalized   \n",
       "92329         7540         1      DATE       20 June 2011  ToBeNormalized   \n",
       "92349         7542         1      DATE     4 October 2015  ToBeNormalized   \n",
       "92375         7544         2      DATE               2012  ToBeNormalized   \n",
       "92426         7549         1      DATE              9 May  ToBeNormalized   \n",
       "92445         7549        20   LETTERS               USSB  ToBeNormalized   \n",
       "\n",
       "                                               after  \n",
       "20                              nineteen eighty four  \n",
       "27                                          nineteen  \n",
       "68                        august nineteen ninety one  \n",
       "74            january twenty second two thousand one  \n",
       "84                                       thirty five  \n",
       "99                               june twenty fifteen  \n",
       "151    the thirtieth of september two thousand eight  \n",
       "157                                two thousand five  \n",
       "177                  may twelfth nineteen eighty one  \n",
       "215                            nineteen eighty seven  \n",
       "236                  june twenty ninth twenty eleven  \n",
       "238                                          c h e k  \n",
       "239                                              one  \n",
       "246                                            a t m  \n",
       "248                                            a t r  \n",
       "298                                         theaters  \n",
       "308                               two thousand three  \n",
       "328                                                a  \n",
       "330                              nineteen eighty six  \n",
       "364                                          i a r c  \n",
       "392                                              d c  \n",
       "417                                     organization  \n",
       "430                                              f d  \n",
       "431                                              and  \n",
       "444                               nineteen seventies  \n",
       "446                                nineteen eighties  \n",
       "468                                seventeen hundred  \n",
       "477        august thirty first nineteen eighty seven  \n",
       "484                                            p b s  \n",
       "493                                              one  \n",
       "...                                              ...  \n",
       "92018                                            w d  \n",
       "92022                                              g  \n",
       "92026                                            b h  \n",
       "92028         the eighth of april two thousand eight  \n",
       "92037               the fifth of march twenty eleven  \n",
       "92055                         nineteen seventy three  \n",
       "92076      the twenty seventh of april twenty eleven  \n",
       "92095     the seventeenth of october twenty thirteen  \n",
       "92097            the twenty third of july twenty ten  \n",
       "92198                 may thirteenth twenty thirteen  \n",
       "92208                          nineteen ninety nines  \n",
       "92228            the third of january twenty sixteen  \n",
       "92230                                          yahoo  \n",
       "92254     the fourth of september two thousand eight  \n",
       "92260                                            and  \n",
       "92263                           nineteen seventy one  \n",
       "92280         the twenty first of june twenty eleven  \n",
       "92287                                            two  \n",
       "92289                      july nineteen sixty three  \n",
       "92303      november twenty second two thousand eight  \n",
       "92306                                      p p i h c  \n",
       "92311                          twelve point four two  \n",
       "92314            nineteen point nine nine kilometers  \n",
       "92320                          one hundred fifty six  \n",
       "92326        four thousand seven hundred twenty feet  \n",
       "92329            the twentieth of june twenty eleven  \n",
       "92349           the fourth of october twenty fifteen  \n",
       "92375                                  twenty twelve  \n",
       "92426                               the ninth of may  \n",
       "92445                                        u s s b  \n",
       "\n",
       "[6778 rows x 6 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classified_data.loc[id_tobenormalized]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Store the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "classified_data.to_csv(result_dir+result_csv, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Results Analysis\n",
    "<a id=\"result\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.read_csv(result_dir+result_csv)\n",
    "truth = pd.read_csv(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "results, truth = classification_report.preprocessing(results, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                precision    recall  f1-score   support\n",
      "\n",
      "    RemainSelf       1.00      1.00      1.00     86045\n",
      "ToBeNormalized       0.94      0.99      0.96      6406\n",
      "\n",
      "   avg / total       1.00      0.99      1.00     92451\n",
      "\n"
     ]
    }
   ],
   "source": [
    "classification_report.f1_scores(results, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEYCAYAAAAdwT4RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xm81GP/x/HX+3RKpVKkkKWQbDep\nkD1yU0R22bKU7De/221339z23S3r7c6W3EJkjUTqtpTKLoqUyBKpVJK2z++P65pM45yZOZ0zZ+ac\n+Tw95uHM9d2umdP5zDXX97quj8wM55xzuVOS7wo451xt54HWOedyzAOtc87lmAda55zLMQ+0zjmX\nYx5onXMuxzzQuoIj6SpJsyR9X4lzbChpgaQ6VVm36ibpYkkDVuG4LpKWx/egWy7qlnK9UZL6xp+P\nkfRyFsc8KOmqNNu/kLRY0qCqrGs+eKDNAUmNJH0p6eikssaSvpJ0WFJZJ0nPS5ojaa6kTyRdLalZ\n3H6CpGXxj2WBpKmSTstx3btImpHFfjtIGhbrPVvSOEknVsH1NwDOBbY0s3VW9Txm9pWZNTKzZZWt\nUypJJmmmpNKkslJJP0jKamB6tu+zmV1jZn1XsarfxvfgpVU8fpWY2SNmtk8VnGcT4JoqqFLeeaDN\nATNbAPQDbpO0diy+AZhgZkMAJO0MjALeBDY3s6ZAN2ApsG3S6cbEP5ZGwGHADZK2q55XUjZJOwEj\ngdHApsBawGlA9yo4/UbAT2b2QxWcK5fmsvLr3Q+YU5UXSA7kroYzM3/k6AE8CDwKdAF+AtZN2vYG\ncHuG408A3kgpGwccnfT8QGAi4Q9/FLBF0rYtYtncuM+BSdv2Az4B5gPfAH8DVgd+BZYDC+JjvTLq\n9QZwZ4a6nwxMAWYDzyafBzDgVOBzQnC6ExCwd8r1H4zv3YyUc38J7B1/3gGYAMwDZgK3xPLW8Tql\n8fl6sR6zY71OTjrf5cDjwMD4fkwEOqV5bQZcCjyRVDYEuCT8Sa0oOxH4NJ5zKnBKLC/zfY71GAIM\niq+nbywbFI87Mp6nSXzeHfgeWLuMOv7hfYvlJ8U6zQGGAxtl+r3EbXWAm4FZwDTgzJT3dxTQN/Xf\nbfy93gr8APwMfAhsnfT3cSfwQnyP3gY2Sanvitdfkx95r0BtfgDNgO/iP84Tk8pXB5YBXTIcv+If\nbHy+PSFobhafbwb8AvwZqAucH4NIvfh8CnBxfL5X/MfcLh77HbBbUj07xJ/L/ANNqkPDWPc90+yz\nV3zNHYDVgNuB/yVtN+B5oCmwIfAj0K2s65dVH1YOtGOA4+LPjYDO8efWKYFgNHAXUB9oH6/ZNW67\nHFhE+PCpA1wLjE3z+gzYmhDYm8bHzFhmSfvtD2wSg80ewMJ073OsxxLgIMK3zQapgQZ4hBCg1gK+\nBXqUU8eyzn9Q/DexBVBK+LB4K8vfy6mED+b147+XV8gu0O4LvBPPqXjtdeO2BwkffDvE+jwCDC7j\nPanxgda7DnLIzOYQWkcNgaeSNjUj/CGtuNkj6YbY3/mLpEuT9u0cyxcQWrMPE1ocEFo4L5jZCDNb\nAtxE+OPcGehMCDzXmdliMxtJ+CM6Kh67BNhSUhMzm2Nm72b5shJ1/y7NPscA95vZu2b2G3ARsJOk\n1kn7XGdmc83sK+A1QvBbFUuATSU1N7MFZjY2dYfY77srcIGZLTKz94EBwHFJu71hZsMs9Ok+zMrd\nN2VZBDxH+B30IrSWFyXvYGYvmNkXFowGXgZ2y3DeMWb2tJktN7Nfy9h+BuGDbBTwnJk9n+F8yU4B\nrjWzT81sKaH/s72kjZL2Ke/3cgRwm5nNiP+ur8vymkuAxsDmhNbxp2aW/G/nKTMbF+vzCKv+76Cg\neaDNIUnHElpWrwDXJ22aQ/jauG6iwMzOt9BPO5Tw6Z4w1syaWuijXQfYit9vEKwHTE86x3Lga6BV\n3PZ1LEuYHrcBHEpowU2XNDr2u2bjD3UvQ2q9FhC6Tlol7ZM8omAh4UNhVfQhtOwnSRovqUc59Zlt\nZvOTypLfi7LqUz+LPtKBQO/4GJi6UVJ3SWPjzcK5hPe7eYZzfp1uo5nNBZ4gtJ5vznCuVBsR7hvM\njfWZTWhlZvN7WS+lbmnrmVTfkcAdhC6CmZLuldQki+vVKh5oc0RSC0Lf1MmElsQRknYHMLNfCP1R\nh1TknGY2E3gSOCAWfUv440lcU8AGhD7Xb4ENJCX/jjeM2zCz8WbWE2gBPE3oo4TwdTBdHRYSvq4f\nmma31HqtTviq+036V1imXwjfCBLnqgMkbjBiZp+b2VGE13E9MCReL7U+a0pqnFS24r2ohNcJHzgt\nCf3WK0hajfC7ugloGT9EhxECG5T/Pqd9/yW1J/SzPgr0r2B9vyb0EzdNejQws7eyOPY7QrdBwgbZ\nXtTM+ptZR0IjYTPgvArVuhbwQJs7dwBPm9lr8avS+cB/4h8g8flJki6MQRlJ6wNtyjuhpLWAgwnd\nERCC4/6SukqqSxgW9RvwFiGQ/wKcL6mupC6EAD1YUr041nGN2OUwj9DvCqGvcS1Ja6R5becDJ0g6\nL9YJSdtKGhy3/xc4UVL7+HqvAd42sy8zv21/8Bmhdbl/fI2XEvp9E+/JsZLWji33ubF4pSFdZvZ1\nfE+ulVRf0jaElvAjq1Cf5PMa4T09MP6crF6s54/AUkndgeQhT9m8zyuRVJ9wo+xiwo22VpJOr0CV\n7wEukrRVPN8akg7P8tjHgbMltZLUFLggyzpvL2nH+Lv7hdC9UuVD7gqdB9ockHQQoU9wxSe3mQ0A\nZgD/iM/fIPS17Q58Fr/KvUToe7s96XQ7JcbREu4W/wicFc8xGTg27j+L8Ed/QOyTXUwYkdA9brsL\n6G1mk+J5jwO+lDSPcKPj2HjOSYTW0tT4FXO91NcXW0B7xcdUSbOBewktNszsVeDvhBbdd4QbQr0q\n/EaGc/0MnE7oU/2G8MeaPP60GzAxvj+3Ab3MbNEfThT6plsTWrdDgcvMbMSq1CmlfhPNbGIZ5fOB\nvxAC1BzgaEI/bmJ7xve5DNcSbnDdHfu+jwWuktQ2y7oOJbT6B8ff+8dkPyTvP4Q+5g+B9wi/66Vk\nDppN4rFzCN01PxFa+UVFf/wgds7VdLGbajjhG86RZja8is/fHbjHzDbKuPOqX2Myof/4cTM7KVfX\nqQ4eaJ1zGUlqAOxJaNW2JHxbGWtm5+S1YjWEB1rnXEaSGhLGIm9OmGzxAnC2mc3La8VqCA+0zjmX\nY34zzDnncswXraihVNrAVK9x5h1dWu232DDfVagVvpr+JbNmzVLmPdOr02Qjs6VlTYj7nf3643Az\ny/nSj1XJA20NpXqNWa3dEfmuRo335tjbM+/kMtql8/ZVch5b+mvGf9eL3r8z0+y6guOB1jlXOCQo\nqdFrtZfJA61zrrB4oHXOuVwSqPbdo/dA65wrHMJbtM45l1sK/bS1jAda51xhqYUt2trXGeKcq8Fi\nH226R6YzSP8naaKkjyU9GpfGbCPpbUmfS3pMUr2472rx+ZS4vXXSeS6K5ZMl7ZtU3i2WTZF0YTav\nygOtc65wJPpo0z3SHS61IixP2cnMtibkgOtFWB7yVjNrS1iysU88pA8wx8w2JSzUf308z5bxuK0I\nS3HeJalOXHj+TsLyklsCR8V90/JA65wrIJVv0RK6RBvEVEQNCWsi70XIMAzwECFRJUDP+Jy4vWvM\nVNKTkCjyNzObRkhquUN8TDGzqXHN58Fx37Q80DrnCoeAOnXSP6C5pAlJj36Jw83sG8LC4l8RAuzP\nhCy8c2MCSAgLxyfypLUi5j+L238mpF1aUZ5yTHnlafnNMOdcAclqZtgsM+tU5tFSM0ILsw0htdET\nlJ1FIrFsYVlDHCxNeVmN04xLIHqgdc4VlsoN79obmGZmP4ZT6SlgZ6CppNLYal2fkNIIQot0A2BG\n7GpYg5AdOFGekHxMeeXl8q4D51zhSKx1sIo3wwhdBp0lNYx9rV2BT4DXgMPiPscDz8Sfn43PidtH\nxkSbzwK94qiENkBbYBwwHmgbRzHUI9wwW5ELrjzeonXOFZZKTME1s7clDQHeJSSPfI+QOPQFQlLK\nq2LZffGQ+4CHJU0htGR7xfNMlPQ4IUgvBc4ws2UAks4k5GOrA9xfVnLOVB5onXMFpPKrd5nZZcBl\nKcVTCSMGUvddBJSZct3MrgauLqN8GDHjc7Y80DrnCotPwXXOuRySoKT2haXa94qcczWbt2idcy7H\nauGiMh5onXOFw1PZOOdc7sm7DpxzLnckUIkHWuecyyF5i9Y553KtpKT2rQzggdY5V1BqY4u29n10\nOOdqLEmoJP0jw/HtJL2f9Jgn6RxJa0oaEVPZjIjLKaKgf0xL86GkDknnOj7u/7mk45PKO0r6KB7T\nX1l8Mnigdc4VFElpH+mY2WQza29m7YGOwEJgKHAh8GpMZfNqfA5hrdq28dEPuDvWYU3Cegk7EtZI\nuCwRnOM+/ZKO65bpNXmgdc4VlJKSkrSPCugKfGFm01k5ZU1qKpuBFowlrFu7LrAvMMLMZpvZHGAE\n0C1ua2JmY+JyigOTzlUu76N1zhWO7IZ3NZc0Ien5vWZ2bxn79QIejT+3NLPvAMzsO0ktYnlFU9a0\nij+nlqflgdY5VzCU3fCuclPZrDhPWJT7QOCijJf8o3SpbMorT8u7DpxzBaUyN8OSdAfeNbOZ8fnM\n+LWf+P8fYnl5KWvSla9fRnlaHmidc4VDlbsZluQofu82gJVT1qSmsukdRx90Bn6OXQzDgX0kNYs3\nwfYBhsdt8yV1jqMNeiedq1zedeAyOuuYPTnh4J0xMyZO+ZZ+lw3i9kt6sVvHTfl5wSIA+v3jYT78\n7BsAduvYlhvPO5S6pXX4ae4C9ul7G+u3bMqAK3vTcq0mLDfj/iff5M5HR624xmm99uDUI3dn6bLl\nvPT6x1xyW8Z/u7XKsmXL2KXz9qzXqhVPPf0cd991B3fefhtTv/iCr779gebNmwMwZ84cTj25D9Om\nfsFq9etzz733sdXWW+e59lWrshMWJDUE/gycklR8HfC4pD6EvGKJrArDgP2AKYQRCicCmNlsSVcS\ncoQBXGFms+PPpwEPAg2AF+MjLQ+0Lq311l6D04/ag+0OvZpFvy1h0PUncfi+HQG4+F9PM/SV91fa\nf41GDbjt4iPoecZdfP39HNZu1giApcuWc+EtT/H+pBk0argab/33Al59exKTpn7P7p3a0qPLn9j+\niGtZvGTpimOKyZ2338bmm2/BvPnzANhpp13Yb78e7PvnPVfa78brr2GbbbflsSFPMXnSJP7v7DMZ\nNvyVfFQ5J7Lso03LzBYCa6WU/UQYhZC6rwFnlHOe+4H7yyifAFTo0827DlxGpXXq0GC1utSpU0KD\n+vX47sefy933yO6deObVD/j6+zkA/DhnAQDfz5rH+5PCzdoFC39j0rTvWW/tpgD0O3w3bnpgBIuX\nLF3pmGIxY8YMXnpxGCec1GdFWfvttmOj1q3/sO+nn37KnnuFeNFu882ZPv1LZs6c+Yf9aixVWR9t\nQfFA69L69sef+dfAV/nsxSuZNuJq5i34lVfHTgLg8jMOYNxjF3HDuYdQr274ctR2oxY0bdKQ4f85\nmzcfOZ+je/whHx4brrsm7dutz/iPvwRg041asMt2m/C/gX/j5QFn03HLDavt9RWC88/9P6669vqs\nvjL/6U/b8MzTTwEwfvw4vpo+nW++mZHhqJqlivpoC0pOAq2ktZKmwH0v6Zuk5/XK2L9U0rKkfd6J\nHdPprlEqySRdn1R2oaRLc/Ga0tRjkKSD4s8PSGpXyfNtKun9zHtWj6aNG9Cjy5/YosdlbLzPJaze\noB699tuef9z+LNsefCW7HnsjzdZYnXNP3BuA0joldNhiAw4+624OPONOLjq5G5tu2GLF+VZvUI9H\nb+rLeTc9yfxfFq04plmThuze+yYuvvVpBt1wUl5eaz4Me+F51m6xNh06dMxq/7+dfyFz5sxlx07b\ncc+dd7Bt++0orVO7egBrY4s2J7+h2B/SHkDS5cACM7spw2Hz47Q5JO1PSPP7hz6VFL8CR0i6Pqmj\nOmuSSs1saUWPK4+ZnVhV5yoUe+24OV9++xOz4tf5p0d+QOdt2zB4WLhHsHjJUgY+M5Zzeodf1Tc/\nzGXW3F9YuGgxCxct5o13p7DNZq2Y8tUPlJaW8OhNJ/PYixN4ZuQHK67xzcy5PP1qeD5h4nSWLzea\nN2u04pq12di33uSF559j+EsvsmjRIubPm8dJxx/H/Q89XOb+TZo04d4BodvQzNhis41p3aZNdVY5\npyTVytW7qv0VSTpf0sfxcVY5uzUB5iQdc6GkcXHRh38k7beY0Fl9dhnXaSPptXjMCEnrx/JBkm6W\n9BpwjaSrJD0o6WVJX0o6KG7/WNILkkrjcf+UND6W31PWQhKS3pDUXtLBSa3zzyR9HrdvL2l0bLG/\nKKllUvmHksYAp67SG5sjX38/mx3+1IYG9esCsOcO7Zg8bSbrNG+yYp8D99yGT74IQwmfG/Uhu2y3\nSezPrcv2W7dm0rTvAbjnsmOYPO17+g8audI1nhv1IV122AyATTdsQb26pUURZAGuuPpapkz7mkmf\nT2PgoEfZY8+9yg2yAHPnzmXx4sUAPHD/AHbddXeaNGlS7v41UW3sOqjW7xySdgCOISzSUAcYJ2k0\n8AnQOH5lrg+sA+wZj9kP2JCwuIOAYZJ2BsbF094OvC/p5pTL3QUMMLNHJPUD/gUcFrdtAnQ1s+WS\nrgLaEFrP2wKvAz3N7FxJzxEWjHgeuM3MLosB9r+xvMxhHWY2FBga9x0CvCxpNeA24EAzmyXpGOBK\nwuIUDwL9zOxNSbdW8G3NqfEfT2foK+8x5r8XsHTZcj6YNIP7nnyTZ+44jebNGiPBh5NncNbVgwGY\nPG0mI976hPGPX8Ty5caDQ9/iky++Y+f2G3NMjx356LNvGDs4rOdx2R3PMvyNT3jo6TH8+/JjmPDE\nxSxesoy+/yg/0BSLu+7ozy0338jM779nh47bsm+37tz97wFMnvQpfU86njolddh8iy25+94B+a5q\nlaup3QPpKIxuyOEFkroOJJ0LrG5mV8Rt1xLmE99LmFbXNJbvCtxtZn+S9C/Cwg+JW92NgKuAQYlj\nJF0DzCdMhSs1s6skzQbWNrNlkuoD082spaRBwItm9ki81lWEbovrY//xXDNrGLddA3xrZndIOhw4\nj/BB0By4Jb6mQcAQM3ta0hvAmWb2fjz+YmBTMztJUntgNDAtvo46hFkmxwHjzGzjeEwH4P5EN0rK\ne9mPEJihbqOO9bc6PnUXV0Gzx92e7yrUCrt03p5335lQ6Qi5Wsu21uqY29LuM+3W/d/JNAW30FR3\nL3pWvwgze0PSegpLlQm4yszuW+lE8St9dAthYPHDhO6ETH5Jef5b/P/ylOOXA6UKA6DvADqY2Tcx\nONdPdwFJ+xI+IPZIFAEfmtluKfs1J4u50gBx4Yx7AUoatsjtJ6RzeSBBSS1s0VZ3H+3/gIMlNZDU\niBCIXk/dSdJWhCA3hzAVro+k1eO29WNwWsHMZhHWnDwhqXgscET8+dh47VXVINZnlqTGwKHpdpbU\nBugPHG5mi2LxJ0Cr2H2CpHqStop1XyRpp7jfMZWop3M1XPr+We+jzYKZjZP0KL9Pa7vbzD6KrdNE\nH21C7zhrY5ikzYGx8U2eDxwNzE05/Y3A6UnPzwTuk3QRMJM4tW4V6/2TpIeAj4HpwNsZDjkJWBN4\nNtb5KzM7UNJhQP8YrEuBm4GJsW4DJP0CvLyq9XSuNqiNLdqc99G63Chp2MJWa3dE5h1dWt5HWzWq\nqo+2/rqbWevj0/9OJl/fLW0fraSmwADCNFkjNHwmA48BrYEvgSPMbE68YX0bYb2DhcAJZvZuPM/x\nQGJc/lVm9lAs78jvax0MA862DIG09g1Yc87VWALq1FHaRxZuA14ys80JI4k+xVPZOOdcFG+GpXuk\nPVxqAuwO3AdgZovNbC55TmXjgdY5VzBEpScsbAz8CDwg6T1JA+KN9JVS2QDVmsrGA61zroCkb83G\nFm1zSROSHv2STlAKdCDcaN+OMJTzwjIu9PsF/6jKU9nUrtUonHM1Xhat1nQ5w2YAM8wsMTJoCCHQ\nzpS0bkzMmG0qmy4p5aPwVDbOuZpOleyjNbPvga/1+yp6XQlj2D2VjXPOJVTBnISzgEfilPqphHHq\nJXgqG+ecCyo7YSGuNVJW10LeUtl4oHXOFQ5l1Udb43igdc4VDJG5H7Ym8kDrnCsoHmidcy6XVCU3\nwwpOuYE2TmUrl5nNq/rqOOeKmaBW5gxL16KdyB9nQiSeGyG9jHPOVamiatGa2QblbXPOuZwo5gwL\nknrF/FeJDAfZJaF3zrkKUC3NsJAx0Eq6g5CR9rhYtBC4J5eVcs4VrzolSvuoibIZdbCzmXWQ9B6s\nmJpWL8f1cs4VqRraaE0rm0C7RFIJcSkwSWsREhU651yVkqixrdZ0sumjvRN4Elhb0j+BN4Drc1or\n51zRqszqXQCSvpT0kaT3JU2IZWtKGiHp8/j/ZrFckvpLmiLpQ0kdks5zfNz/85g/LFHeMZ5/Sjw2\nY6UyBlozG0hIUHYTMJuQQntwxlfrnHMVJOINsTT/ZWlPM2uftG5tjcgZVgdYAiyuwDHOOVcxSn8j\nrBLdCoWdM0zSJcCjwHqE1cT/K+miCr1E55zLkpT+QfpUNhDuJ70s6Z2kbXnNGZbNzbBjgY5mthBA\n0tXAO8C1WRzrnHNZE1ndDEuXygZgFzP7VlILYISkSRkumarKc4Zl0w0wnZUDcilh1XLnnKtylZ2w\nYGbfxv//AAwl9LHOjF/7qUDOsPLKqy5nmKRbJd1CmKAwMabt/Q/wETA304mdc66iEsO7VrWPVtLq\nkhonfibk+vqYAs4Z9nH8/0TghaTysZlO6pxzq6qSo2hbAkNjy7cU+K+ZvSRpPIWYM8zM7qvIq3PO\nucrKso+2XGY2Fdi2jPKfKOScYZI2Aa4GtgTqJ11ss4pcyDnnMlLtTGWTzc2wB4EHCB823YHHAZ+w\n4JzLiaJcvQtoaGbDAczsCzO7lLCal3POValE10Exrt71W7y79oWkU4Fv+H2wr3POVamaGUrTyybQ\n/h/QCPgLoa92DeCkXFbKOVecauvqXRkDrZm9HX+cz++LfzvnXE7U1H7YdNJlwR1KmqllZnZITmrk\nnCtaoub2w6aTrkV7R7XVwlXYdltsyJtv+6+osn5euCTfVagVli3PON0/O78vHFOrpJuw8Gp1VsQ5\n5wDq1MJIm83NMOecqxaVnRlWqHwRb+dcQSlR+kc2JNWR9J6k5+PzNpLejmlpHkskmJW0Wnw+JW5v\nnXSOi2L5ZEn7JpV3i2VTJF2Yeu0yX1O2L17Satnu65xzq6Kyq3clORv4NOn59cCtMZXNHKBPLO8D\nzDGzTYFb435I2hLoBWxFSFVzVwzedQh5FLsTliU4Ku6bVjYZFnaQ9BHweXy+raTbs3mlzjlXUVlk\nWMhwvNYH9gcGxOcC9gKGxF1SU9kkUtwMAbrG/XsCg83sNzObRljda4f4mGJmU81sMWE5gp6Z6pRN\ni7Y/0AP4CcDMPsCn4DrnckBAqZT2QeZUNv8CzgeWx+drAXPNbGl8npx+ZkXKmrj957h/RVPcpJXN\nzbASM5ueMoh4WRbHOedchWXRai03lY2kHsAPZvaOpC6J4jJ2tQzbyisvq3GacWxbNoH2a0k7ABb7\nJ84CPsviOOecqxCp0hMWdgEOlLQfYVnXJoQWblNJpbHVmpx+JpGyZoakUsISA7MpP5UNacrLlU3X\nwWnAX4ENgZlA51jmnHNVrjKjDszsIjNb38xaE25mjTSzY4DXgMPibqmpbBIpbg6L+1ss7xVHJbQB\n2gLjCBkX2sZRDPXiNZ7N9JqyWevgh3gy55zLqRyOo70AGCzpKuA9IJFB5j7gYUlTCC3ZXgBmNlHS\n48AnwFLgDDNbBiDpTEJOsTrA/WY2MdPFs8mw8B/K6IMws9QOaOecqxxV3cwwMxsFjIo/TyWMGEjd\nZxG/5w9L3XY1YcXC1PJhhFxjWcumj/aVpJ/rAwez8l0355yrEiL7SQk1STZdB48lP5f0MDAiZzVy\nzhW12jgFd1XWOmgDbFTVFXHOuaJt0Uqaw+99tCWEDuOs5vc651yFFGOGhTgVbVtCnjCA5XHog3PO\nVbna2qJNO442BtWhZrYsPjzIOudySNRR+kdNlM2EhXGSOuS8Js65oicqv6hMIUqXMywxXW1X4GRJ\nXwC/EN4LMzMPvs65qiUorYV9B+n6aMcBHfh9OTHnnMup2pphIV2gFYCZfVFNdXHOuRrbPZBOukC7\ntqS/lrfRzG7JQX2cc0VMlZyCK6k+8D9gNUJ8G2Jml8WFYQYDawLvAseZ2eKYOWYg0JGw5vaRZvZl\nPNdFhAwMy4C/mNnwWN4NuI2w1sEAM7suU73S3QyrAzQCGpfzcM65KqcMjwx+A/Yys22B9kA3SZ3J\ncyqbdC3a78zsisyvyznnqoaoXIs2DkFdEJ/WjQ8jpLI5OpY/BFwO3E1IQ3N5LB8C3JGaygaYFlf3\nSixKMyUuUoOkRCqbT9LVK12Lthb2lDjnCl0Ww7vSprKJLc/3gR8I67J8QQGnsuma6WDnnKtKIqtJ\nCeWmsgGI68a2l9QUGApsUdZuKy5Z9rbqSWVjZrMzHeycc1VNVTTswMzmShpFyApT8KlsnHOueghK\npLSPtIdLa8eWLJIaAHsDn1LoqWycc666VPZmGLAu8FAcHVACPG5mz0v6hEJOZeOcc9WpMmHWzD4E\ntiujvOBT2TjnXLWoghZtQfJA65wrKLUwznqgdc4Vksw3vGoiD7TOuYIhoKQWzpXy4V2uyixbtozO\nnbbjkJ49AOjaZTd27NieHTu2p82G63H4ob7iZsLPc+fS57gj2bXT1uy2/Z+YMG4s1191GXvu3IGu\nu3biyIP24/vvwvDMN18fTdsNmtN110503bUTN19/1Yrz/Ofu29mjc3t233Fb7r2rf75eTtURlJSk\nf9RE3qJ1VeaO/rfRbostmD9vHgCvjnp9xbZeRxzKAQf0zFfVCs6lF/6Vvfbel/sefozFixfz68KF\ntNt8Sy649J8ADLjnDm65/mojcobhAAAZSUlEQVRu+NedAOy4064Mevzplc7x6ScfM+ih+3hx5FvU\nq1ePow7pwd77dmfjTdpW++upSvIWrXNlmzFjBi+9+AInntT3D9vmz5/P6NdGckBPb9ECzJ83j7Fv\nvsHRvU8EoF69eqzRtCmNmzRZsc/CX37JeFfo88mT6NhpRxo2bEhpaSk77bobw557Ju0xhS4x6qAY\nc4Y5l9F5557D1dfeQEkZ3+2efXooXfbqSpOkQFLMpn85lbWaN+fs0/uy967b89czT+GXX34B4Nor\n/k6HLTfmySce5fxLLltxzDvjxrLXLh056tADmPRpGB+/+ZZbMfat15k9+ycWLlzIqy+/xLffzMjL\na6pKlZkZVqhqdKCVtEzS+5I+lvRcYupdDq/XSVLGjjBJl0iaKOnDWL8dM+z/oKTD4s+7xWPfj1MI\nC96wF56nxdot6NCxY5nbH3/sUY448qhqrlXhWrp0GR998B4n9DmFV94YT8PVV+eOW28A4KJ/XMm7\nn0zl0MOP4v577wJgm223Y8LHUxj55jv0OeV0Tjw6jK/frN0WnHnOeRzZsztHH9qDrbbehtLSmt0b\nmEg3nu5RE9XoQAv8ambtzWxrwvS5M3J5MTObYGZ/SbePpJ2AHkAHM9uGMNf663THpDgGuCm+rl9X\nvbbVZ8xbb/L888/SbtPW9D6mF6NeG8mJvY8F4KeffmLC+HF032//PNeycKzXqhXrtlqfDp3CRKUe\nPQ/hww/eX2mfgw/vxQvPDgWgcZMmrN6oEQB779OdJUuX8NNPswA4uveJjHh9HE+/OJKmzZrRZuNN\nq/GV5ECG1qy3aPNvDEnrQko6T9L42Kr8ZyxrLWmSpAGxFfyIpL0lvSnpc0k7xP12kPSWpPfi/9vF\n8i6Sno8/Xy7pfkmjJE2VlAjA6xKWcfsNwMxmmdm38ZiOkkZLekfScEnrJr8ASX2BI4B/SHokp+9W\nFbry6mv54ssZTJ7yJQMfGUyXPffigYGDAHhqyBN0368H9evXz3MtC0eLluvQqtX6TPl8MgCvjx7J\nZu22YOoXn6/YZ/iLz7Np23YA/DDze8I6J/DuO+Ox5ctZc821APjxxx8AmPH1Vwx77mkOPuzI6nwp\nOVGZDAuSNpD0mqRP4zfDs2P5mpJGxL/zEZKaxXJJ6i9pSowVHZLOdXzc/3NJxyeVd5T0UTymv7JY\nbqxmf8+I4gISXYkLRUjah7Dazg6E382zknYHvgI2Jcxt7kdYiedoQkr1A4GLCVl/JwG7m9lSSXsD\n1wCHlnHpzYE9Cal9Jku6G3iZECg/A14BHjOz0ZLqArcDPc3sR0lHEuZRn5Q4mZkNkLQr8LyZDSnj\ndfaL9WaDDTdc1berWj3x+GD+dv6F+a5Gwbn6hls5ve/xLFmymI1at+Ffdw7g3LNOYcqUzygpKWH9\nDTbkhlvDiIPnnnmKh+77N6WlpdSv34B77h+0YinBvscdyezZP1G3bl2uvak/TZs1y+fLqrQqmIK7\nFDjXzN6V1Bh4R9II4ATgVTO7TtKFwIXABYSUNG3jY0dC1oUdJa0JXAZ0Iqw3+46kZ81sTtynHzCW\nsOZBN+DFdJWq6YG2QVxJvTXwDmE1dYB94uO9+LwR4Y38CphmZh8BSJpIePNN0kfxPBDWpHxIUlvC\nm1y3nOu/EFuuv0n6AWhpZjMkdQR2IwThx+IvdgKwNTAi/pHUAb6ryIs1s3uBewE6duyUcbHhfNh9\njy7svkeXFc9ffnVU3upSyLbepj0vjx67Utl9gx4vc98+/U6nT7/Ty9z2zEuvVXnd8q4ScdbMviP+\nXZnZfEmfEr7p9gS6xN0eAkYRAm1PYGBcGnGspKbxm2YXYERiXe4YrLvF9W2bmNmYWD6Q0Dir1YH2\nVzNrL2kN4HlCH21/wq/qWjP7d/LOkloTkrclLE96vpzf348rgdfM7OB4zKhyrp98rmWJ4+NyaqOA\nUTGAH0/4IJhoZjtV8DU6V1Sqqh82/u1uB7xNaAQlAvB3klrE3SqasqZV/Dm1PK1a0UdrZj8DfwH+\nFr+iDwdOktQIQFKrpDc2G2sA38SfT6hIXSS1iy3hhPbAdGAyIYX7TnG/upK2qsi5nSsGWfTRps0Z\nBhD/9p8EzjGzeRkulypdKpvyytOq6S3aFczsPUkfAL3M7GFJWwBj4tf0BcCxhFZnNm4gdB38FRhZ\nwao0Am5XGGq2FJgC9Is55A8D+scWeCnwLyDjosHOFQuRVSqbtDnDYmPrSeARM3sqFs+UtG5sza5L\nSNwI5aesmcHvXQ2J8lGxfP0y9k9LibuZrmbp2LGTvfn2hHxXo8b7eeGSfFehVthnj8588N47lf7O\nv+U229mgZ0en3adjmzXeKS/QxhEADwGzzeycpPIbgZ+SboataWbnS9ofOBPYj3AzrL+Z7RBvhr0D\nJEYhvAt0NLPZksYDZxG6JIYBt8fFwMtVa1q0zrlaonLhehfgOOCjeKMcwmii64DHJfUh3BRPZFUY\nRgiyU4CFwIkQktNKupIwMgngiqSEtacBDwINCDfB0t4IAw+0zrmCUrlJCWb2BuWH6q5l7G+UM9HJ\nzO4H7i+jPDGCKGseaJ1zBSObSQk1kQda51xByeJmWI3jgdY5V1BqYZz1QOucKyDyQOucczlXGzMs\neKB1zhWMxHq0tY0HWudcQfGbYc45l2O1MM56oHXOFRYPtM45l0NhwkLti7QeaJ1zhaMGJ2BMp1as\nR+ucq0UqkzQMiLn8fpD0cVJZXnOGeaB1zhWQKsmC+yAhj1eyCwlpq9oCr8bnsHLOsH6EfGAk5Qzb\nkZB78LJEcOb3nGGJ41Kv9QceaJ1zBSNTYzabMGtm/wNmpxT3JKxTS/z/QUnlAy0YCyRyhu1LzBkW\nEzImcoatS8wZFlf+Gph0rnJ5H61zrqBk8U28uaTkVe/vjYlL08lrzjAPtM65gpLFzbC0qWwqqFpy\nhnnXgXOucMRFZdI9VtHM+LWfCuQMK6+8wjnDPNA65wpGIjljuscqehZIjBw4Hngmqbx3HH3QGfg5\ndjEMB/aR1CzeBNsHGB63zZfUOY426J10rnJ514FzrqBUdhitpEcJGWybS5pBGD3gOcOccy6hMjnD\nAMzsqHI2ec4w55wDamXSMA+0zrmCoVo6BdcDrXOuoPiiMs45l2O+TKJzzuVU1usZ1CgeaJ1zBSOM\no813LaqeB1rnXEHxQOuccznmN8Occy6HfHiXc85VBw+0zjmXWz7qwDnncqz2hVkPtM65AlOJpRAL\nlsLiNa6mkfQjMD3f9cigOTAr35WoBWrC+7iRma1d2ZNIeonwetOZZWYZEyIWEg+0LmckTajClCNF\ny9/Hms8zLDjnXI55oHXOuRzzQOtyKVMKaJcdfx9rOO+jdc65HPMWrXPO5ZgHWuecyzEPtK5GkrSj\npE3yXQ/nsuGB1tU4kkqAPwNDJbXJd30KmaRDJV2T9Lz2TbuqATzQuhpFksxsuZldBbwMPChpw3zX\nqxBJaglcAoyUtD6A+d3vvPBA62qURKCQdDJQD1gdGOTdCCuLrf75wEjgEOBhSQ3yW6vi5YHW1TiS\ndgfOA24AjgdeAgZ6yzaQtCWwl5ktBJYAxwFDzezX/NaseHmgdQWvjH7FxcAbZjYDmAzcBcwBnvFg\nC4QW7ImS9gbGAGcDW0s6VtKa+a1acfJA6wpa7JNNdBesEYs/BXaRdKqZLTWzucBbwDsU8b/pxAdS\n7L+eCPQEFpjZ/cArQDfgz5LWyl8ti5OvR+sKWlKQPZ0QJL4GxgGHA4PjTZ6fgO7AoWb2Q94qm0fJ\nH0gAZnaNpL8CvSUtN7PHJS0DegNLJQ01s+V5q3CR8Sm4ruBJ6g2cAPQl9Mv+YmbHS2oHnExYlP9B\nM/sof7XMn5RW//HAasCvZvZwDLZbAIPMbLSknsB4M/s2j1UuOt6idQVHUklKa6sucAZh7GwToJek\nOsBcM/tbPupYSJKC7DmE7oLbgBskLTWzWyRdAJwuaZmZPZPPuharou3PcoUpMU42/nxcvLm1BHgT\nONjM9jGzpUAf4FRJq+WxunklaS1J9eLPqwPbmtmewOaEm4RPApjZ9YT374t81bXYeaB1BSXlK/C9\nwKHAIELQmC9pzTiG9kzgCTP7LW+VzSNJ+wMPAZvEYCtgTUmPANsDh5vZYkknS2pvZv3N7Lt81rmY\neaB1BSG5ZRq/AvcFbgdaxBbuZcAPwAOEr8dHmdkn+ahrvknaD7gG+LeZfWpmi81sAfACsDdwnZkt\nin3bZwOz81hdh/fRugIgaRvgYElXAhsQvvoeCtQHrgKIN2/OiPuvVsQt2SaEG4Nnm9koSY0I/dat\nCGNm/woMkTQM2BE40sy+yld9XeCB1uVVbMl+BMwAtgEWAOfEFlkHwh1zJPUFdiWMMlicp+rmnZnN\nk7QQ2EbSR8DlwGZAS+Br4FJgf2AZMD9O6nB55l0HLm8kbQVcAWxlZrMJQeIyoHkcfD8NmBK/AvcF\nbjSzJcW6MErSDLkngD0JN7xWB+4gzAZ7H9jFzD6OXQoeZAuEj6N1eRMnG1xLCKj/Br4l3ACrA/zd\nzL6RNB5YEzigiPtklfrhIqkZsI2ZjU4quwH4zcz+Xt11dOl5oHXVLmmqqEnaAPgH4YbNHYQuhPuA\n5YQbPvsAw81sWp6qm1cpkxFOAbYFPjezW1P2OwY4l3CTcHL119Sl410HrlolAkcMsvXM7Gvg70BT\n4CzCTZ0+QDPgVOD+Yg2yySQdTnhfXgHOlHSZgpZxuNtFQG8PsoXJW7Su2qS0zvoQxnuOAYYRbt5c\nDcwF7gG+AtYt1qmicSTG52b2q6RdCK3VB8zsOUmtgcHAc8AtwCbAPB9dULi8ReuqTVKQPY0wROlp\n4G/AP4F1gIsJw7tOAuoUcZBtSOgyaRyL6hLWL9hf0gZm9iVwJGGBmLPizS8PsgXMA62rVrE1tiVh\nCNImwDzCkK6/AmsTZnzdHafZFqW4YPe/gHUkDSK0+q8gTEU+RFIrM5tOmJzweP5q6rLlgdblVOqi\n3bE1dgkhyB5kZrsQvgZ3IbTQFprZ99VczYKQ/F7FD5rFwK+E7oH3gCHAxoSlD9c1s6/j++kKnAda\nl1NJ3QXHSbpIUntCACkB1ou7tQTeBvqbWVFORkjpv24tqamZTSJ0q5QA/Qkt2+cINwqLcmZcTeU3\nw1xOpASOQ4ALgQ8IQWO0mQ2UNJAw86sBYaroxLxVuEBIOgs4jJBFYo6ZXSRpPUL/dVPgRKDU83/V\nLD4F11W5lCC7DmHI1glm9omkk4AdJS0F+gFtgVm+slRo9QNHAAcR+mSPl7SOmZ0YJyOcDaxVrF0r\nNZkHWlflUhai7k34qrspYSGU+yUZ4a76cjMbnL+a5lfKB1IdYCZhMZ1eQGvC8LeXJD0Qg+35ZrYs\nbxV2q8wDrcsJSXsQFoHZm3ADZ5ikGWZ2o5k9IGkJMCqfdcynlCDbEFhsZi/H1bl2By6OU5BHAZ1i\ny9ZbsjWUB1pXJVICx87A0YThWpjZBEldCcG2oZn908wG5bG6eZf0Xp0P7AV8J+l+M3tdIYlix/hh\nlVjnYVYeq+sqyUcduEpLCbInEMbIjiTM7jo8tsY+Ag4Ejo5ZElTuCWux5Ncdb3LtSFjqcCxwe5wR\ndhfQETgYuN6DbM3nLVpXaUlBdgdC2u8T4tRRATuHTXrWzN6T9CcfwgWSDiJM3HjfzMYCY2NL9gHg\ndDM7o5gXOK9tPNC6SpOUGBN7GeGr7mbAB2Y2OAaP7sASSQ8SZjcVpaQgexhhuNvXQFNJrwFjzWxA\nXAj9Zkn7ECYruFrAx9G6VVLOGqlbEgbYvwY8m1h4WtLBwFtmNrP6a1pYYpA9HTjEzOZKupqQiuZx\nYIyZLZW0hpn9nNeKuirlfbSuwlK+AveT9ICke4BGhOX69gB6SNoIwMyGFmuQLaMvehlhVEGv+PwK\n4GfCEog7AHiQrX080LoKSwqyxxMCxGDCXPxhwLqELoQDgK6SirZ7KuUDaQtJLc1sKGFCwhmSDo99\nsFcBXwBT81hdl0PedeCyJmlz4E9m9kR8fgHwvZk9FJ/vT7iDvivQGZhiZt/kqboFQ9J5QFfCSmWT\nCGnUdyZ8IN1qZg/nsXquGniL1mUUV/IvBfYltFIPTWwCeiT2M7MXgE+AJmY2uhiDbGpXgaQ9gT+b\nWbdYtAlhyvEzhFQ9p0hqXKzD3YqFB1qXjZK4bN+jhNTgXSR1A24AWkt6SCGlynHAVoTkisWqXsrz\n1YDXYqu2MXBSTOPT3syGAN3MbH7qjUVXu3jXgUsrzujqTFgLdUos7kdYdesZwjTawYSvxZsBpxbr\nKlxxSNZphLTfH5vZk7G75U5CloSuZrZE0l+APxNWLFuYvxq76lK0Nypc1i4G9iTcKf8M+AEYQWid\ndSGktz4cQFJjM5ufp3rmVWzh/xMYCLQAukl6lzBW9k3C+3WBpNmEG4jHeJAtHt6idRlJepTQJbA/\nITX4QkLf7DfAGoRVuUaVNba2GEhaE5gF9IzJE9cnJJq8x8zGSFod2I3wgWXAQDP7JH81dtXNA637\ng3ImI4wCvjaz4+LzY4DNCYvH7G1FnhI8jri4AdjJzOZJGgY0BD4EJgOPmdmsYv0wKnYeaN1KUsZ+\n7kK4sTXWzBZLGknI6dUjaf96xbp2QSpJ3QkpZ14i9GH/mzAluS/hJuI5ZjYvfzV0+eKB1pUp3iXf\nD5hBSKFytplNlTQCqG9mu8X9vIWWRNLewMvAuonZcHEtiDV9Fa7i5cO73ErimNntgF3MbE/gy7hp\nBoCZ/RmYK2mD+NyDbBIze4W4TKSklrFsuQfZ4uYtWoektvHH78xsgaTNgFMIo1I2I9zkWSyph5k9\nn7eK1iCSehJmfnUys+X5ro/LL2/RFrk4LOkFwrJ9z0hqTFjkZG2gPXBiDLJ9gcslrZ2/2tYccebX\n7h5kHXiLtqhJ6gGcT1hB6i3gFuAKM/tW0l6EEQWrEYZxHQAcUayTEZyrDA+0RSjOq28MfAc8YGZn\nStqEMBTpSWBboCewlDApoQHwmplNKfuMzrl0PNAWMUm7E6bP3kYYTD/CzG6Oi1GfCLQr1plezlUl\nD7RFJjEcS1KJmS2PwfZ54BUzOyRpvyHApWY2KW+Vda6W8JthRSRlzOu6klqY2f8IrdndJJ0Y9zuW\nMOB+Tp6q6lyt4i3aIiFpIzObHn/+G7A30JKwCMp/gfUJLdv/xfIz/MaXc1XDA20RSJoauithXOwt\nhEVOdiOs9I+Z/VNSF+BhYD8z+yg/tXWu9vFlEms5SfsCNwPHmdlMSTsCP5rZImCEpLnAnZJeiitw\ntY3bnHNVxPtoa7G4EPVAQnqZubH4bcIU2oMBzGw88C6wXtz+W3XX07nazrsOaqmYGeFuwmLULePj\nWTN7U9JphCUOSwnZay8C9kr04TrnqpYH2lpK0vZAXTN7S1I74FhCPqv/AhOBjsChhASLD/hC1M7l\njgfaWi5pvGxb4DjClNohscsASaUx8aJzLkc80BaRGGyPBpoD/zWzMXmuknNFwW+GFREz+xx4DPiW\n3zPaOudyzFu0RUhSXTNbku96OFcsPNA651yOedeBc87lmAda55zLMQ+0zjmXYx5onXMuxzzQOudc\njnmgdc65HPNA66qVpGWS3pf0saQnJDWsxLm6SHo+/nygpAvT7NtU0umrcI3L40LpWZWn7POgpMMq\ncK3Wkj6uaB1d4fNA66rbr2bW3sy2BhYDpyZvVFDhf5dm9qyZXZdml6ZAhQOtc1XBA63Lp9eBTWNL\n7lNJdxHWxt1A0j6Sxkh6N7Z8GwFI6iZpkqQ3gORkkidIuiP+3FLSUEkfxMfOwHXAJrE1fWPc7zxJ\n4yV9KOmfSee6RNJkSa8A7TK9CEknx/N8IOnJlFb63pJel/SZpB5x/zqSbky69imVfSNdYfNA6/JC\nUinQHUikzGkHDDSz7YBfgEuBvc2sAzAB+Kuk+sB/gAMIaXjWKef0/YHRZrYt0IGwLOSFwBexNX1e\nXBS9LbAD0B7oKGl3SR2BXsB2hEC+fRYv5ykz2z5e71OgT9K21sAewP7APfE19AF+NrPt4/lPltQm\ni+u4GspT2bjq1kDS+/Hn14H7CNkdppvZ2FjeGdgSeFMShHV0xxAWK58WF8dB0iCgXxnX2AvoDWBm\ny4CfJTVL2Wef+HgvPm9ECLyNgaFmtjBe49ksXtPWkq4idE80AoYnbXvczJYDn0uaGl/DPsA2Sf23\na8Rrf5bFtVwN5IHWVbdfzax9ckEMpr8kFwEjzOyolP3aA1W1OIeAa83s3ynXOGcVrvEgcJCZfSDp\nBKBL0rbUc1m89llmlhyQkdS6gtd1NYR3HbhCNBbYRdKmAJIaStoMmAS0kbRJ3O+oco5/FTgtHltH\nUhNgPqG1mjAcOCmp77eVpBaEdOsHS2ogqTGhmyKTxsB3kuoCx6RsO1xSSazzxsDkeO3T4v5I2kzS\n6llcx9VQ3qJ1BcfMfowtw0clrRaLLzWzzyT1A16QNAt4A9i6jFOcDdwrqQ+wDDjNzMZIejMOn3ox\n9tNuAYyJLeoFwLFm9q6kx4D3gemE7o1M/k5Iejmd0OecHNAnA6MJOdtONbNFkgYQ+m7fVbj4j8BB\n2b07ribyZRKdcy7HvOvAOedyzAOtc87lmAda55zLMQ+0zjmXYx5onXMuxzzQOudcjnmgdc65HPt/\nvaHz43auIIMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85f42deb00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "classification_report.confusion_matrix(results, truth, lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHipJREFUeJzt3Xu4XEWd7vHvm4QQEAhKACUJBCGo\nkQGFiHjwAB4YDBxJmHkcJcIRFEEdGfE6ojNHI+ioqOPogKNRMIoIBhw9UYOoiOKFOImHiyaCRgQS\nASFcAhFCEvKbP6qarHS61+69u1fv3r3fz/P0s9elelV17e7+dVWtVUsRgZmZWTNjhrsAZmbW2xwo\nzMyslAOFmZmVcqAwM7NSDhRmZlbKgcLMzEo5UPQBScslHT1Amr0lrZM0tkvFqpykOyQdm5fnSfrq\ncJfJykk6WtLm/F6c1YX8fizpDXn5FEnfb+E5CyR9qGT/HyRtGE3vNweKCuUvssfzh+LPkr4kaadO\n5xMRz4+IHw+Q5q6I2Ckinux0/vlLemN+nQ9L+oWkl3Q6n3ZI2kXSv0m6K5dzZV6fNNxlayZ/yT0k\nafu67QvyF9U6SQ9K+oGk5w7h+G+XdK+ktZIuqc+nLu0bcp2tk/Q9SXsV9r1N0u2SHpF0t6RPSRpX\nkvXd+b34vcGWuR0RcVlEHNeB4+wH/EsHijRiOFBU78SI2Ak4BHgR8M/1CZSM9P/F1/PrnARcB1w5\nzOV5iqTxwLXA84FZwC7A/wAeAA4bwvHKvgQ7QtI04H8CAcxukOSCXN9TgPuABYM8/suBc4FjgGnA\ns4EPNkl7FOmLcQ7wDOCPwOWFJN8GDomIXYADgYOBtw6mPNbbRvqX04gREX8CriZ9kGq/Fj8s6efA\nY8CzJU2UdLGkeyT9SdKHil1Fks6U9FtJj0paIemQvL3YBXOYpGX5192fJf1r3j5NUtS+5CTtJWlR\n/kW6UtKZhXzmSVoo6Ss5r+WSZrb4OjcBlwGTJe1eOOYrJN1UaHEcVNg3VdJ/Srpf0gOSLszb95P0\no7xtjaTLJO06hOp/LbA38DcRsSIiNkfEfRFxfkQsznmFpP0LZXqq+yF3l6yW9B5J9wJfyv+HVxTS\nj8tlrP1PDs+v82FJN2uArsEmZV5CCgCnNUsUEY8BXyO/rwbhNODiiFgeEQ8B5wOnN0l7InBlTrsh\npz1S0n65DH+IiIdzWgGbgf0bH6oxSa/PdfqQpGsk7VPYF5LeJOn3ef9FkpT3jZX0yVz3f5R0dvF9\nXpfH6ZJ+lpeVWz735RbVLZKKdfh0Sd/N7/9f1l7raOVA0SWSpgInADcWNv8f4CxgZ+BO4MvAJtKH\n7IXAcUCtf/XvgHmkL5BdSL8yH2iQ1aeBT+dfd/sBC5sU6XJgNbAX8ErgXyQdU9g/G7gC2BVYBFzY\n4uscn8v4APBQ3nYIcAnwRmA34PPAIknb50D4nfz6pwGTc76QvnQ+ksv4PGBqroPBOhb4XkSsG8Jz\na55J+jW9D+l/djkwt7D/5cCaiPj/kiYD3wU+lJ/zLuAbtcAp6VxJ3xkgv9eSAu5lwMsl7dkokVJX\n5ink95Wk1+Tg1Oyxd37q84GbC4e6GdhT0m6NssmP4joUglPO9xFgDalF8fkBXl/xNZwEvA/4W2B3\n4Kds3WIBeAWpRX4w8CpSfQOcCRwPvIDUaj+pxWyPA44EDiC9x1/N1p+nuaQW1tOBlcCHW309fSki\n/KjoAdwBrAMeJn0RfhbYIe/7MXBeIe2ewBO1/XnbXOC6vHwNcE5JPsfm5etJb/BJdWmmkboxxpG+\ncJ8Edi7s/wiwIC/PA35Y2DcDeLzkdc4DNuTX+STpA3d0Yf9/AOfXPec24CjgJcD9wLgW6vMk4MYm\nr3se8NUmz/sB8NEBjh3A/oX1BcCH8vLR+fVNKOzfH3gU2DGvXwa8Py+/B7i07vjXAKe1+L55KbCx\n9j8EbgXeXle29bm+7yUF8v0G+d78AzCrsL5droNpDdIeQwoABwE7kILAZmBug7TTSS2OZzbJ92hg\ndd22q4EzCutjSK3sfQr/m5cW9i8Ezs3LPwLeWNh3bO19XvicvSEvnw78LC//L+B3wOHAmLryLAC+\nWFg/Abi1wXu+4futHx9uUVTvpIjYNSL2iYi/j4jHC/tWFZb3IX1Y76n9+iN9IPfI+6eSPtwDOYP0\nK+lWSUuL3SMFewEPRsSjhW13kn7N19xbWH4MmJC7V05RGtBcJ+nqQpqFEbErKeD9Bji07rW9s/jL\nNr+evfLfOyN1WW1F0h6SrlDqhnsE+CppDGSwHgCeNYTnFd0fEetrKxGxEvgtcKKkHUktsK/l3fsA\nf1f3el86iDKcBnw/Itbk9a+xbffTJ/L76pkRMTsiWnlvFK0jtUxrasuP1ieMiGuBDwDfIL1P7sjp\nVjdI+3tgOelHUav2AT5dqKsHSa2Wsvdj7aSQvdj6c1RcbioifkRqJV8E/FnSfEnF+miW36jkQDG8\nilP3riK1KCblL4BdI2KXiHh+Yf+A/aQR8fuImEsKMB8DrpL0tLpkdwPPkLRzYdvewJ9aOP5lkc5Y\n2Skijm+wfw2pi2mepNoX4yrgw4XXtWtE7BgRl+d9ezfqUya1cgI4KFJX2qls3QXSqh+Sum/q66Ho\nMWDHwvoz6/Y3mma51v00B1iRgwek13Rp3et9WkR8dKCCStqB1LVylNIZSfcCbwcOlnRwC88vBvJG\nj1rX03JSN07NwcCfI6JRdyYRcVFETI+IPUgBYxzpB0Ej42jhvVqwitQqKNbXDhHxixaeew9pQL9m\naquZRsRnIuJQUjfcAcC7B1HmUcWBokdExD3A94FPKp3KOUZpMPeonOSLwLskHZoH4vYvDvjVSDpV\n0u4RsZnUNQGpO6iY1yrgF8BHJE1QGlg+g9R90onXciupq+Uf86YvAG+S9OJc9qdJ+t85UP0X6cP+\n0bx9gqQj8vN2Jnfd5X7/oX6QLyV9GX1D0nNz3e4m6X2STshpbgJekwdHZ5G6xQZyBamv+81saU1A\navmcKOnl+XgTlAbEpzQ8ytZOIv2/ZpD63V9AGp/5KWncolRdIG/0uCsn/QpwhqQZkp5OOhtvQaNj\n5vIfmP93ewPzSeNgtTGoN0jaIy/PAN5LOsusVZ8D3ivp+fkYE/OYXCsWAudImqx0osN7WnmSpBfl\n9+N2wF9IXXkdP3W8XzhQ9JbXAuOBFaSB4KvI3RURcSVpQO1rpGb/t0gDpfVmAcslrSMNbJ9c7DIp\nmEsat7gb+CbwgYj4QQdfy8eBsyTtERHLSIOOF+bXtZJ8hk2k6zpOJPX530Xqznh1PsYHSQOUa0mD\nw/85lIJExBOkvutbSeMVj5AC1CTglznZObkcD5MGh7/VwnHvAW4gnWr79cL2VaRWxvtI4y+rSEFu\nDEAOUFdvc8DkNOBLka57ubf2INXdKU1aXoMW6RqGC0inMt+ZHx+o7Vc60+2UvDqB9L5bR6q3G4D/\nWzjcEcCvJf0FWJwf7xtEWb5Jav1ekbsYf0MaoG7FF0g/sG4hDegvJp0QMtCX/i75uQ+RXvsDwCda\nLfNoowjfuMjMukPSkaTW5hPAqyPimg4f/3jgcxGxTWu7g3ncRho/WRgRr68qn17iQGFmI1Ye03kZ\nqVWxJ2n8ZElEvG1YC9ZnHCjMbMTKZ5z9BHgu8Dipi/KciHhkWAvWZxwozMyslAezzcysVOWTm3Xa\npEmTYtq0acNdDDOzEeVXv/rVmojYfeCU2xpxgWLatGksW7ZsuIthZjaiSLpzqM9115OZmZVyoDAz\ns1IOFGZmVsqBwszMSjlQmJlZKQcKMzMrVVmgkHSJ0v1oG85Zn6cs/ozS/ZpvUb7XsJmZ9ZYqWxQL\nSFNeN3M86baJ00n3IP6PCstiZmZDVNkFdxFxvaRpJUnmAF+JNNnUEkm7SnpWnuO/qfXr4dFHYcyY\nxg8N5f5nZmbW1HBemT2Zre9vuzpv2yZQSDqL1Opgt92ezc9+loJCo/kMJRg3DsaO3fZv/bbaY+zY\n5oHHAcnMRrvhDBSNvmobTmUbEfNJt19k331nxjOekb7gG6eFzZu3/N28GZ54YtttxXT1AacsCNTS\nlgWkYgCqX3dAMrORZjgDxWq2vhH6FNJtOdsipS/kqnUiIBXLXKaVgFQfnByQzKxThjNQLALOlnQF\n8GJg7UDjE71kJAak+vIX8xgzZutAM1BAKraQJAcks35WWaCQdDlwNDBJ0mrSjdu3A4iIz5Fugn4C\nsBJ4DHhdVWUZyUZ6QKqRtg1Ixb+17a2MIbUSmByQzDqnyrOe5g6wP4C3VJW/Dc5wBaQnn4RNmzoX\nkGqaBaRGJze0MobULDg5INloMOLuR2Ej22gLSPXbtttucN10DkjWCxworC+N5oBUewzlhAYHJGvE\ngcKsDQ5IDkijgQOF2QjQrYBUDDIOSFbjQGFmTxkzpjv51AeckRyQulVnw8mBwsy6rltfsI0C0saN\nW7ea6gMSDD4o9XtAGnGBIgLuuqs7zXAzs1bUd9nVL2/enNLVB6SiVoJT8YLYMWO2Xq+/JmnsWJg0\nCaZPh+23b+/1jbhA4T5HM+s13eyyG2gMqZZu/fr0d4890qMdIy5QjB8Pe+/dfFJAMzODO++EBx/s\nzLFGwTCMmdnoNZSTAOo5UJiZ9anaHGvtcqAwM+tTnQgS4EBhZtbX3KIwM7Om3PVkZmYDqp0y2w4H\nCjOzPuYWhZmZNeWuJzMzG5C7nszMrCm3KMzMrJQDhZmZlZLc9WRmZgNwi8LMzEo5UJiZWSl3PZmZ\nWSm3KMzMrCmf9WRmZqUcKMzMbEAeozAzs6bcojAzswE5UJiZWVO+MtvMzAbU8y0KSbMk3SZppaRz\nG+zfW9J1km6UdIukE6osj5nZaCJ15jiVBQpJY4GLgOOBGcBcSTPqkv0zsDAiXgicDHy2qvKYmY1G\nvd71dBiwMiJuj4gNwBXAnLo0AeySlycCd1dYHjOzUafXA8VkYFVhfXXeVjQPOFXSamAx8A+NDiTp\nLEnLJC1bu/b+KspqZtZ3RsLpsY16x+qLPBdYEBFTgBOASyVtU6aImB8RMyNi5sSJu1dQVDOz/tTr\ngWI1MLWwPoVtu5bOABYCRMQNwARgUoVlMjMbVXq962kpMF3SvpLGkwarF9WluQs4BkDS80iBwn1L\nZmYd0PNnPUXEJuBs4Brgt6Szm5ZLOk/S7JzsncCZkm4GLgdOj+hEQ8nMzDo1RjGu/UM0FxGLSYPU\nxW3vLyyvAI6osgxmZqOVr8w2M7MB9fpgtpmZDaORcHqsmZkNM3c9mZlZKbcozMysKXc9mZnZgBwo\nzMyslMcozMysKXc9mZlZqZ6fwsPMzIZfRPutCgcKMzMr5UBhZtbHOjFO4UBhZtbH3PVkZmaVc6Aw\nM+tzblGYmVkpBwozM6uUA4WZWZ9zi8LMzJry6bFmZlbKp8eamVnlHCjMzPqYu57MzKyUu57MzKxy\nDhRmZn3OLQozMyvlQGFmZpVyoDAz63NuUZiZWVM+PdbMzEr59FgzM6vcuFYTSpoM7FN8TkRcX0Wh\nzMysMzrR9dRSoJD0MeDVwArgybw5gNJAIWkW8GlgLPDFiPhogzSvAubl490cEa9ptfBmZlauE11P\nrbYoTgKeExFPtHpgSWOBi4C/BlYDSyUtiogVhTTTgfcCR0TEQ5L2aL3oZmbWDa2OUdwObDfIYx8G\nrIyI2yNiA3AFMKcuzZnARRHxEEBE3DfIPMzMbADdalE8Btwk6VrgqVZFRLy15DmTgVWF9dXAi+vS\nHAAg6eek7ql5EfG9FstkZmYt6FagWJQfg6EG2+qLOw6YDhwNTAF+KunAiHh4qwNJZwFnAey5596D\nLIaZmbWjpUAREV+WNJ7cAgBui4iNAzxtNTC1sD4FuLtBmiX5WH+UdBspcCyty38+MB/gOc+Z2WZs\nNDMbXbpyHYWko4HfkwanPwv8TtKRAzxtKTBd0r45yJzMtq2SbwEvy3lMIgWi21suvZmZlera6bHA\nJ4HjIuK2lLEOAC4HDm32hIjYJOls4BrS+MMlEbFc0nnAsohYlPcdJ6l22u27I+KBob8cMzMr6ubp\nsdvVgkTKOH4nacCzoCJiMbC4btv7C8sBvCM/zMysB7UaKJZJuhi4NK+fAvyqmiKZmVmndLPr6c3A\nW4C3ks5mup40VmFmZj2sa11P+Yrsf80PMzMbRUoDhaSFEfEqSb9m22sgiIiDKiuZmZl1RNUtinPy\n31e0l42ZmQ2XSq+jiIh78uIaYFVE3AlsDxzMthfPmZlZj+nmHe6uBybke1JcC7wOWNBe1mZm1g3d\nChSKiMeAvwX+PSL+BpjRXtZmZlY1CTZvbu8YLQcKSS8hXT/x3byt5bvjmZnZ8OlWoHgb6QZD38zT\ncDwbuK69rM3MrGpdu+AuIn4C/KSwfjvp4jszM+thlQcKSf8WEW+T9G0aX0cxu73szcysau12PQ3U\noqjN7fSJ9rIxM7PhUHmLIiJqE/8tAx6PiM0pY40lXU9hZmY9rlunx14L7FhY3wH4YXtZm5lZN3Tr\nrKcJEbGutpKXdyxJb2ZmPUBq/xitBoq/SDpkS8Y6FHi8/ezNzKxK3bwfxduAKyXV5nd6FvDq9rI2\nM7OqdeLK7Favo1gq6bnAc0g3Lro1Ija2l7WZmXVDV8YoJO0IvAc4JyJ+DUyT5KnHzcx6XDdnj/0S\nsAF4SV5fDXyovazNzKwbuhUo9ouIC4CNKdN4nNQFZWZmPa5bp8dukLQDeRoPSfsBT7SXtZmZVa2b\nZz19APgeMFXSZcARwOntZW1mZlXrxHUUAwYKSQJuJd206HBSl9M5EbGm/ezNzKxqlZ8eGxEh6VsR\ncShbblpkZmYjQDfPeloi6UXtZWVmZt3WzTGKlwFvknQH8BdS91NExEHtZW9mZlXrypXZwPHtZWNm\nZsOl0kAhaQLwJmB/4NfAxRGxqb0szcysW7oxe+yXgZmkIHE88Mn2szQzs27pxhjFjIj4q5SZLgb+\nq73szMys26q+MvupGWLd5WRmNvJ04/TYgyU9kh+PAgfVliU9MnABNUvSbZJWSjq3JN0rJYWkmYN9\nAWZm1lzlXU8RMXaoB5Y0FrgI+GvSbLNLJS2KiBV16XYG3gr8cqh5mZlZc92aFHAoDgNWRsTtEbEB\nuAKY0yDd+cAFwPoKy2JmZkNUZaCYDKwqrK/O254i6YXA1Ij4TtmBJJ0laZmkZWvX3t/5kpqZ9alu\nTuExFI3O3n2quJLGAJ8C3jnQgSJifkTMjIiZEyfu3sEimpn1t07cM7vKQLEamFpYnwLcXVjfGTgQ\n+HGeGuRwYJEHtM3MOquXWxRLgemS9pU0HjgZWFTbGRFrI2JSREyLiGnAEmB2RCyrsExmZqNKT3c9\n5esuzgauAX4LLIyI5ZLOkzS7qnzNzGyLbs4eOyQRsRhYXLft/U3SHl1lWczMRquebVGYmVl/cKAw\nM+tz7c4g60BhZtbn3PVkZmaVcqAwM+tzqetp6P1PDhRmZn0udz0NOVI4UJiZWSkHCjOzPueznszM\nrJTPejIzs0o5UJiZjQo+68nMzCriQGFmZqUcKMzMRgdfR2FmZo359FgzMyuVTo/1YLaZmVXEgcLM\nrM+568nMzEr5ymwzM6uUA4WZ2ejg02PNzKwaDhRmZlbKgcLMbFTwdRRmZtaET481M7NSPj3WzMwq\n5UBhZtbncteTT481M7PGPCmgmZlVyoHCzMxKOVCYmVmpSgOFpFmSbpO0UtK5Dfa/Q9IKSbdIulbS\nPlWWx8zMBq+yQCFpLHARcDwwA5graUZdshuBmRFxEHAVcEFV5TEzs6GpskVxGLAyIm6PiA3AFcCc\nYoKIuC4iHsurS4ApFZbHzGxUSqfHDv367CoDxWRgVWF9dd7WzBnA1Y12SDpL0jJJy9auvb+DRTQz\n63+9fGV2o+jVsLiSTgVmAh9vtD8i5kfEzIiYOXHi7h0soplZ/2t3rqdxnSlGQ6uBqYX1KcDd9Ykk\nHQv8E3BURDxRYXnMzGwIqmxRLAWmS9pX0njgZGBRMYGkFwKfB2ZHxH0VlsXMbJTrwSuzI2ITcDZw\nDfBbYGFELJd0nqTZOdnHgZ2AKyXdJGlRk8OZmdkQtTvXU5VdT0TEYmBx3bb3F5aPrTJ/MzPz/SjM\nzKwlPdj1ZGZmvcEtCjMzK+X7UZiZWQvc9WRmZhVxoDAz63MeozAzs1IeozAzsxZ4jMLMzJpw15OZ\nmZVyoDAzsxb05o2LzMysDzhQmJn1OXc9mZlZqV6+Z7aZmfUBBwozsz7nriczMyvlQGFmZi3wGIWZ\nmVXEgcLMrM+12/U0rjPFMDOzXrN+Pdx5J2zY0N5xHCjMzPrQxIlbltu9jsKBwsysD+26a3oAbNzY\n3rE8RmFmZqUcKMzMrJQDhZmZlXKgMDOzUg4UZmZWyoHCzMxKOVCYmVkpBwozMyvlQGFmZqUcKMzM\nrFSlgULSLEm3SVop6dwG+7eX9PW8/5eSplVZHjMzG7zKAoWkscBFwPHADGCupBl1yc4AHoqI/YFP\nAR+rqjxmZjY0VU4KeBiwMiJuB5B0BTAHWFFIMweYl5evAi6UpIiIsgM/8QRs2tT5ApuZ9aN2vy+r\nDBSTgVWF9dXAi5uliYhNktYCuwFriokknQWcldc2HHXUzn+opsgjzcanw3YPDXcpeoPrYgvXxRau\niy3+ss9Qn1lloGg093l9S6GVNETEfGA+gKRlEY/ObL94I1+qi/WuC1wXRa6LLVwXW0haNtTnVjmY\nvRqYWlifAtzdLI2kccBE4MEKy2RmZoNUZaBYCkyXtK+k8cDJwKK6NIuA0/LyK4EfDTQ+YWZm3VVZ\n11MeczgbuAYYC1wSEcslnQcsi4hFwMXApZJWkloSJ7dw6PlVlXkEcl1s4brYwnWxhetiiyHXhfwD\n3szMyvjKbDMzK+VAYWZmpXo2UHj6jy1aqIt3SFoh6RZJ10oa8vnSvW6guiike6WkkNS3p0a2UheS\nXpXfG8slfa3bZeyWFj4je0u6TtKN+XNywnCUs2qSLpF0n6TfNNkvSZ/J9XSLpENaOnBE9NyDNPj9\nB+DZwHjgZmBGXZq/Bz6Xl08Gvj7c5R7GungZsGNefvNoroucbmfgemAJMHO4yz2M74vpwI3A0/P6\nHsNd7mGsi/nAm/PyDOCO4S53RXVxJHAI8Jsm+08AriZdw3Y48MtWjturLYqnpv+IiA1AbfqPojnA\nl/PyVcAxkhpdwDfSDVgXEXFdRDyWV5eQrlnpR628LwDOBy4A1nezcF3WSl2cCVwUEQ8BRMR9XS5j\nt7RSFwHskpcnsu01XX0hIq6n/Fq0OcBXIlkC7CrpWQMdt1cDRaPpPyY3SxMRm4Da9B/9ppW6KDqD\n9IuhHw1YF5JeCEyNiO90s2DDoJX3xQHAAZJ+LmmJpFldK113tVIX84BTJa0GFgP/0J2i9ZzBfp8A\n1U7h0Y6OTf/RB1p+nZJOBWYCR1VaouFTWheSxpBmIT69WwUaRq28L8aRup+OJrUyfyrpwIh4uOKy\ndVsrdTEXWBARn5T0EtL1WwdGxObqi9dThvS92astCk//sUUrdYGkY4F/AmZHxBNdKlu3DVQXOwMH\nAj+WdAepD3ZRnw5ot/oZ+X8RsTEi/gjcRgoc/aaVujgDWAgQETcAE4BJXSldb2np+6RerwYKT/+x\nxYB1kbtbPk8KEv3aDw0D1EVErI2ISRExLSKmkcZrZkfEkCdD62GtfEa+RTrRAUmTSF1Rt3e1lN3R\nSl3cBRwDIOl5pEBxf1dL2RsWAa/NZz8dDqyNiHsGelJPdj1FddN/jDgt1sXHgZ2AK/N4/l0RMXvY\nCl2RFutiVGixLq4BjpO0AngSeHdEPDB8pa5Gi3XxTuALkt5O6mo5vR9/WEq6nNTVOCmPx3wA2A4g\nIj5HGp85AVgJPAa8rqXj9mFdmZlZB/Vq15OZmfUIBwozMyvlQGFmZqUcKMzMrJQDhZmZlXKgMKsj\n6UlJN0n6jaRvS9q1w8c/XdKFeXmepHd18vhmneZAYbatxyPiBRFxIOkanbcMd4HMhpMDhVm5GyhM\nmibp3ZKW5rn8P1jY/tq87WZJl+ZtJ+Z7pdwo6YeS9hyG8pu1rSevzDbrBZLGkqZ9uDivH0eaK+kw\n0uRqiyQdCTxAmmfriIhYI+kZ+RA/Aw6PiJD0BuAfSVcIm40oDhRm29pB0k3ANOBXwA/y9uPy48a8\nvhMpcBwMXBURawAiojY55RTg63m+//HAH7tSerMOc9eT2bYej4gXAPuQvuBrYxQCPpLHL14QEftH\nxMV5e6O5cP4duDAi/gp4I2kiOrMRx4HCrImIWAu8FXiXpO1Ik869XtJOAJImS9oDuBZ4laTd8vZa\n19NE4E95+TTMRih3PZmViIgbJd0MnBwRl+Ypqm/Is/SuA07NM5V+GPiJpCdJXVOnk+6qdqWkP5Gm\nPN93OF6DWbs8e6yZmZVy15OZmZVyoDAzs1IOFGZmVsqBwszMSjlQmJlZKQcKMzMr5UBhZmal/hsF\nufOMtTjelgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85f4306f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "classification_report.pr_curve(results, truth, lang)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Overall Accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = accuracy_score(truth['after'].tolist(),\n",
    "                       results['after'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on english language is 0.99427\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy on {} language is {:.5f}'.format(lang, score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Semiotic class-wise accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_group = results.groupby('semiotic')\n",
    "truth_group = truth.groupby('semiotic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_accuracy = pd.DataFrame(columns=['semiotic-class', 'accuracy', 'count', 'correct'])\n",
    "row = {'semiotic-class': 'ALL',\n",
    "       'accuracy': score,\n",
    "       'count': results.shape[0],\n",
    "       'correct': score*results.shape[0]}\n",
    "class_accuracy = class_accuracy.append(row, ignore_index=True)\n",
    "\n",
    "for results_items, truth_items in zip(results_group, truth_group):\n",
    "    semiotic_class = results_items[0]\n",
    "    results_items = results_items[1]\n",
    "    truth_items = truth_items[1]\n",
    "    score = accuracy_score(truth_items['after'].tolist(),\n",
    "                          results_items['after'].tolist())\n",
    "    row = {'semiotic-class': semiotic_class,\n",
    "           'accuracy': score,\n",
    "           'count': results_items.shape[0],\n",
    "           'correct': score*results_items.shape[0]}\n",
    "    class_accuracy = class_accuracy.append(row, ignore_index=True)\n",
    "class_accuracy['correct'] = class_accuracy['correct'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.994267</td>\n",
       "      <td>92451</td>\n",
       "      <td>91921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADDRESS</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.994214</td>\n",
       "      <td>1037</td>\n",
       "      <td>1031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.997175</td>\n",
       "      <td>2832</td>\n",
       "      <td>2824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>0.989130</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>44</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>0.734694</td>\n",
       "      <td>49</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.971611</td>\n",
       "      <td>1409</td>\n",
       "      <td>1369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>142</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.980583</td>\n",
       "      <td>103</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.993961</td>\n",
       "      <td>67894</td>\n",
       "      <td>67484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.998873</td>\n",
       "      <td>17746</td>\n",
       "      <td>17726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.994006</td>\n",
       "      <td>1001</td>\n",
       "      <td>995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  accuracy  count  correct\n",
       "0             ALL  0.994267  92451    91921\n",
       "1         ADDRESS  1.000000      4        4\n",
       "2        CARDINAL  0.994214   1037     1031\n",
       "3            DATE  0.997175   2832     2824\n",
       "4         DECIMAL  0.989130     92       91\n",
       "5           DIGIT  0.795455     44       35\n",
       "6      ELECTRONIC  0.734694     49       36\n",
       "7        FRACTION  0.687500     16       11\n",
       "8         LETTERS  0.971611   1409     1369\n",
       "9         MEASURE  0.971831    142      138\n",
       "10          MONEY  0.972973     37       36\n",
       "11        ORDINAL  0.980583    103      101\n",
       "12          PLAIN  0.993961  67894    67484\n",
       "13          PUNCT  0.998873  17746    17726\n",
       "14      TELEPHONE  0.918919     37       34\n",
       "15           TIME  0.750000      8        6\n",
       "16       VERBATIM  0.994006   1001      995"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/metal_geek/anaconda3/envs/deep-tf/lib/python3.6/site-packages/pandas/plotting/_core.py:1714: UserWarning: Pandas doesn't allow columns to be created via a new attribute name - see https://pandas.pydata.org/pandas-docs/stable/indexing.html#attribute-access\n",
      "  series.name = label\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJQAAAKRCAYAAAACt/PMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3X+0X3V95/vXmwRMIBGEIBcMNkzL\n9QdcfplBR7EmjQPBtuIaYZaWQhBdyFwFO9OOpnXmykwFtVp/UFtdTEF+SJsy1EFatApCOoUiApVR\nU0RQUSP+oIkioETQz/3juw8e4knO+eSckBAfj7XOyvnu7/71fZ+sBXmuvfep1loAAAAAYKp22tYn\nAAAAAMATi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAgMdNVb2wqu6YxvZr\nqmrJDJ7S2H5XV9VrZnq/07W1Pi8AwHQJSgBAquqoqvrHqrqvqtZX1Q1V9a9n+jittX9orT1jiud0\nYVW9daPtD2qtre49blXtUlVnVdWdVfVgVd1dVRdU1aLefT2etvTzTsUQ0b5XVU/aGvsHAHZsghIA\n/IKrqicn+dskf5JkzyRPS/LfkmzYluc1wy5P8tIkv5Vk9ySHJrk1ybJteVLbyhDSXpikZTSXx/PY\nsx/P4wEAW4egBAD830nSWvvL1tpPWms/aq19srX2ubEVqurUqrp9uKLlE1X1S+Pea1X1/w5X/9xf\nVX9YVb9cVTdW1Q+q6rKq2mVYd0lVrR237bOGK2W+P9ze9dJh+WlJTkzyxqp6oKr+Zlh+d1W9ePh+\nVlX9QVV9eTjurVW1/8Yfblj/3yY5rrV2c2vtkdbafa21P22tnT/B+r9cVddW1bqq+pequrSq9hj3\n/puq6pvDMe+oqmXD8iOr6pbhM3+nqt490bCramlVfX7c62uq6jPjXl9fVS+b4PNucv9V9bzhCrPv\nV9X/mcJtcicn+XSSC5Os2Oj85lbVH1fV14Yr1q6vqrnDe0eNO843quqUYfljbhmsqlOq6vpxr1tV\nva6q7kxy57DsfcM+fjD87F44bv0Jf7ZV9adV9ccbne/fVNXvTPJ5AYAZJigBAF9K8pOquqiqjq2q\np4x/c4gbf5Dk3yXZO8k/JPnLjfaxPMlzkjwvyRuTnJdRENo/ycFJXrnxQatq5yR/k+STSZ6a5Iwk\nl1bVM1pr5yW5NMkftdbmtdZ+c4Lz/k/Dfl+S5MlJTk3ywwnWe3GSz7TWvjHZIMZOLcnbkuyX5FnD\nZzhrOOdnJHl9kn/dWpuf5Jgkdw/bvS/J+1prT07yy0ku28T+b0zyK1W1YLha5+AkC6tq/hBunpPR\njDc24f6r6mlJrkry1oyuMPu9JH9dVXtv5jOenNF8L01yTFXtM+69dw3n8Pxhf29M8tOqenqSj2d0\nJdveSQ5LcttmjrGxlyV5bpJnD69vHvaxZ5K/SPI/q2rO8N6mfrYXJXllVe00fPYFGV1ltvHfRwBg\nKxOUAOAXXGvtB0mOyuj2p/+R5N6qunJcZHhtkre11m5vrT2S5Jwkh42/SinJO1prP2itrUnyhSSf\nbK19pbV2X0YR4vAJDv28JPOSvL219uPW2rUZ3Xr3c/FpE16T5L+01u5oI/+ntbZugvX2SvKtKe4z\nrbW7WmtXt9Y2tNbuTfLuJC8a3v5JkicleXZV7dxau7u19uXhvYczhKLW2gOttU9vYv8PJbklya8m\nWZzkc0muT/KCjGZy5yY+x6b2/9tJPtZa+1hr7aettauH/b9kouNX1VFJfinJZa21W5N8OaNbATOE\nmlOTvKG19s3hirV/bK1tyCgQXjNcyfZwa21da60nKL2ttba+tfajYQ4fHvbxSGvtjzOa69jztSb8\n2bbWPpPkvvzsVsVXJFndWvtOx3kAADNAUAIAMsSiU1prCzO6Yma/JO8d3v6lJO8bbnP6fpL1GV3F\n87Rxuxj/D/ofTfB63gSH3S/JN1prPx237Gsb7Xdz9s8ohkxmXZJ9p7jPVNVTq2rVcFvbD5J8OMmC\nZBSbkvxORlcsfXdYb79h01dndPvgF6vq5qr6jWF/Hxxu23ugqv5gWPfvkyzJKCr9fZLVGUWrFw2v\nJzLh/jP6+Zww9vMZfkZHbeYzr8go+P3L8Pov8rPb3hYkmZOJ5zrVeW/KY64Qq6rfrdFtlPcN57z7\ncPzJjnVRRhEtw5+XTOOcAIAtJCgBAI/RWvtiRs/WOXhY9I0kr22t7THua25r7R+neah7kuw/dvvS\n4OlJvjl2KpNs/42Mbv2azDVJjqyqhVM8r7cNxz5kuL3stzMKaKOTau0vWmtjV/m0JO8Ylt/ZWntl\nRrfvvSPJ5VW1W2vt9OG2vXmttXOG3WwclP4+kwSlTe1/mMMlG/18dmutvX3jfQy31P37JC+qqm9X\n1beT/Mckh1bVoUn+JclDmXium5v3g0l2Hff6/5roI4w7jxcmedNwLk9pre2R0ZVHY3Pe3LE+nOS4\n4XyfleSKTawHAGxFghIA/IKrqmcOV4ssHF7vn9FtZ2O3VH0wye9X1UHD+7tX1QkzcOibMgoRb6yq\nnYcHSf9mklXD+99J8q82s/2fJ/nDqjqwRg6pqr02Xqm1dk2Sq5P8r6p6TlXNHp5XdHpVnTrBfucn\neSDJ94fnE/3nsTeq6hlV9WtV9aSMwsuPMroNLlX121W193DF1feHTX6yiXP/x4xu7zoyo+c7rcko\nUD03yf+eaIPN7P/DSX6zqo4ZHmY9p0YPP58ooL1s2ObZGT2/6LCMosw/JDl52PcFSd5dVfsN+/s3\nw+e9NMmLq+rfDzPcq6oOG/Z7W5J/V1W7VtWvZHQ11ebMT/JIknuTzK6q/y+jZyWN2eTPtrW2NqPn\nL12S5K/HbqEDAB5fghIAcH9GIeOmqnowo5D0hSS/mySttf+V0RUxq4ZbwL6Q5NjpHrS19uOMfmX9\nsRldGfNnGUWNLw6rnJ/Rs4q+X1UTXYXy7oweTP3JJD8Y1p+7icMdn+RjSf4qoythvpDR84uumWDd\n/5bkiGG9q5J8ZNx7T0ry9uF8v53R1UJjt7EtT7Kmqh7I6AHarxielzTRZ38wyT8lWTPMIRk9rPtr\nrbXvbuIzTLj/4WHjxw3ncW9GV/f850z8/3krknyotfb11tq3x76SvD/JicNDwn8vyeczijbrM/rZ\n79Ra+3pGz2X63WH5bUkOHfb7niQ/zigCXpRRfNqcT2T0bK0vZXSb40N57C1xk/1sL0ry/8TtbgCw\nzVRrk11NDgAA24+q+tWMrsxatNEzuACAx4krlAAAeMKoqp2TvCHJn4tJALDtCEoAADwhVNWzMnp+\n1L752W8hBAC2Abe8AQAAANDFFUoAAAAAdJm9rU9gSy1YsKAtWrRomx3/wQcfzG677bbNjr+9MIcR\nczCDMeYwYg5mMMYcRszBDMaYw4g5mMEYczCDMeYwsj3M4dZbb/2X1trek633hA1KixYtyi233LLN\njr969eosWbJkmx1/e2EOI+ZgBmPMYcQczGCMOYyYgxmMMYcRczCDMeZgBmPMYWR7mENVfW0q67nl\nDQAAAIAughIAAAAAXQQlAAAAALo8YZ+hBAAAALA5Dz/8cNauXZuHHnpoW5/KlOy+++65/fbbH5dj\nzZkzJwsXLszOO++8RdsLSgAAAMAOae3atZk/f34WLVqUqtrWpzOp+++/P/Pnz9/qx2mtZd26dVm7\ndm0OOOCALdqHW94AAACAHdJDDz2Uvfba6wkRkx5PVZW99tprWlduCUoAAADADktMmth05yIoAQAA\nANDFM5QAAACAXwiLVl41o/u7++2/PqP72xLvfe97c9ppp2XXXXd9XI/rCiUAAACAJ6j3vve9+eEP\nf/i4H1dQAgAAANiKLr744hxyyCE59NBDc9JJJ+VrX/tali1blkMOOSTLli3L17/+9STJ6aefnssv\nv/zR7ebNm5ckWb16dZYsWZLjjz8+z3zmM3PiiSemtZZzzz0399xzT5YuXZqlS5c+rp/JLW8AAAAA\nW8maNWty9tln54YbbsiCBQuyfv36rFixIieffHJWrFiRCy64IGeeeWauuOKKze7ns5/9bNasWZP9\n9tsvL3jBC3LDDTfkzDPPzLvf/e5cd911WbBgweP0iUZcoQQAAACwlVx77bU5/vjjHw0+e+65Z268\n8cb81m/9VpLkpJNOyvXXXz/pfo488sgsXLgwO+20Uw477LDcfffdW/O0JyUoAQAAAGwlrbVU1WbX\nGXt/9uzZ+elPf/rodj/+8Y8fXedJT3rSo9/PmjUrjzzyyFY426kTlAAAAAC2kmXLluWyyy7LunXr\nkiTr16/P85///KxatSpJcumll+aoo45Kkjz96U/PrbfemiT56Ec/mocffnjS/c+fPz/333//Vjr7\nTfMMJQAAAOAXwt1v//XH/ZgHHXRQ3vzmN+dFL3pRZs2alcMPPzznnntuTj311Lzzne/M3nvvnQ99\n6ENJklNOOSUnnnhijjzyyCxbtiy77bbbpPs/7bTTcuyxx2bffffNddddt7U/zqMEJQAAAICtaMWK\nFVmxYsVjll177bU/t95Tn/rUfPrTn3709dve9rYkyZIlS7JkyZJHl7///e9/9PszzjgjZ5xxxgyf\n8eTc8gYAAABAF0EJAAAAgC6CEgAAAABdBCUAAAAAughKAAAAAHQRlAAAAADoMntbnwAAAADA4+Ks\n3Wd4f/fN7P5m2IUXXpijjz46++2334zv+xc2KC1aedW0tr9w+W4zdCYAAAAAj/XII49k9uzZm3w9\nFRdeeGEOPvhgQQkAAADgiebiiy/Ou971rlRVDjnkkLz1rW/NqaeemnvvvTd77713PvShD+XpT396\nTj/99Oyzzz757Gc/myOOOCLz58/PPffck7vvvjsLFizIJZdckpUrV2b16tXZsGFDXve61+W1r31t\nkuSP/uiPcskll2SnnXbKsccem8WLF+eWW27JiSeemLlz5+bGG2/M3LlzZ+wzCUoAAAAAW8maNWty\n9tln54YbbsiCBQuyfv36rFixIieffHJWrFiRCy64IGeeeWauuOKKJMmXvvSlXHPNNZk1a1bOOuus\n3Hrrrbn++uszd+7cnHfeedl9991z8803Z8OGDXnBC16Qo48+Ol/84hdzxRVX5Kabbsquu+6a9evX\nZ88998z73//+vOtd78rixYtn/HMJSlvqW7clZx03vX1s5/daAgAAANNz7bXX5vjjj8+CBQuSJHvu\nuWduvPHGfOQjH0mSnHTSSXnjG9/46PonnHBCZs2a9ejrl770pY9eWfTJT34yn/vc53L55ZcnSe67\n777ceeedueaaa/KqV70qu+6666PH2NoEJQAAAICtpLWWqtrsOuPf3223xz6zefzr1lr+5E/+JMcc\nc8xj1vm7v/u7SY8x03Z6XI8GAAAA8Atk2bJlueyyy7Ju3bokyfr16/P85z8/q1atSpJceumlOeqo\no6a0r2OOOSYf+MAH8vDDDycZ3R734IMP5uijj84FF1yQH/7wh48eI0nmz5+f+++/f6Y/UhJXKAEA\nAAC/KLbBo2cOOuigvPnNb86LXvSizJo1K4cffnjOPffcnHrqqXnnO9/56EO5p+I1r3lN7r777hxx\nxBFprWXvvffOFVdckeXLl+e2227L4sWLs8suu+QlL3lJzjnnnJxyyik5/fTTPZQbAAAA4IlmxYoV\nWbFixWOWXXvttT+33gc/+MHMnz//0ddnnXXWY97faaedcs455+Scc875uW1XrlyZlStXPmbZy1/+\n8rz85S+fxplvmlveAAAAAOgiKAEAAADQRVACAAAAdlittW19Ctul6c5FUAIAAAB2SHPmzMm6detE\npY201rJu3brMmTNni/fhodwAAADADmnhwoVZu3Zt7r333m19KlPy0EMPTSvy9JgzZ04WLly4xdsL\nSgAAAMAOaeedd84BBxywrU9jylavXp3DDz98W5/GlLjlDQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAA\nAAB0EZQAAAAA6DKloFRV/7Gq1lTVF6rqL6tqTlUdUFU3VdWdVfVXVbXLsO6Thtd3De8vGref3x+W\n31FVx4xbvnxYdldVrZzpDwkAAADAzJk0KFXV05KcmWRxa+3gJLOSvCLJO5K8p7V2YJLvJXn1sMmr\nk3yvtfYrSd4zrJeqevaw3UFJlif5s6qaVVWzkvxpkmOTPDvJK4d1AQAAANgOTfWWt9lJ5lbV7CS7\nJvlWkl9Lcvnw/kVJXjZ8f9zwOsP7y6qqhuWrWmsbWmtfTXJXkiOHr7taa19prf04yaphXQAAAAC2\nQ9Vam3ylqjckOTvJj5J8Mskbknx6uAopVbV/ko+31g6uqi8kWd5aWzu89+Ukz01y1rDNh4fl5yf5\n+HCI5a211wzLT0ry3Nba6yc4j9OSnJYk++yzz3NWrVq1pZ87n//mfVu8bZIcMHdD5m24Z1r7yL6H\nTW/77cADDzyQefPmbevT2ObMwQzGmMOIOZjBGHMYMQczGGMOI+ZgBmPMwQzGmMPI9jCHpUuX3tpa\nWzzZerMnW6GqnpLRFUMHJPl+kv+Z0e1pGxsrU7WJ9za1fKKrpCasXK2185KclySLFy9uS5Ys2dyp\nb9YpK6/a4m2T5MJDv5old7xlWvvIK6cXtbYHq1evznR+DjsKczCDMeYwYg5mMMYcRszBDMaYw4g5\nmMEYczCDMeYw8kSaw1RueXtxkq+21u5trT2c5CNJnp9kj+EWuCRZmGTscp21SfZPkuH93ZOsH798\no202tRwAAACA7dBUgtLXkzyvqnYdnoW0LMk/J7kuyfHDOiuSfHT4/srhdYb3r22j++quTPKK4bfA\nHZDkwCSfSXJzkgOH3xq3S0YP7r5y+h8NAAAAgK1h0lveWms3VdXlSf4pySNJPpvRbWdXJVlVVW8d\nlp0/bHJ+kkuq6q6Mrkx6xbCfNVV1WUYx6pEkr2ut/SRJqur1ST6R0W+Qu6C1tmbmPiIAAAAAM2nS\noJQkrbW3JNn4gUFfyeg3tG287kNJTtjEfs7O6OHeGy//WJKPTeVcAAAAANi2pnLLGwAAAAA8SlAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGUAAAAAOgiKAEAAADQRVAC\nAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6zN7WJwAAAPBEs2jlVdPa/sLlu83QmQBs\nG65QAgAAAKCLK5QAAAAeb9+6LTnruOnt46z7ZuZcALaAK5QAAAAA6CIoAQAAANBFUAIAAACgi6AE\nAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AE\nAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AE\nAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AE\nAAAAQBdBCQAAAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AE\nAAAAQBdBCQAAAIAughIAAAAAXaYUlKpqj6q6vKq+WFW3V9W/qao9q+rqqrpz+PMpw7pVVedW1V1V\n9bmqOmLcflYM699ZVSvGLX9OVX1+2ObcqqqZ/6gAAAAAzISpXqH0viR/11p7ZpJDk9yeZGWST7XW\nDkzyqeF1khyb5MDh67QkH0iSqtozyVuSPDfJkUneMhahhnVOG7fd8ul9LAAAAAC2lkmDUlU9Ocmv\nJjk/SVprP26tfT/JcUkuGla7KMnLhu+PS3JxG/l0kj2qat8kxyS5urW2vrX2vSRXJ1k+vPfk1tqN\nrbWW5OJx+wIAAABgO1OjhrOZFaoOS3Jekn/O6OqkW5O8Ick3W2t7jFvve621p1TV3yZ5e2vt+mH5\np5K8KcmSJHNaa28dlv/XJD9KsnpY/8XD8hcmeVNr7TcmOJfTMrqSKfvss89zVq1atcUf/PPfvG+L\nt02SA+ZuyLwN90xrH9n3sOltvx144IEHMm/evG19GtucOZjBGHMYMQczGGMOI+ZgBmPMYWRHmIN/\nT8yMHeHvwnSZwYg5jGwPc1i6dOmtrbXFk603ewr7mp3kiCRntNZuqqr35We3t01koucftS1Y/vML\nWzsvo7iVxYsXtyVLlmzmNDbvlJVXbfG2SXLhoV/NkjveMq195JXT+4/Q9mD16tWZzs9hR2EOZjDG\nHEbMwQzGmMOIOZjBGHMY2RHm4N8TM2NH+LswXWYwYg4jT6Q5TOUZSmuTrG2t3TS8vjyjwPSd4Xa1\nDH9+d9z6+4/bfmGSeyZZvnCC5QAAAABshyYNSq21byf5RlU9Y1i0LKPb365MMvab2lYk+ejw/ZVJ\nTh5+29vzktzXWvtWkk8kObqqnjI8jPvoJJ8Y3ru/qp43/Ha3k8ftCwAAAIDtzFRueUuSM5JcWlW7\nJPlKkldlFKMuq6pXJ/l6khOGdT+W5CVJ7kryw2HdtNbWV9UfJrl5WO+/t9bWD9//hyQXJpmb5OPD\nFwAAAADboSkFpdbabUkmeiDTsgnWbUlet4n9XJDkggmW35Lk4KmcCwAAAADb1lSeoQQAAAAAjxKU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAAdBGU\nAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF2mHJSqalZVfbaq/nZ4fUBV\n3VRVd1bVX1XVLsPyJw2v7xreXzRuH78/LL+jqo4Zt3z5sOyuqlo5cx8PAAAAgJnWc4XSG5LcPu71\nO5K8p7V2YJLvJXn1sPzVSb7XWvuVJO8Z1ktVPTvJK5IclGR5kj8bItWsJH+a5Ngkz07yymFdAAAA\nALZDUwpKVbUwya8n+fPhdSX5tSSXD6tclORlw/fHDa8zvL9sWP+4JKtaaxtaa19NcleSI4evu1pr\nX2mt/TjJqmFdAAAAALZD1VqbfKWqy5O8Lcn8JL+X5JQknx6uQkpV7Z/k4621g6vqC0mWt9bWDu99\nOclzk5w1bPPhYfn5ST4+HGJ5a+01w/KTkjy3tfb6Cc7jtCSnJck+++zznFWrVm3hx04+/837tnjb\nJDlg7obM23DPtPaRfQ+b3vbbgQceeCDz5s3b1qexzZmDGYwxhxFzMIMx5jBiDmYwxhxGdoQ5+PfE\nzNgR/i5MlxmMmMPI9jCHpUuX3tpaWzzZerMnW6GqfiPJd1trt1bVkrHFE6zaJnlvU8snukpqwsrV\nWjsvyXlJsnjx4rZkyZKJVpsxD6FnAAAgAElEQVSSU1ZetcXbJsmFh341S+54y7T2kVdO7z9C24PV\nq1dnOj+HHYU5mMEYcxgxBzMYYw4j5mAGY8xhZEeYg39PzIwd4e/CdJnBiDmMPJHmMGlQSvKCJC+t\nqpckmZPkyUnem2SPqprdWnskycIkY3l9bZL9k6ytqtlJdk+yftzyMeO32dRyAAAAALYzkz5DqbX2\n+621ha21RRk9VPva1tqJSa5Lcvyw2ookHx2+v3J4neH9a9vovrork7xi+C1wByQ5MMlnktyc5MDh\nt8btMhzjyhn5dAAAAADMuKlcobQpb0qyqqremuSzSc4flp+f5JKquiujK5NekSSttTVVdVmSf07y\nSJLXtdZ+kiRV9fokn0gyK8kFrbU10zgvAAAAALairqDUWludZPXw/Vcy+g1tG6/zUJITNrH92UnO\nnmD5x5J8rOdcAAAAANg2Jr3lDQAAAADGE5QAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALoISgAAAAB0EZQAAAAA6CIoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXSYNSlW1f1VdV1W3V9WaqnrDsHzPqrq6qu4c/nzKsLyq6tyququqPldVR4zb14ph\n/TurasW45c+pqs8P25xbVbU1PiwAAAAA0zeVK5QeSfK7rbVnJXlektdV1bOTrEzyqdbagUk+NbxO\nkmOTHDh8nZbkA8koQCV5S5LnJjkyyVvGItSwzmnjtls+/Y8GAAAAwNYwaVBqrX2rtfZPw/f3J7k9\nydOSHJfkomG1i5K8bPj+uCQXt5FPJ9mjqvZNckySq1tr61tr30tydZLlw3tPbq3d2FprSS4ety8A\nAAAAtjM1ajhTXLlqUZL/neTgJF9vre0x7r3vtdaeUlV/m+TtrbXrh+WfSvKmJEuSzGmtvXVY/l+T\n/CjJ6mH9Fw/LX5jkTa2135jg+KdldCVT9tlnn+esWrWq8+P+zOe/ed8Wb5skB8zdkHkb7pnWPrLv\nYdPbfjvwwAMPZN68edv6NLY5czCDMeYwYg5mMMYcRszBDMaYw8iOMAf/npgZO8LfhekygxFzGNke\n5rB06dJbW2uLJ1tv9lR3WFXzkvx1kt9prf1gM485muiNtgXLf35ha+clOS9JFi9e3JYsWTLJWW/a\nKSuv2uJtk+TCQ7+aJXe8ZVr7yCun9x+h7cHq1asznZ/DjsIczGCMOYyYgxmMMYcRczCDMeYwsiPM\nwb8nZsaO8HdhusxgxBxGnkhzmNJveauqnTOKSZe21j4yLP7OcLtahj+/Oyxfm2T/cZsvTHLPJMsX\nTrAcAAAAgO3QVH7LWyU5P8ntrbV3j3vryiRjv6ltRZKPjlt+8vDb3p6X5L7W2reSfCLJ0VX1lOFh\n3Ecn+cTw3v1V9bzhWCeP2xcAAAAA25mp3PL2giQnJfl8Vd02LPuDJG9PcllVvTrJ15OcMLz3sSQv\nSXJXkh8meVWStNbWV9UfJrl5WO+/t9bWD9//hyQXJpmb5OPDFwAAAADboUmD0vBw7U09MGnZBOu3\nJK/bxL4uSHLBBMtvyehB3wAAAABs56b0DCUAAAAAGCMoAQAAANBFUAIAAACgi6AEAAAAQBdBCQAA\nAIAughIAAAAAXQQlAAAAALrM3tYnAAAAwBPPopVXTXsfFx56Z3LWcVu+g7Pum/Y5AFvGFUoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoAAAAA\ndBGUAAAAAOgiKAEAAADQRVACAAAAoIugBAAAAEAXQQkAAACALoISAAAAAF0EJQAAAAC6CEoA/P/t\n3XecbFWV9vHfQxJEUBjRUfIliEqQpAJjwgFFUQQkqYO++sroayCIiAwGdDCBg4qOcRyYGZIOQQRB\nZiQZBgkSryJcQVQMgDiI5PC8f+xT3Lp1q/t205ezq+s838+nP5xQ3aw+t7pq1zp7rxURERERETEt\nSShFRERERERERMS0JKEUERERERERERHTkoRSRERERERERERMSxJKERERERERERExLUkoRURERERE\nRETEtCShFBERERERERER05KEUkRERERERERETEsSShERERERERERMS1JKEVERERERERExLQsVTuA\niBgNax185oy+/5hNrocP7zSzID58x8y+PyIiIiIiIlqRGUoRERERERERETEtSShFRERERERERMS0\nJKEUERERERERERHTkoRSRERERERERERMSxJKERERERERERExLUkoRURERERERETEtCShFBERERER\nERER07JU7QAiIiIiRtlaB585459xzCbXw4d3evQ/4MN3zDiGiIiIiMUpM5QiIiIiIiIiImJaMkMp\nOi93niMiIiIiImKmuvbZMjOUIiIiIiIiIiJiWpJQioiIiIiIiIiIaRmZhJKkl0v6uaR5kg6uHU9E\nRERERERERAw3EgklSUsCXwB2AJ4F7CXpWXWjioiIiIiIiIiIYUalKPdzgXm2bwCQdCKwE/DTqlFF\nREREBDDzQqMzLjIKaWIxIrpWdDYiIoaT7doxIOm1wMtt/99m/++A59l+58Dj9gH2aXafAfy81UAX\n9GTgtor//1GR61DkOuQa9OQ6FLkOuQY9uQ5FrkOuQU+uQ5HrkGvQk+uQa9CT61CMwnVY0/Yqi3rQ\nqMxQ0pBjC2W6bH8F+MpjH86iSbrU9ha146gt16HIdcg16Ml1KHIdcg16ch2KXIdcg55chyLXIdeg\nJ9ch16An16GYTddhJGooAb8BVu/bXw34baVYIiIiIiIiIiJiEqOSULoEWE/S2pKWAfYETq8cU0RE\nREREREREDDESS95sPyjpncB3gSWBr9ueWzmsRRmJpXcjINehyHXINejJdShyHXINenIdilyHXIOe\nXIci1yHXoCfXIdegJ9ehmDXXYSSKckdERERERERExOwxKkveIiIiIiIiIiJilkhCKSIiIiIiIiIi\npiUJpYiIiIiIiIiImJYklCIWE0lPrx1DRA2Snl87hoiIUSQpY+1JZOwUXSNp277ttQfO7dJ+RHVI\nWnGSc2u0GUvMTIpyz4CkH9repnYcbZD0eOAB2w80+88AXgHcZPuUqsGNCEm/st2JF0BJG9i+ttl+\nnO37+s493/ZF9aIbDZJOsr1H7TjaIOkntjerHUdNklYBVrH904HjzwZusX1rncjaJemDk5y27Y+2\nFkxFktYE/tf2Hc3+S4DXADcBn7d9f8342iDpG7Z3b7Y/aft9fefOsb19vejaI+kK4O22/6d2LKOo\nS2OnKCQdY/tNteOopX/MNDh+6tJ4auA6fM/2S4edG3eLSiLOhs/ZuWsyM116AzwbWAtA0rrA/wBz\ngHdI+njFuEaJagfQouP7tgcHyf/cZiAjbKvaAUSrjgZWGXJ8NeCzLcdS011Dvgy8BXjfJN83br4B\nLA8g6TnAN4FfAZvQndfI9fq2txs4N+xvZVz9PfBZSV+VtFLtYEZQZ8ZOks6TdO4EX9+rHV+LNq4d\nQGWaYHvY/jjr/11XnuTcuPtP4FBgx+brVX1fO1aMa8qWqh3ALNel6V0r2b6+2X4jcILtd0laBrgM\neH+90EZGl54PeTOMfnMknT7RSduvbjOYSjayfcHgQdvflfTpGgHVYPuR31XSCsC+wJuBE4HOXAdg\nOdu/bbbfAHzd9qeb5U9XVIyrTZO9J3bm/dL2jyU9D3gbcKmks4CH+86/u1pwo6EzzwXgwCHHng8c\nBNzSciw1PV7SpkwwXrT9k5bjaZsn2B62P85yHYpdgT0oidZvUT5jz6sb0vQkobQIk0xDE7Bcm7FU\n1v+HvS1wBIDt+yU9PPxbxo+koxn+IifgSS2HU1PeBABJE03HFbB0m7FUdivdShYMM9m/d5eeC0ha\nGTgAeD1wLLCZ7T/Vjap1/R+UtqW56WL7YakzOffeh8YlgOX6PkB2bfwE5e77lpTXysvoSyh1QcZO\nhe3LetuSXgR8AHgc8DbbZ1ULrH2rUsYMw14MTXnNHGe9m3BiwRtyAtae+NvGzlMkHUD5vXvbNPud\nmcVq+1TgVEnLAzsBn5b0V8A/DLtROYqSUFq0V01y7ozWoqjvKklHAjcD6wLnAEjqzECgcemjPDdu\nVpP0OcqLfm+bZn/VemG1brIkyrWtRVHfX2bLm95j6HpJr7D9nf6DknYAbqgUU+skHQHsAnyFMmvr\nL5VDquVcSd8AfgesBJwLIOlpwNjXT2r8DvinZvv3fdu9/U6Q9DbgvZQbcW9xN4uXZuzUkPQySiLp\nXuBw2+dVDqmGebbHPWk0mZ36to8cODe4P86+CqwwZBvga+2HU929wB3AnylldZatG87UpSj3DEja\n1fbJteNog6TlKEsXnkaZun9lc3xrYB3b/14zvtokLQu8yvY3a8fSBklvnOy87WPbimVUSVq6V8R+\n3Ek6xXZnOpMMI2l9yk2GH1FmIABsQamltaPt62rF1qZmxup9wIMsOCNBlKLcE3Z1GScq05D2oLxn\nfsP2zc3xTYGn2P5uzfiiPZKOA/a33aUlTVPSwbHTJZSZF0ewcP3JLiz1AkDS5bY3rR1HxChomnbs\nBTwX+G/gRNuzKtGehNIMdLkzhaSlgQ2Bm7s6SJK0JLA95UXgZcD3bb+2blTtkPQx24fUjmPUNB8i\nXwK8jjJIfmrlkFohaVcmWeo4GzpULA6SHkf5t9+wOTQXON72vfWiilHSvG/safu42rHUImk74CDb\ng4W6x5KkN9j+j2Z7G9s/7Dv3Ttufrxdd+zo+djqfid8r3ZVZO5J2B67paldUSecx+fPgpROcGyt9\nqxuG6kp9ueZG3FXADyjPiwWeG7PhOiShNAOSfm179dpxtEHSl4Cjbc+V9ETKnZWHKHUBDrR9QtUA\nWyTphZQPja8ELga2AebYvrtqYC3qUjvPqWgKrr4O2JnyN/EO4PSu1I2R9K+TnLbtN7cWTIyE5o7b\nsykDo7m2z68bUbskrUh5HVgVOB34L+CdlKK8V9jeaZJvHwuStgW+BDwdOA34GPBvlNlqh3co0ZwW\n4WTsFPNJOhH44uBS+WY54Bttv65OZO2QtPmQw48UZ7e9ZcshVSHpfuAaSlfU3zJQU6srqx0kvYnJ\nb8qO/HVIQmkGujRDSdJc289utvcDXmz7NZL+GjirK1NXJf2G0vr5i8Bptu+UdKPtLhXRQ9KVwIuZ\nuEPH7a0GVImkw4HdKc+JE4BTgUu79nwIkHQnExed7dJSr1WBUyi1AC6j/P6bUYow79xb+jXuJH0L\n+BPl5stLKXWUlgH2td2JLm+SLgf2p1yDHSjJpA/Y/mzVwFrWv7xncKlPV5b+ZOxUTNLoB+jUbN5H\nPlMMOXeN7Q2HnRtHA8XZP9al4uxN4endKMvDHwROAk7uys3YcZKi3Isg6Wom/pDQieUsjf4iotsB\n3wSw/fsOdawBOBl4DeXF76HmQ0MXs7IbMP/D4iADc9oNp5p9gJ9TBsln2L5XUueeD5JWA9ay/YNm\n/wDgCc3p42db+9NHw/YKi35UJ3yecuf5mP6DkvYG/pkFi5GOszm2NwKQ9DXgNmAN23fWDatV7puZ\ndpqkW7uWTGqkK2rGTj2TNfoxJRnfBctMcq4TXVFTnB1s/5Eyi/VLzc2ovYC5kt7Xpdq8kr7N5DOU\nXt1iOI9KZigtgqQ1hx0GVgMOsf2KlkOqolnv+2lKl7fzgA2aZNJSlHXQG1QNsEV9dXL2Al4BrAi8\nBfhOVzoadeWu6qIM1ILYlvK38bfA6rYfrBlbmySdABxn+4xm/+eULl+Pp7xWvL5mfNEeST+3/Yzp\nnhs3XV7a1CPpBsoSv54j+/c7NBvjbmAeZey4TrNNsz/H9vK1YmtTxk6Tk/RU23+oHUcbJJ0JfGGC\nrqjvtr1DncjakeLsC5K0GeV1YTvKzepPD9bXGmfNLLUJDS4NHUVJKE2DpOdQ1n/vDtxImZbXiWKK\nTQejzwF/DXymd/e5ybBvb/s9FcOrpilOvgOwJ+U6PLlySK1IQmlhTbeaHSlvin8DfG/c6wD0DPnw\n3L/E4/u2X1Avunb0LXnrn7VnykzgZWx3YkawpHm21x1yfAngumHnxpGkh4C7eruUJX9306ElkKmt\nVkxwY/IRtm9qK5ZR0dWx06CmJumulM8Wz7S9auWQWtH1rqgpzl5IOowybv4ZcCJwdpduxvZIOsb2\nm2rHMRNJKC1C86K3J+VD4h8p6zsPtD3pACG6R9Jytu+pHUcbJL1pcElLc7xTLYAnImkFYJfZUEhv\ncZD0U9vP6ttfuVdHa/BcVzTPgf8H/D1waleS7pKOoix33M/2Xc2x5YGjgHtnQ7eSiGhPl8ZOUH5f\n4NWUJNJmwAqU5YAX2n64ZmxtSlfUaLqb3QD0/v57SYnejZeNqwTWsnGYwdyJO6YzdC3wfcqH5HkA\nkvavG1L7JH3D9u7N9idtv6/v3Dm2t68XXXsW1eqTUnx17PUnk4a1AKapsTXumlpBAXdKWr93V7Ev\nmbQB0KmlDJKeBOwH7A0cD2zZ1AnoioOAjwM3SerNvFgDOBY4pFpULZO0re1zm+21bd/Yd26XLiz3\nGvL6aEotqR/0X49xJ+lGFhw3qG/fttdpP6p2ZexUSDoOeCFwDqXe3LnAPHesCyaA7fuAyWYxjjVJ\nT6F0Au11Q/0pZRngLVUDa1enivJP4vGSNmXiRkcjvwQyCaVF25UyQ+k8SWdTpuR1qgp1Y72+7e2A\n9/Xtr9JyLDUdOOTYI60+W46lqglaAK/tbrUAPhK4AjgLuI+Flzt1xYeAM5qud703vs0pCYR9q0XV\nIklPBt5DKTr7dWBT23fUjap9th8ADpT0AWBdyt/EvI69LkB5bejdcTy5bxvgULpRfHdYofq1gH+Q\n9GHbJ7YcTy1bDOwvQSmdcCBwefvhVJGxU7Ehpfvjz4BrbT/U0UYeg0nWfmOfZJW0DeWG0zGU7pe9\nbqgXS3q97R9WDK81XVzuO4FVKXWKJ2p0NPJLILPkbYqaKfuvYX7x3WMpyxjOqRpYS/qn46XYaNHx\nVp9pAcwjddX2BF5OqQNwAqV2UudeWCVtSPlw0GsFPBf4lO1r6kXVHkl3AbdS7rgu1MnL9j+1HlQF\naYtdpFX8xCStDPx318YNTR2xvwPeS7kR8bEuFZ7t6fLYCR6Zufs6ys2HWyhdczey/fuqgbWoaRff\nrz/J+hPbu7YfVXskXQS83fblA8efA3zZ9vPqRNauZtz00LBTdKTWIIzHmCAzlKaoqQVxHHBcMxja\nDTiYMm21C3rT8ZYAluubmtcrNtoZafUJpAUwALavoHwwOFjS1pSE89FNy9PT60bXriZxtHftOCo6\ngvl/A8NmZnRF2mIXaRU/Adu3Nx2/OqEpQP1mYH/gB8BOtn9RN6r2ZexU2L4W+CDwQUlbUMYNF0v6\nje2t60bXjt4y8CFJ1ld2JMm64mAyCcqYsqm/2BXXzfZEShSZoRRT0qx/n5Dtl7QVS01p9TlfWgDP\nJ2kVyt213YAHgA/YvqhuVO2R9G0m+ZBs+9UthhMVdaU+0KJI+l/gQspNlxc02zT7f2N7pVqx1SZp\nW+DQDnUy+g3wIPAZyszeBXTh7yVjp8k1iZV9bR9VO5Y2DEmyfrxLSVZJPwO2tv2ngeMrAz+yvUGd\nyNrV1RUugyRtP2zFk6TVgT1tH1EhrGlJQiliGtLqc7iutgCW9H8os7SWBf4T+EbHCioCjyxhmJDt\nC9qKpSZJOwDvB57F/CKbn7T9naqBtSgDxCJ/EyDpahZ+v1wZ+C2wdzNTY+xJOobJxw1vbjGcKjJ2\nWjRJv7K9Ru042tD1JKukfYC30izxaw5vDnwS+LrtL9eKrU3N82DCcgBdKRXQr6nHuRvlZv2qlPI6\nw2rQjZQklGJGJG0HHGR7u9qxRB3N7Bxs39rsd6YFcNPy9GrmD4gWeEHt4sycwedDV0h6K/D3lDpS\nlzaHtwA+AXzN9ldqxdamJJQKScfYflPtOGqStObAIQN/bEoIREQfSb+2vXrtONqQJCtI2pGF604e\nYfvb9aJql6TfUWqxTtTd7LB2I6qjWea4M6W22vrAqcAetlerGtg0JKEUU9JMUf8S8HTgNOBjzO9M\ncPi4303oScHZolnu9iHgnZS6WqLcbTra9kdqxtamzEKYT9KHgHdRngtL0LHng6SfUpYy3T5w/K8o\nbdKfWSeydkm6G5g37BTlg8LGLYdURRJrIOnxwANN5z8kPYOyPPqXtk+tGlzLmqYF72XBFuFH2r66\namAtydhp0bo0QykC8j7ZI+keSrfsQynjRUu6wfacyqFNWYpyx1R9GtiHsvZ9B+AiSp2Yz1aNqn0p\nOFvsB2wDbGn7RgBJc4AvStq/K3UAupQwmoyk/YG/odvPBw0mk6AUH+1Q/WGAG5n8dbIreo0sJrrz\n2oWaMWdT6updL2ldyvjhOGBHSc+1/f6q0bVE0k7AkcDHmd8aenPgFEkH2v5WzfhakrETIOlOyu/b\ne13o3dXvVIMbSQcMHDJwG+XD9I0VQmqVpKOZvO7ku1sMp6ZODY4mcQilZMgXgeMlnVQ5nmnLDKWY\nksEssqRf2F6nZkxRj6TLge1s3zZwfBXgnK50bZigRgh0bzZG558Pkn4M7GP7yoHjmwBftf3cOpG1\naxza3y4OzQfHSxg+YO5EzRhJV9veqNn+KLCy7XdIWga4rHdu3Em6ktLZ7ZcDx9cCvmV7kwphRVTT\nzGgetDLwMuDDtk9sOaRWSXpj3+5hlBn/j7B9bLsR1SFp5WE34rqquRG7FyW5tB7leXGq7euqBjYF\nmaEUU/WkgSnL6t/v0nTlZtr+PkCvC8PPgK/Mhj/4xWjpweQBlLo5TYHurtixdgAjIs8HeA9wuqR/\nBS6jJBq3BN4IvKFmYC37Ye0ARsS8LiSNFqE/2b4tpcMXtu9v6s91xdKDySQA27/s0Otjxk6ApGWB\ntwHrAldRCjA/WDeq9k1UG6fpcvbfwFgnlPoTRpL260oCaVCSSQuyfQNwOHC4pI0oNZXOAkZ+AscS\ntQOIWeMCypTl3lf/fmc+VEvaCjgf+AvwFeCrwF3A+ZKeXzG0tt3/KM+Nm6WB1Wzf1P8FrEG3Evad\nfz7Y/gHwPMr76psoLZGXAJ7fnOuKGyW9ZfCgpHdJ2q9GQJVk+jdcJenIZknsusA5AJKeVDes1j0g\naaHaOE3R8k4kEzJ2esSxlGYNV1PqiX26bjijpUkwdG0ZVN4rYpjfAYfMltVAWfIWMQ2SzqK0AT9/\n4PiLgINt71AlsJZJeogyGFzoFLCs7U7cdZV0BuUF/6qB41sAH7LdiVoyeT5MTtI2tjsxc0fSNcBm\ntu8fOP444JIOLQO92faqteOoSdJywL7A0ygzMa5sjm8NrGP732vG1xZJrwE+RWlm0j978WDgfbZP\nqxheKzJ2KgaWgS4FXJyixPM1DYAO7dLszhSmjiap/gngduCjwL8DT6bclNzb9tkVw5uSLt1BjxnK\ndGWgDILPHzxo+wJJnWgLDmB7ydoxjIi1BpNJALYvbepjdEKeDyBpSWB3YFXgLNtzm7bAh1CKrXal\nrpAHk0nNwfvUrerkt1fUcUYAABA6SURBVNQOoDbb91AGyYN+TWnq0Am2T5N0I2VZbK8T5lxg98Ga\na2MsY6figd6G7Qe79ZI43wT1J1cGfgvs3X5E7Roozr6cpD/3TlHeQ1esFlzU8HnKWPGJwLnADrYv\nkrQBcAKlwcVIS0IppqSZrnwKZaryVygveptSpivvYvuimvG16M5Jzg2boRHjbdlJznWmY0sA8C/A\n6pTWr0dLugnYinL3fexnIPST9FTbfxg8ViueSpZNl7f5JD0Z2I1ScHRV4NS6EbWrSRyN/QflSWTs\nVGwykDzoJRO6lkgYLJVh4I+2O/FcsL1C7RhipCxlu7ck/CO9z9S2r50tSecklGKqPgjsNXCH6TRJ\n51Kq0HdiujKwuqTPDTkuyiA5uuUSSW+1/dX+g00NmcsqxRR1bAFsbPvhpvDqbcC6tn9fOa62HQGc\nKek9QC9psjllyc+R1aJq32rMbxE/yJQi1WNN0grAzpTCoutTkkhzbK9WNbCWSTp9svO2X91WLBVl\n7ERm8/b5A/OLk18N/EuXipOnOHsM6G9Scc/AuVlRmyg1lGJKJF1ne/0Jzv3c9jPajqmGgVafC+lq\np4auamZdnEopPN1LIG0BLAPs3MFkQmcN1kHocl0ESTtQ6sNsSBkMzQU+YfusqoG1SNLltruyzHEo\nSfdQZuwdCvzAtiXdYHtO5dBaJelWyjK/E4AfM5BktH1BjbjalLFT9JN0EmX53/cpN6Rvsr1v3aja\n0/XfPxbUV4dUlNUNd/dOMUvqkCahFFMi6TLbm09wrrMfnPpJWrPp8BUdI+kllA/PAHNtn1sznmif\npLuBeb1dSpvXecxfytCJYtSTkbSl7Utqx9GGJJSg6e62J7A8cDxwEvBfHUwoLQlsR1nutzFwJnCC\n7blVAxsRGTt1T9eLk3f994/xkyVvMVWZrtxo6kmtClxo+xZJG1Puxr+AUkMlOkLStrbPtX2epF/a\nvrHv3C62T6kZX7TqmbUDGEWSnkVJKuwF3EGZwdcFH5/ohKQ1bP+qzWBqsH0UcJSkOZR//9OAp0t6\nH3BqVxp62H6IUlT17Kbb4V6U+pMfsX103ejak7FT9Ol6cfKu//4xBZKeBLzD9uG1Y1mUzFCKKcl0\n5ULSEZRigldQ1j6fAfw/SjvgL9u+t2J40bL+2XlZ8tRtks6xvX3tOEaBpDUpH5r3Ah4E1gS2sP3L\nmnG1aeC14Xu2XzrsXNdI2ojyvNjD9jq142lLk0h6JeV3Xws4nVI35eaacbUlY6fo17fEBxZc5tOJ\n4uRd//1jQZJWBz4APJ1y4+V44KOURg7Hz4blkJmhFFPSlYTRFLwS2NT2vZJWorQ43dj29ZXjijo0\nwfaw/Rhvq9QOYBRI+hGl9e2JwGttXy/pxi4lkxr9f/8rT3JubEnawPa1zfbjbN9n+2rgaknfrhxe\nayQdS1kSfRZwmO1rKodUQ8ZO8YiuFyfv+u8fC/k34ALgZODlwEWU2pMbzZZarEkoxZQ1s5T2BXoF\nuH8GfM72v9WLqnX39O6k2f5TU5A8A6Lu8gTbw/ZjvD1R0i4TnezQ8sdbKR3OnkpJsl1PN/8W8tpQ\n7rL2ZmL9T982wBcG9sfZ31FmI6wPvLtveUuXZiNk7BTR6JVLaLbXTrmEzlvZ9oeb7e9K+gOwpe37\nKsY0LUkoxZRI2hvYDziA0gpalMHgEZLoUFJpnYEWwGv173ek/W/MN6f591ffNs3+2vXCigqeSFnS\nMVGb+E4MEG3vJOmJwK7AYZLWBZ4k6bm2L64cXpueIukAyvOht02z35XZbJnBCdheonYMI2CisVMv\nqZaxU3TJkcxPqJ/Mgsn1Q+nIeCHma2Zu9t4Xfw88XtLyALZvrxbYFKWGUkyJpIuAPQeXLUhaCzjR\n9vMrhNU6SS+a7HwX2v/GfHk+RE+X6+JMRtJTgD0otWNWt92J4ruSPjTZeduHtRVLLakxFz1975XL\nAesBDwO/AO6BvFdGt/R3AR3sCJoOod0j6ZeU18ShNyRnQ2fUzFCKqVpxWA0M27+U1IXp2sDEg56m\noNqelDWw0RH9zwdJqzTHbq0XUVTUmRkX02H7FuBo4OjmBkQnTJYwkrRfm7FUtFrTHVZ929DB7rDB\nj4DDgTcDv6J5TgDHAIfUCyuiiiyJjkfYXqt2DDOVabgxVfc8ynNjS9KTJb1d0oXA+ZSaIdEhKj4k\n6TbgWuA6SbdK+mDt2KJ1b5S0zeBBSS+Q1JluVlDag0t6bTM7CUkbSzoe+H7l0EbFAYt+yFh4L3AZ\ncGnfdm//oIpxRfs+BawErG17s2YGxjqUpcJHVI0son1zJJ3eNCfobff2Uy6hYyS9oW97m4Fz72w/\nounLkreYEkl3A/OGnQLm2F6+5ZCqkLQCsDPwOkqBzVMp7Y9XqxpYVCFpf+AVwD69ooqS5gBfBM62\nfVTN+KI9ks4ADrF91cDxLYAP2X5Vncjalfbgiybp111Z+hcBIOl6YH0PfOiQtCRwre316kQW0b6U\nS4h+47A8PEveYqqeOeRYb8pyl6Yr3wJcTCma9wPblrRz5Ziinr2B7Wzf1jtg+4bmbsM5QBJK3bHW\nYDIJwPalXVrqRdqDT0Un7uQNFGFeSAoxd4oHk0nNwYckdeLvIaJnsoSRpJNI+YyumfUNLJJQiimx\nfVNvW9JzKDN0dgdupHQo6IpDKLWSvggc37zwR3ct3Z9M6rF9q6SlawQU1Sw7ybnlWouivrQHByTd\nyfDEkejO82Er4NfACcCPmSUD43hM/FTS3oMdgZubL9dWiiliFG1VO4Bo3ayvqZUlbzElktanJFL2\nAv4InAQcaHvNqoFV0ixr2otyTdYDPgicZvu6qoFFqyabijpbpqnG4iHpBOBc218dOP4WYHvbe9SJ\nrF2S/he4sO/QC/v3MyulO5rlTNtR3is3Bs4ETrA9t2pg0TpJq1Jaod9DqaNlYEtKcnVn2zdXDC9i\nZEj6le01ascR7ekrKyNKbbleiZlZU1YmCaWYEkkPUwqqvsX2vObYDbOhleHiJGld4Km2f9h3bGPg\nM8CLbC9ZLbhonaSHgLuGnQKWtZ1ZSh0h6amUmmr3Uz4wAWwBLEP5wPT7WrG1KbUhYhhJj6Mklo4A\nPmL76MohRQWStgWeTXmPnGv7e5VDimidpIluNgo4w/bT2own6pJ0JqXO5M0MmZHUv0poVCWhFFPS\n1AnaE9gaOBs4Efia7U51I5ik8O6WlMK7O9aJLCJGgaSXABs2u3Ntn1sznrZJWsP2r2rHEaOhSSS9\nkpJMWgs4Hfh6ZqRERFdJOm+y87Zf0lYsUZ+kfSmfsZ9GWQF0gu0r6kY1PUkoxbRIWh54DWVwuC1w\nLHCq7XOqBtYSSdfY3nCCc1fb3qjtmCKiPknb9pJHktbudf1r9nexfUq96Noz0K3kZNu71o4p6pB0\nLCW5ehZwou1rKocUERExkiStSUks7Umpy3kC5b1z5MupJKEUj5qklYHdgD1sb1s7njZImmd73eme\ni4jxNg5tXxcHSZfb3nRwO7qnWSrfWxLcP9gUpevXiu1HFRFRl6SDbH+q2d7N9jf7zn3Mdpe6Z8cQ\nkjYFvk7pkjvy5VSWqB1AzF62b7f95a4kkxqXSHrr4MGm8O5lQx4fEd0w69u+LiaTdSuJDrG9hO0V\nmq8V+75WSDIpIjpsz77t9w+ce3mbgcTokLS0pFdJOo4ys/c6YFbM8l6qdgARs8x+wKmSXs+QwrvV\nooqI2mZ929fFZBNJf6Yk0ZZrtiGzUiIiIiA3oKKPpF431FcCF1PqFO9je1jTn5GUhFLENNj+A7D1\nQOHdM7tWeDciFjJH0uk0bV6bbZr9zjQvmA1TsyMiIirKDajodwhwPHCg7dtrB/NopIZSRETEDEl6\n0WTnbV/QViwRERExmpr6cn+hmckL3N07BSxre+lasUU8GkkoRUREPIYknWR7j9pxRERERF1pWBHj\nJkW5IyIiHltb1Q4gIiIiRkJmc8RYSQ2liIiIiIiIiMfeUyQdMNFJ2//UZjARM5WEUkRExAxJ2myi\nU0DqIURERATAksATSEe3GBOpoRQRETFDks6b7Lztl7QVS0RERIwmST+xPdFNqIhZJzOUIiIiZigJ\no4iIiJiCzEyKsZKi3BERETMk6aC+7d0Gzn2s/YgiIiJiBL20dgARi1MSShERETO3Z9/2+wfOvbzN\nQCIiImI02b69dgwRi1MSShERETOnCbaH7UdEREREzHpJKEVERMycJ9geth8RERERMeuly1tERMQM\nSXoIuIsyG2k54O7eKWBZ20vXii0iIiIi4rGQhFJERERERERERExLlrxFRERERERERMS0JKEUERER\nERERERHTkoRSRERExBRJ+tEizq8l6XV9+1tI+txi+P+eL2mLmf6ciIiIiMUlCaWIiIiIKbK99SIe\nshbwSELJ9qW23/2YBhURERFRQRJKERERMZYkLS/pTElXSrpG0h6SNpd0gaTLJH1X0tOax54v6ShJ\nF0r6maQtJZ0i6XpJ/9j3M//S/FeSjmh+7tWS9mge8gngBZKukLS/pBdLOqP5nidI+tfm8VdJ2nVI\nzEtKOrLvMe8a8pgvSrpU0lxJh/Ud/4Sknzbfd2RzbLcmxislXbg4r29ERER021K1A4iIiIh4jLwc\n+K3tVwJIeiJwFrCT7VubJNDhwJubx99v+4WS9gW+BWwO3A78QtJRtv/Y97N3AZ4DbAI8GbikSdgc\nDBxoe8fm//nivu/5AHCH7Y2acysNiXkfYG1gU9sPSlp5yGP+wfbtkpYEvidpY+A3wM7ABrYt6UnN\nYz8IvMz2zX3HIiIiImYsM5QiIiJiXF0N/K2kT0p6AbA6sCHwX5KuAA4FVut7/Ol93zfX9u9s3wfc\n0Hxvv78BTrD9kO0/ABcAWy4inr8FvtDbsf2nCR7zJdsPNo+5fchjdpf0E+By4NnAs4A/A/cCX5O0\nC3B389gfAsdIeiuw5CLii4iIiJiyJJQiIiJiLNm+jjLL6Grg48CulETRc5qvjWxv3/ct9zX/fbhv\nu7c/OKtbjyIkAV7ggLRzszzuiqbo9kKPGXj82sCBwEttbwycCSzbJKCeC5wMvAY4G8D22yiJs9WB\nKyT91aOIOyIiImIhSShFRETEWJL0dOBu2/8BHAk8D1hF0lbN+aUlPftR/vgLgT2amkerAC8ELgbu\nBFaY4HvOAd7ZF99Ktk/tS3Bd2jzmbZKWah4zuORtReAu4A5JTwV2aB73BOCJtr8D7EdZjoekdWz/\n2PYHgdtYeKZVRERExKOSGkoRERExrjYCjpD0MPAA8HbgQeBzTT2lpYDPAHMfxc8+FdgKuJIyo+gg\n27+X9EfgQUlXAsdQlqX1/CPwBUnXAA8BhwGnDPzcrwHrA1dJegD4KvD53knbV0q6vIn5BsqSNihJ\nrG9JWpYyy2n/5vgRktZrjn2viTciIiJixmRPOKs6IiIiIiIiIiJiIVnyFhERERERERER05KEUkRE\nRERERERETEsSShERERERERERMS1JKEVERERERERExLQkoRQREREREREREdOShFJERERERERERExL\nEkoRERERERERETEt/x+UUFiOgC9VQwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f783e225518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class_accuracy.plot(title='Semiotic Class-wise Accuracy',\n",
    "                    y=['count', 'correct'], x='semiotic-class',\n",
    "                    kind='bar', figsize=(20,10), grid=True)\n",
    "plt.savefig(result_dir+'Semiotic_Class-wise_Accuracy.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_accuracy.to_csv(result_dir+'classwise_accuracy.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Normalization Mistakes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "mistake_mask = (results['after'] != truth['after'])\n",
    "mistakes = results[mistake_mask]\n",
    "mistakes = mistakes.assign(truth = truth.loc[mistake_mask, 'after'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>semiotic</th>\n",
       "      <th>before</th>\n",
       "      <th>class</th>\n",
       "      <th>after</th>\n",
       "      <th>truth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>88069</th>\n",
       "      <td>7196</td>\n",
       "      <td>10</td>\n",
       "      <td>TIME</td>\n",
       "      <td>18:00:00Z</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>eighteen hours u seconds</td>\n",
       "      <td>eighteen hours zero minutes and zero seconds z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88776</th>\n",
       "      <td>7250</td>\n",
       "      <td>3</td>\n",
       "      <td>TIME</td>\n",
       "      <td>0:02:01</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>zero hours two minutes and one seconds</td>\n",
       "      <td>zero hours two minutes and one second</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentence_id  token_id semiotic     before           class  \\\n",
       "88069         7196        10     TIME  18:00:00Z  ToBeNormalized   \n",
       "88776         7250         3     TIME    0:02:01  ToBeNormalized   \n",
       "\n",
       "                                        after  \\\n",
       "88069                eighteen hours u seconds   \n",
       "88776  zero hours two minutes and one seconds   \n",
       "\n",
       "                                                truth  \n",
       "88069  eighteen hours zero minutes and zero seconds z  \n",
       "88776           zero hours two minutes and one second  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mistakes[mistakes['semiotic']=='TIME']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Class-wise mistakes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "mistakes_grouped = mistakes.groupby('semiotic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>semiotic</th>\n",
       "      <th>before</th>\n",
       "      <th>class</th>\n",
       "      <th>after</th>\n",
       "      <th>truth</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>semiotic</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">CARDINAL</th>\n",
       "      <th>69063</th>\n",
       "      <td>5683</td>\n",
       "      <td>3</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>310780751</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>thirty one million seventy eight thousand seve...</td>\n",
       "      <td>three hundred ten million seven hundred eighty...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89790</th>\n",
       "      <td>7334</td>\n",
       "      <td>1</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>1298015</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one two nine eight o one five</td>\n",
       "      <td>one million two hundred ninety eight thousand ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>184</td>\n",
       "      <td>28</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>14356007</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one million four hundred thirty five thousand ...</td>\n",
       "      <td>fourteen million three hundred fifty six thous...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DATE</th>\n",
       "      <th>65463</th>\n",
       "      <td>5381</td>\n",
       "      <td>6</td>\n",
       "      <td>DATE</td>\n",
       "      <td>00s</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>hundreds</td>\n",
       "      <td>o o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88533</th>\n",
       "      <td>7230</td>\n",
       "      <td>16</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1968</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one thousand nine hundred sixty eight</td>\n",
       "      <td>nineteen sixty eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88533</th>\n",
       "      <td>7230</td>\n",
       "      <td>16</td>\n",
       "      <td>DATE</td>\n",
       "      <td>1968</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one thousand nine hundred sixty eight</td>\n",
       "      <td>nineteen sixty eight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DECIMAL</th>\n",
       "      <th>71203</th>\n",
       "      <td>5851</td>\n",
       "      <td>5</td>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>.763</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>seven hundred sixty three</td>\n",
       "      <td>point seven six three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71203</th>\n",
       "      <td>5851</td>\n",
       "      <td>5</td>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>.763</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>seven hundred sixty three</td>\n",
       "      <td>point seven six three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71203</th>\n",
       "      <td>5851</td>\n",
       "      <td>5</td>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>.763</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>seven hundred sixty three</td>\n",
       "      <td>point seven six three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DIGIT</th>\n",
       "      <th>59585</th>\n",
       "      <td>4895</td>\n",
       "      <td>12</td>\n",
       "      <td>DIGIT</td>\n",
       "      <td>1271</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>twelve seventy one</td>\n",
       "      <td>one two seven one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27785</th>\n",
       "      <td>2270</td>\n",
       "      <td>7</td>\n",
       "      <td>DIGIT</td>\n",
       "      <td>21770</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>twenty one thousand seven hundred seventy</td>\n",
       "      <td>two one seven seven o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11778</th>\n",
       "      <td>942</td>\n",
       "      <td>4</td>\n",
       "      <td>DIGIT</td>\n",
       "      <td>314</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>three hundred fourteen</td>\n",
       "      <td>three one four</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ELECTRONIC</th>\n",
       "      <th>15976</th>\n",
       "      <td>1289</td>\n",
       "      <td>2</td>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>http://www.knchr.org/dmdocuments/KNCHR%20doc.p...</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>h_letter  _letter t_letter  _letter t_letter  ...</td>\n",
       "      <td>h_letter  _letter t_letter  _letter t_letter  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50368</th>\n",
       "      <td>4136</td>\n",
       "      <td>3</td>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>http://www.jstor.org/stable/2799027;</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>h_letter  _letter t_letter  _letter t_letter  ...</td>\n",
       "      <td>h_letter  _letter t_letter  _letter t_letter  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13932</th>\n",
       "      <td>1117</td>\n",
       "      <td>15</td>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>//web.archive.org/web/20110105051516/http://ww...</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>s_letter l_letter a_letter s_letter h_letter  ...</td>\n",
       "      <td>s_letter l_letter a_letter s_letter h_letter  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FRACTION</th>\n",
       "      <th>71394</th>\n",
       "      <td>5865</td>\n",
       "      <td>3</td>\n",
       "      <td>FRACTION</td>\n",
       "      <td>⅞</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>upsilon</td>\n",
       "      <td>seven eighths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3887</th>\n",
       "      <td>309</td>\n",
       "      <td>0</td>\n",
       "      <td>FRACTION</td>\n",
       "      <td>1/0</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>one meter</td>\n",
       "      <td>one zeroth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70920</th>\n",
       "      <td>5828</td>\n",
       "      <td>2</td>\n",
       "      <td>FRACTION</td>\n",
       "      <td>-133/94</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>minus thirty three hundred ninety fourths</td>\n",
       "      <td>minus one hundred thirty three ninety fourths</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">LETTERS</th>\n",
       "      <th>3751</th>\n",
       "      <td>297</td>\n",
       "      <td>8</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>Vit</td>\n",
       "      <td>RemainSelf</td>\n",
       "      <td>Vit</td>\n",
       "      <td>v i t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90363</th>\n",
       "      <td>7372</td>\n",
       "      <td>6</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>Graz'zt</td>\n",
       "      <td>RemainSelf</td>\n",
       "      <td>Graz'zt</td>\n",
       "      <td>g r a z z t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6848</th>\n",
       "      <td>550</td>\n",
       "      <td>0</td>\n",
       "      <td>LETTERS</td>\n",
       "      <td>Ligi'ne</td>\n",
       "      <td>RemainSelf</td>\n",
       "      <td>Ligi'ne</td>\n",
       "      <td>l i g i n e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">MEASURE</th>\n",
       "      <th>51723</th>\n",
       "      <td>4244</td>\n",
       "      <td>12</td>\n",
       "      <td>MEASURE</td>\n",
       "      <td>2 mA</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two a m</td>\n",
       "      <td>two milli amperes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49134</th>\n",
       "      <td>4039</td>\n",
       "      <td>18</td>\n",
       "      <td>MEASURE</td>\n",
       "      <td>295 ch</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two hundred ninety five hours</td>\n",
       "      <td>two hundred ninety five chains</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023</th>\n",
       "      <td>156</td>\n",
       "      <td>3</td>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.001251 g/cm3</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>zero point o o one two five one sil g per hour</td>\n",
       "      <td>zero point o o one two five one grams per c c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">MONEY</th>\n",
       "      <th>21543</th>\n",
       "      <td>1726</td>\n",
       "      <td>11</td>\n",
       "      <td>MONEY</td>\n",
       "      <td>88.5 million HRK</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>eighty eight point five million yen</td>\n",
       "      <td>eighty eight point five million croatian kunas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21543</th>\n",
       "      <td>1726</td>\n",
       "      <td>11</td>\n",
       "      <td>MONEY</td>\n",
       "      <td>88.5 million HRK</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>eighty eight point five million yen</td>\n",
       "      <td>eighty eight point five million croatian kunas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21543</th>\n",
       "      <td>1726</td>\n",
       "      <td>11</td>\n",
       "      <td>MONEY</td>\n",
       "      <td>88.5 million HRK</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>eighty eight point five million yen</td>\n",
       "      <td>eighty eight point five million croatian kunas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ORDINAL</th>\n",
       "      <th>84918</th>\n",
       "      <td>6945</td>\n",
       "      <td>1</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>II</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two</td>\n",
       "      <td>the second</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74192</th>\n",
       "      <td>6079</td>\n",
       "      <td>10</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>III</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>three</td>\n",
       "      <td>the third</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84918</th>\n",
       "      <td>6945</td>\n",
       "      <td>1</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>II</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>two</td>\n",
       "      <td>the second</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">PLAIN</th>\n",
       "      <th>7721</th>\n",
       "      <td>626</td>\n",
       "      <td>7</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>ms</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>m s</td>\n",
       "      <td>ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8190</th>\n",
       "      <td>659</td>\n",
       "      <td>0</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>MATCH</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>m a t c h</td>\n",
       "      <td>MATCH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69147</th>\n",
       "      <td>5688</td>\n",
       "      <td>0</td>\n",
       "      <td>PLAIN</td>\n",
       "      <td>Issa</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>i s s a</td>\n",
       "      <td>Issa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">PUNCT</th>\n",
       "      <th>59444</th>\n",
       "      <td>4885</td>\n",
       "      <td>5</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84253</th>\n",
       "      <td>6887</td>\n",
       "      <td>18</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84253</th>\n",
       "      <td>6887</td>\n",
       "      <td>18</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">TELEPHONE</th>\n",
       "      <th>34188</th>\n",
       "      <td>2816</td>\n",
       "      <td>16</td>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>40 (1969) 111-124</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>four o sil one nine six nine sil one four sil ...</td>\n",
       "      <td>four o sil one nine six nine sil one one one s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9474</th>\n",
       "      <td>754</td>\n",
       "      <td>11</td>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>3-0 LEAD</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>three sil o sil l e a d</td>\n",
       "      <td>three sil o sil lead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9474</th>\n",
       "      <td>754</td>\n",
       "      <td>11</td>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>3-0 LEAD</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>three sil o sil l e a d</td>\n",
       "      <td>three sil o sil lead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">TIME</th>\n",
       "      <th>88069</th>\n",
       "      <td>7196</td>\n",
       "      <td>10</td>\n",
       "      <td>TIME</td>\n",
       "      <td>18:00:00Z</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>eighteen hours u seconds</td>\n",
       "      <td>eighteen hours zero minutes and zero seconds z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88776</th>\n",
       "      <td>7250</td>\n",
       "      <td>3</td>\n",
       "      <td>TIME</td>\n",
       "      <td>0:02:01</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>zero hours two minutes and one seconds</td>\n",
       "      <td>zero hours two minutes and one second</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88776</th>\n",
       "      <td>7250</td>\n",
       "      <td>3</td>\n",
       "      <td>TIME</td>\n",
       "      <td>0:02:01</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>zero hours two minutes and one seconds</td>\n",
       "      <td>zero hours two minutes and one second</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">VERBATIM</th>\n",
       "      <th>29989</th>\n",
       "      <td>2454</td>\n",
       "      <td>4</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43455</th>\n",
       "      <td>3555</td>\n",
       "      <td>9</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6141</th>\n",
       "      <td>490</td>\n",
       "      <td>7</td>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>-</td>\n",
       "      <td>ToBeNormalized</td>\n",
       "      <td>to</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  sentence_id  token_id    semiotic  \\\n",
       "semiotic                                              \n",
       "CARDINAL   69063         5683         3    CARDINAL   \n",
       "           89790         7334         1    CARDINAL   \n",
       "           2399           184        28    CARDINAL   \n",
       "DATE       65463         5381         6        DATE   \n",
       "           88533         7230        16        DATE   \n",
       "           88533         7230        16        DATE   \n",
       "DECIMAL    71203         5851         5     DECIMAL   \n",
       "           71203         5851         5     DECIMAL   \n",
       "           71203         5851         5     DECIMAL   \n",
       "DIGIT      59585         4895        12       DIGIT   \n",
       "           27785         2270         7       DIGIT   \n",
       "           11778          942         4       DIGIT   \n",
       "ELECTRONIC 15976         1289         2  ELECTRONIC   \n",
       "           50368         4136         3  ELECTRONIC   \n",
       "           13932         1117        15  ELECTRONIC   \n",
       "FRACTION   71394         5865         3    FRACTION   \n",
       "           3887           309         0    FRACTION   \n",
       "           70920         5828         2    FRACTION   \n",
       "LETTERS    3751           297         8     LETTERS   \n",
       "           90363         7372         6     LETTERS   \n",
       "           6848           550         0     LETTERS   \n",
       "MEASURE    51723         4244        12     MEASURE   \n",
       "           49134         4039        18     MEASURE   \n",
       "           2023           156         3     MEASURE   \n",
       "MONEY      21543         1726        11       MONEY   \n",
       "           21543         1726        11       MONEY   \n",
       "           21543         1726        11       MONEY   \n",
       "ORDINAL    84918         6945         1     ORDINAL   \n",
       "           74192         6079        10     ORDINAL   \n",
       "           84918         6945         1     ORDINAL   \n",
       "PLAIN      7721           626         7       PLAIN   \n",
       "           8190           659         0       PLAIN   \n",
       "           69147         5688         0       PLAIN   \n",
       "PUNCT      59444         4885         5       PUNCT   \n",
       "           84253         6887        18       PUNCT   \n",
       "           84253         6887        18       PUNCT   \n",
       "TELEPHONE  34188         2816        16   TELEPHONE   \n",
       "           9474           754        11   TELEPHONE   \n",
       "           9474           754        11   TELEPHONE   \n",
       "TIME       88069         7196        10        TIME   \n",
       "           88776         7250         3        TIME   \n",
       "           88776         7250         3        TIME   \n",
       "VERBATIM   29989         2454         4    VERBATIM   \n",
       "           43455         3555         9    VERBATIM   \n",
       "           6141           490         7    VERBATIM   \n",
       "\n",
       "                                                             before  \\\n",
       "semiotic                                                              \n",
       "CARDINAL   69063                                          310780751   \n",
       "           89790                                            1298015   \n",
       "           2399                                            14356007   \n",
       "DATE       65463                                                00s   \n",
       "           88533                                               1968   \n",
       "           88533                                               1968   \n",
       "DECIMAL    71203                                               .763   \n",
       "           71203                                               .763   \n",
       "           71203                                               .763   \n",
       "DIGIT      59585                                               1271   \n",
       "           27785                                              21770   \n",
       "           11778                                                314   \n",
       "ELECTRONIC 15976  http://www.knchr.org/dmdocuments/KNCHR%20doc.p...   \n",
       "           50368               http://www.jstor.org/stable/2799027;   \n",
       "           13932  //web.archive.org/web/20110105051516/http://ww...   \n",
       "FRACTION   71394                                                  ⅞   \n",
       "           3887                                                 1/0   \n",
       "           70920                                            -133/94   \n",
       "LETTERS    3751                                                 Vit   \n",
       "           90363                                            Graz'zt   \n",
       "           6848                                             Ligi'ne   \n",
       "MEASURE    51723                                               2 mA   \n",
       "           49134                                             295 ch   \n",
       "           2023                                      0.001251 g/cm3   \n",
       "MONEY      21543                                   88.5 million HRK   \n",
       "           21543                                   88.5 million HRK   \n",
       "           21543                                   88.5 million HRK   \n",
       "ORDINAL    84918                                                 II   \n",
       "           74192                                                III   \n",
       "           84918                                                 II   \n",
       "PLAIN      7721                                                  ms   \n",
       "           8190                                               MATCH   \n",
       "           69147                                               Issa   \n",
       "PUNCT      59444                                                  -   \n",
       "           84253                                                  -   \n",
       "           84253                                                  -   \n",
       "TELEPHONE  34188                                  40 (1969) 111-124   \n",
       "           9474                                            3-0 LEAD   \n",
       "           9474                                            3-0 LEAD   \n",
       "TIME       88069                                          18:00:00Z   \n",
       "           88776                                            0:02:01   \n",
       "           88776                                            0:02:01   \n",
       "VERBATIM   29989                                                  -   \n",
       "           43455                                                  -   \n",
       "           6141                                                   -   \n",
       "\n",
       "                           class  \\\n",
       "semiotic                           \n",
       "CARDINAL   69063  ToBeNormalized   \n",
       "           89790  ToBeNormalized   \n",
       "           2399   ToBeNormalized   \n",
       "DATE       65463  ToBeNormalized   \n",
       "           88533  ToBeNormalized   \n",
       "           88533  ToBeNormalized   \n",
       "DECIMAL    71203  ToBeNormalized   \n",
       "           71203  ToBeNormalized   \n",
       "           71203  ToBeNormalized   \n",
       "DIGIT      59585  ToBeNormalized   \n",
       "           27785  ToBeNormalized   \n",
       "           11778  ToBeNormalized   \n",
       "ELECTRONIC 15976  ToBeNormalized   \n",
       "           50368  ToBeNormalized   \n",
       "           13932  ToBeNormalized   \n",
       "FRACTION   71394  ToBeNormalized   \n",
       "           3887   ToBeNormalized   \n",
       "           70920  ToBeNormalized   \n",
       "LETTERS    3751       RemainSelf   \n",
       "           90363      RemainSelf   \n",
       "           6848       RemainSelf   \n",
       "MEASURE    51723  ToBeNormalized   \n",
       "           49134  ToBeNormalized   \n",
       "           2023   ToBeNormalized   \n",
       "MONEY      21543  ToBeNormalized   \n",
       "           21543  ToBeNormalized   \n",
       "           21543  ToBeNormalized   \n",
       "ORDINAL    84918  ToBeNormalized   \n",
       "           74192  ToBeNormalized   \n",
       "           84918  ToBeNormalized   \n",
       "PLAIN      7721   ToBeNormalized   \n",
       "           8190   ToBeNormalized   \n",
       "           69147  ToBeNormalized   \n",
       "PUNCT      59444  ToBeNormalized   \n",
       "           84253  ToBeNormalized   \n",
       "           84253  ToBeNormalized   \n",
       "TELEPHONE  34188  ToBeNormalized   \n",
       "           9474   ToBeNormalized   \n",
       "           9474   ToBeNormalized   \n",
       "TIME       88069  ToBeNormalized   \n",
       "           88776  ToBeNormalized   \n",
       "           88776  ToBeNormalized   \n",
       "VERBATIM   29989  ToBeNormalized   \n",
       "           43455  ToBeNormalized   \n",
       "           6141   ToBeNormalized   \n",
       "\n",
       "                                                              after  \\\n",
       "semiotic                                                              \n",
       "CARDINAL   69063  thirty one million seventy eight thousand seve...   \n",
       "           89790                      one two nine eight o one five   \n",
       "           2399   one million four hundred thirty five thousand ...   \n",
       "DATE       65463                                           hundreds   \n",
       "           88533              one thousand nine hundred sixty eight   \n",
       "           88533              one thousand nine hundred sixty eight   \n",
       "DECIMAL    71203                          seven hundred sixty three   \n",
       "           71203                          seven hundred sixty three   \n",
       "           71203                          seven hundred sixty three   \n",
       "DIGIT      59585                                 twelve seventy one   \n",
       "           27785          twenty one thousand seven hundred seventy   \n",
       "           11778                             three hundred fourteen   \n",
       "ELECTRONIC 15976  h_letter  _letter t_letter  _letter t_letter  ...   \n",
       "           50368  h_letter  _letter t_letter  _letter t_letter  ...   \n",
       "           13932  s_letter l_letter a_letter s_letter h_letter  ...   \n",
       "FRACTION   71394                                            upsilon   \n",
       "           3887                                           one meter   \n",
       "           70920          minus thirty three hundred ninety fourths   \n",
       "LETTERS    3751                                                 Vit   \n",
       "           90363                                            Graz'zt   \n",
       "           6848                                             Ligi'ne   \n",
       "MEASURE    51723                                            two a m   \n",
       "           49134                      two hundred ninety five hours   \n",
       "           2023      zero point o o one two five one sil g per hour   \n",
       "MONEY      21543                eighty eight point five million yen   \n",
       "           21543                eighty eight point five million yen   \n",
       "           21543                eighty eight point five million yen   \n",
       "ORDINAL    84918                                                two   \n",
       "           74192                                              three   \n",
       "           84918                                                two   \n",
       "PLAIN      7721                                                 m s   \n",
       "           8190                                           m a t c h   \n",
       "           69147                                            i s s a   \n",
       "PUNCT      59444                                                 to   \n",
       "           84253                                                 to   \n",
       "           84253                                                 to   \n",
       "TELEPHONE  34188  four o sil one nine six nine sil one four sil ...   \n",
       "           9474                             three sil o sil l e a d   \n",
       "           9474                             three sil o sil l e a d   \n",
       "TIME       88069                           eighteen hours u seconds   \n",
       "           88776             zero hours two minutes and one seconds   \n",
       "           88776             zero hours two minutes and one seconds   \n",
       "VERBATIM   29989                                                 to   \n",
       "           43455                                                 to   \n",
       "           6141                                                  to   \n",
       "\n",
       "                                                              truth  \n",
       "semiotic                                                             \n",
       "CARDINAL   69063  three hundred ten million seven hundred eighty...  \n",
       "           89790  one million two hundred ninety eight thousand ...  \n",
       "           2399   fourteen million three hundred fifty six thous...  \n",
       "DATE       65463                                                o o  \n",
       "           88533                               nineteen sixty eight  \n",
       "           88533                               nineteen sixty eight  \n",
       "DECIMAL    71203                              point seven six three  \n",
       "           71203                              point seven six three  \n",
       "           71203                              point seven six three  \n",
       "DIGIT      59585                                  one two seven one  \n",
       "           27785                              two one seven seven o  \n",
       "           11778                                     three one four  \n",
       "ELECTRONIC 15976  h_letter  _letter t_letter  _letter t_letter  ...  \n",
       "           50368  h_letter  _letter t_letter  _letter t_letter  ...  \n",
       "           13932  s_letter l_letter a_letter s_letter h_letter  ...  \n",
       "FRACTION   71394                                      seven eighths  \n",
       "           3887                                          one zeroth  \n",
       "           70920      minus one hundred thirty three ninety fourths  \n",
       "LETTERS    3751                                               v i t  \n",
       "           90363                                        g r a z z t  \n",
       "           6848                                         l i g i n e  \n",
       "MEASURE    51723                                  two milli amperes  \n",
       "           49134                     two hundred ninety five chains  \n",
       "           2023       zero point o o one two five one grams per c c  \n",
       "MONEY      21543     eighty eight point five million croatian kunas  \n",
       "           21543     eighty eight point five million croatian kunas  \n",
       "           21543     eighty eight point five million croatian kunas  \n",
       "ORDINAL    84918                                         the second  \n",
       "           74192                                          the third  \n",
       "           84918                                         the second  \n",
       "PLAIN      7721                                                  ms  \n",
       "           8190                                               MATCH  \n",
       "           69147                                               Issa  \n",
       "PUNCT      59444                                                  -  \n",
       "           84253                                                  -  \n",
       "           84253                                                  -  \n",
       "TELEPHONE  34188  four o sil one nine six nine sil one one one s...  \n",
       "           9474                                three sil o sil lead  \n",
       "           9474                                three sil o sil lead  \n",
       "TIME       88069     eighteen hours zero minutes and zero seconds z  \n",
       "           88776              zero hours two minutes and one second  \n",
       "           88776              zero hours two minutes and one second  \n",
       "VERBATIM   29989                                                  -  \n",
       "           43455                                                  -  \n",
       "           6141                                                   -  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mistakes_grouped.apply(lambda x: x.sample(n=3, replace=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "mistakes.to_csv(result_dir+'mistakes.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Comparison\n",
    "<a id=\"comparison\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Semiotic Class</th>\n",
       "      <th>En Count</th>\n",
       "      <th>En Accuracy</th>\n",
       "      <th>Ru Count</th>\n",
       "      <th>Ru Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>92416.0</td>\n",
       "      <td>0.997</td>\n",
       "      <td>93184</td>\n",
       "      <td>0.993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>68029.0</td>\n",
       "      <td>0.998</td>\n",
       "      <td>60747</td>\n",
       "      <td>0.999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>17726.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>20263</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>2808.0</td>\n",
       "      <td>0.999</td>\n",
       "      <td>1495</td>\n",
       "      <td>0.976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TRANS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4103</td>\n",
       "      <td>0.921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>0.971</td>\n",
       "      <td>1839</td>\n",
       "      <td>0.991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>1067.0</td>\n",
       "      <td>0.989</td>\n",
       "      <td>2387</td>\n",
       "      <td>0.940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>894.0</td>\n",
       "      <td>0.980</td>\n",
       "      <td>1298</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>142.0</td>\n",
       "      <td>0.986</td>\n",
       "      <td>409</td>\n",
       "      <td>0.883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>103.0</td>\n",
       "      <td>0.971</td>\n",
       "      <td>427</td>\n",
       "      <td>0.956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>89.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>60</td>\n",
       "      <td>0.867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>21.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.865</td>\n",
       "      <td>16</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.972</td>\n",
       "      <td>19</td>\n",
       "      <td>0.842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.923</td>\n",
       "      <td>23</td>\n",
       "      <td>0.826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>TIME</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.750</td>\n",
       "      <td>8</td>\n",
       "      <td>0.750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Semiotic Class   En Count   En Accuracy   Ru Count   Ru Accuracy\n",
       "0             ALL    92416.0         0.997      93184         0.993\n",
       "1           PLAIN    68029.0         0.998      60747         0.999\n",
       "2           PUNCT    17726.0         1.000      20263         1.000\n",
       "3            DATE     2808.0         0.999       1495         0.976\n",
       "4           TRANS        NaN           NaN       4103         0.921\n",
       "5         LETTERS     1404.0         0.971       1839         0.991\n",
       "6        CARDINAL     1067.0         0.989       2387         0.940\n",
       "7        VERBATIM      894.0         0.980       1298         1.000\n",
       "8         MEASURE      142.0         0.986        409         0.883\n",
       "9         ORDINAL      103.0         0.971        427         0.956\n",
       "10        DECIMAL       89.0         1.000         60         0.867\n",
       "11     ELECTRONIC       21.0         1.000          2         1.000\n",
       "12          DIGIT       37.0         0.865         16         1.000\n",
       "13          MONEY       36.0         0.972         19         0.842\n",
       "14       FRACTION       13.0         0.923         23         0.826\n",
       "15           TIME        8.0         0.750          8         0.750"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base = pd.read_csv('../results/base-paper_classwise_accuracy.csv')\n",
    "base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.994267</td>\n",
       "      <td>92451</td>\n",
       "      <td>91921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADDRESS</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.994214</td>\n",
       "      <td>1037</td>\n",
       "      <td>1031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.997175</td>\n",
       "      <td>2832</td>\n",
       "      <td>2824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>0.989130</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>44</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>0.734694</td>\n",
       "      <td>49</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.971611</td>\n",
       "      <td>1409</td>\n",
       "      <td>1369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>142</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.980583</td>\n",
       "      <td>103</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.993961</td>\n",
       "      <td>67894</td>\n",
       "      <td>67484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.998873</td>\n",
       "      <td>17746</td>\n",
       "      <td>17726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.994006</td>\n",
       "      <td>1001</td>\n",
       "      <td>995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  accuracy  count  correct\n",
       "0             ALL  0.994267  92451    91921\n",
       "1         ADDRESS  1.000000      4        4\n",
       "2        CARDINAL  0.994214   1037     1031\n",
       "3            DATE  0.997175   2832     2824\n",
       "4         DECIMAL  0.989130     92       91\n",
       "5           DIGIT  0.795455     44       35\n",
       "6      ELECTRONIC  0.734694     49       36\n",
       "7        FRACTION  0.687500     16       11\n",
       "8         LETTERS  0.971611   1409     1369\n",
       "9         MEASURE  0.971831    142      138\n",
       "10          MONEY  0.972973     37       36\n",
       "11        ORDINAL  0.980583    103      101\n",
       "12          PLAIN  0.993961  67894    67484\n",
       "13          PUNCT  0.998873  17746    17726\n",
       "14      TELEPHONE  0.918919     37       34\n",
       "15           TIME  0.750000      8        6\n",
       "16       VERBATIM  0.994006   1001      995"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_accuracy = pd.read_csv('../results/english/classwise_accuracy.csv')\n",
    "en_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.992875</td>\n",
       "      <td>93196</td>\n",
       "      <td>92532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.941792</td>\n",
       "      <td>2388</td>\n",
       "      <td>2249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.973244</td>\n",
       "      <td>1495</td>\n",
       "      <td>1455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>60</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>48</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.608696</td>\n",
       "      <td>23</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.990761</td>\n",
       "      <td>1840</td>\n",
       "      <td>1823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.897810</td>\n",
       "      <td>411</td>\n",
       "      <td>369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.946136</td>\n",
       "      <td>427</td>\n",
       "      <td>404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.994688</td>\n",
       "      <td>64764</td>\n",
       "      <td>64420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.999852</td>\n",
       "      <td>20264</td>\n",
       "      <td>20261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TELEPHONE</td>\n",
       "      <td>0.820225</td>\n",
       "      <td>89</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.998512</td>\n",
       "      <td>1344</td>\n",
       "      <td>1342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  accuracy  count  correct\n",
       "0             ALL  0.992875  93196    92532\n",
       "1        CARDINAL  0.941792   2388     2249\n",
       "2            DATE  0.973244   1495     1455\n",
       "3         DECIMAL  0.900000     60       54\n",
       "4           DIGIT  1.000000     16       16\n",
       "5      ELECTRONIC  0.604167     48       29\n",
       "6        FRACTION  0.608696     23       14\n",
       "7         LETTERS  0.990761   1840     1823\n",
       "8         MEASURE  0.897810    411      369\n",
       "9           MONEY  0.894737     19       17\n",
       "10        ORDINAL  0.946136    427      404\n",
       "11          PLAIN  0.994688  64764    64420\n",
       "12          PUNCT  0.999852  20264    20261\n",
       "13      TELEPHONE  0.820225     89       73\n",
       "14           TIME  0.750000      8        6\n",
       "15       VERBATIM  0.998512   1344     1342"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ru_accuracy = pd.read_csv('../results/russian/classwise_accuracy.csv')\n",
    "ru_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_base = pd.DataFrame(columns=['semiotic-class', 'base accuracy', 'base count'])\n",
    "en_base['semiotic-class'] = base['Semiotic Class']\n",
    "en_base['base accuracy'] = base[' En Accuracy']\n",
    "en_base['base count'] = base[' En Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "ru_base = pd.DataFrame(columns=['semiotic-class', 'base accuracy', 'base count'])\n",
    "ru_base['semiotic-class'] = base['Semiotic Class']\n",
    "ru_base['base accuracy'] = base[' Ru Accuracy']\n",
    "ru_base['base count'] = base[' Ru Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>base accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>base count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.994267</td>\n",
       "      <td>0.997</td>\n",
       "      <td>92451</td>\n",
       "      <td>92416.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.993961</td>\n",
       "      <td>0.998</td>\n",
       "      <td>67894</td>\n",
       "      <td>68029.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.998873</td>\n",
       "      <td>1.000</td>\n",
       "      <td>17746</td>\n",
       "      <td>17726.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.997175</td>\n",
       "      <td>0.999</td>\n",
       "      <td>2832</td>\n",
       "      <td>2808.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.971611</td>\n",
       "      <td>0.971</td>\n",
       "      <td>1409</td>\n",
       "      <td>1404.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.994214</td>\n",
       "      <td>0.989</td>\n",
       "      <td>1037</td>\n",
       "      <td>1067.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.994006</td>\n",
       "      <td>0.980</td>\n",
       "      <td>1001</td>\n",
       "      <td>894.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>0.986</td>\n",
       "      <td>142</td>\n",
       "      <td>142.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.980583</td>\n",
       "      <td>0.971</td>\n",
       "      <td>103</td>\n",
       "      <td>103.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>0.989130</td>\n",
       "      <td>1.000</td>\n",
       "      <td>92</td>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>0.734694</td>\n",
       "      <td>1.000</td>\n",
       "      <td>49</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>0.865</td>\n",
       "      <td>44</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>0.972</td>\n",
       "      <td>37</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>0.923</td>\n",
       "      <td>16</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750</td>\n",
       "      <td>8</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  accuracy  base accuracy  count  base count\n",
       "0             ALL  0.994267          0.997  92451     92416.0\n",
       "1           PLAIN  0.993961          0.998  67894     68029.0\n",
       "2           PUNCT  0.998873          1.000  17746     17726.0\n",
       "3            DATE  0.997175          0.999   2832      2808.0\n",
       "4         LETTERS  0.971611          0.971   1409      1404.0\n",
       "5        CARDINAL  0.994214          0.989   1037      1067.0\n",
       "6        VERBATIM  0.994006          0.980   1001       894.0\n",
       "7         MEASURE  0.971831          0.986    142       142.0\n",
       "8         ORDINAL  0.980583          0.971    103       103.0\n",
       "9         DECIMAL  0.989130          1.000     92        89.0\n",
       "10     ELECTRONIC  0.734694          1.000     49        21.0\n",
       "11          DIGIT  0.795455          0.865     44        37.0\n",
       "12          MONEY  0.972973          0.972     37        36.0\n",
       "13       FRACTION  0.687500          0.923     16        13.0\n",
       "14           TIME  0.750000          0.750      8         8.0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_compared = pd.merge(en_base, en_accuracy, on='semiotic-class')\n",
    "en_compared[['semiotic-class', 'accuracy', 'base accuracy', 'count', 'base count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>base accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>base count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.992875</td>\n",
       "      <td>0.993</td>\n",
       "      <td>93196</td>\n",
       "      <td>93184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.994688</td>\n",
       "      <td>0.999</td>\n",
       "      <td>64764</td>\n",
       "      <td>60747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.999852</td>\n",
       "      <td>1.000</td>\n",
       "      <td>20264</td>\n",
       "      <td>20263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.973244</td>\n",
       "      <td>0.976</td>\n",
       "      <td>1495</td>\n",
       "      <td>1495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.990761</td>\n",
       "      <td>0.991</td>\n",
       "      <td>1840</td>\n",
       "      <td>1839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.941792</td>\n",
       "      <td>0.940</td>\n",
       "      <td>2388</td>\n",
       "      <td>2387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.998512</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1344</td>\n",
       "      <td>1298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.897810</td>\n",
       "      <td>0.883</td>\n",
       "      <td>411</td>\n",
       "      <td>409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.946136</td>\n",
       "      <td>0.956</td>\n",
       "      <td>427</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.867</td>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>1.000</td>\n",
       "      <td>48</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.842</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.608696</td>\n",
       "      <td>0.826</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  accuracy  base accuracy  count  base count\n",
       "0             ALL  0.992875          0.993  93196       93184\n",
       "1           PLAIN  0.994688          0.999  64764       60747\n",
       "2           PUNCT  0.999852          1.000  20264       20263\n",
       "3            DATE  0.973244          0.976   1495        1495\n",
       "4         LETTERS  0.990761          0.991   1840        1839\n",
       "5        CARDINAL  0.941792          0.940   2388        2387\n",
       "6        VERBATIM  0.998512          1.000   1344        1298\n",
       "7         MEASURE  0.897810          0.883    411         409\n",
       "8         ORDINAL  0.946136          0.956    427         427\n",
       "9         DECIMAL  0.900000          0.867     60          60\n",
       "10     ELECTRONIC  0.604167          1.000     48           2\n",
       "11          DIGIT  1.000000          1.000     16          16\n",
       "12          MONEY  0.894737          0.842     19          19\n",
       "13       FRACTION  0.608696          0.826     23          23\n",
       "14           TIME  0.750000          0.750      8           8"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ru_compared = pd.merge(ru_base, ru_accuracy, on='semiotic-class')\n",
    "ru_compared[['semiotic-class', 'accuracy', 'base accuracy', 'count', 'base count']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Latex Output**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>semiotic-class</th>\n",
       "      <th>base accuracy</th>\n",
       "      <th>base count</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>count</th>\n",
       "      <th>correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALL</td>\n",
       "      <td>0.997</td>\n",
       "      <td>92416.0</td>\n",
       "      <td>0.994267</td>\n",
       "      <td>92451</td>\n",
       "      <td>91921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PLAIN</td>\n",
       "      <td>0.998</td>\n",
       "      <td>68029.0</td>\n",
       "      <td>0.993961</td>\n",
       "      <td>67894</td>\n",
       "      <td>67484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>1.000</td>\n",
       "      <td>17726.0</td>\n",
       "      <td>0.998873</td>\n",
       "      <td>17746</td>\n",
       "      <td>17726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATE</td>\n",
       "      <td>0.999</td>\n",
       "      <td>2808.0</td>\n",
       "      <td>0.997175</td>\n",
       "      <td>2832</td>\n",
       "      <td>2824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LETTERS</td>\n",
       "      <td>0.971</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>0.971611</td>\n",
       "      <td>1409</td>\n",
       "      <td>1369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>0.989</td>\n",
       "      <td>1067.0</td>\n",
       "      <td>0.994214</td>\n",
       "      <td>1037</td>\n",
       "      <td>1031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>VERBATIM</td>\n",
       "      <td>0.980</td>\n",
       "      <td>894.0</td>\n",
       "      <td>0.994006</td>\n",
       "      <td>1001</td>\n",
       "      <td>995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MEASURE</td>\n",
       "      <td>0.986</td>\n",
       "      <td>142.0</td>\n",
       "      <td>0.971831</td>\n",
       "      <td>142</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>0.971</td>\n",
       "      <td>103.0</td>\n",
       "      <td>0.980583</td>\n",
       "      <td>103</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DECIMAL</td>\n",
       "      <td>1.000</td>\n",
       "      <td>89.0</td>\n",
       "      <td>0.989130</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ELECTRONIC</td>\n",
       "      <td>1.000</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.734694</td>\n",
       "      <td>49</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DIGIT</td>\n",
       "      <td>0.865</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>44</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>MONEY</td>\n",
       "      <td>0.972</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FRACTION</td>\n",
       "      <td>0.923</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TIME</td>\n",
       "      <td>0.750</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   semiotic-class  base accuracy  base count  accuracy  count  correct\n",
       "0             ALL          0.997     92416.0  0.994267  92451    91921\n",
       "1           PLAIN          0.998     68029.0  0.993961  67894    67484\n",
       "2           PUNCT          1.000     17726.0  0.998873  17746    17726\n",
       "3            DATE          0.999      2808.0  0.997175   2832     2824\n",
       "4         LETTERS          0.971      1404.0  0.971611   1409     1369\n",
       "5        CARDINAL          0.989      1067.0  0.994214   1037     1031\n",
       "6        VERBATIM          0.980       894.0  0.994006   1001      995\n",
       "7         MEASURE          0.986       142.0  0.971831    142      138\n",
       "8         ORDINAL          0.971       103.0  0.980583    103      101\n",
       "9         DECIMAL          1.000        89.0  0.989130     92       91\n",
       "10     ELECTRONIC          1.000        21.0  0.734694     49       36\n",
       "11          DIGIT          0.865        37.0  0.795455     44       35\n",
       "12          MONEY          0.972        36.0  0.972973     37       36\n",
       "13       FRACTION          0.923        13.0  0.687500     16       11\n",
       "14           TIME          0.750         8.0  0.750000      8        6"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_compared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} & semiotic-class &  base count &  count &  base accuracy &  accuracy \\\\\n",
      "\\midrule\n",
      "0  &            ALL &     92416.0 &  92451 &          0.997 &  0.994267 \\\\\n",
      "1  &          PLAIN &     68029.0 &  67894 &          0.998 &  0.993961 \\\\\n",
      "2  &          PUNCT &     17726.0 &  17746 &          1.000 &  0.998873 \\\\\n",
      "3  &           DATE &      2808.0 &   2832 &          0.999 &  0.997175 \\\\\n",
      "4  &        LETTERS &      1404.0 &   1409 &          0.971 &  0.971611 \\\\\n",
      "5  &       CARDINAL &      1067.0 &   1037 &          0.989 &  0.994214 \\\\\n",
      "6  &       VERBATIM &       894.0 &   1001 &          0.980 &  0.994006 \\\\\n",
      "7  &        MEASURE &       142.0 &    142 &          0.986 &  0.971831 \\\\\n",
      "8  &        ORDINAL &       103.0 &    103 &          0.971 &  0.980583 \\\\\n",
      "9  &        DECIMAL &        89.0 &     92 &          1.000 &  0.989130 \\\\\n",
      "10 &     ELECTRONIC &        21.0 &     49 &          1.000 &  0.734694 \\\\\n",
      "11 &          DIGIT &        37.0 &     44 &          0.865 &  0.795455 \\\\\n",
      "12 &          MONEY &        36.0 &     37 &          0.972 &  0.972973 \\\\\n",
      "13 &       FRACTION &        13.0 &     16 &          0.923 &  0.687500 \\\\\n",
      "14 &           TIME &         8.0 &      8 &          0.750 &  0.750000 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(en_compared[['semiotic-class', 'base count','count','base accuracy','accuracy']].to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} & semiotic-class &  base count &  count &  base accuracy &  accuracy \\\\\n",
      "\\midrule\n",
      "0  &            ALL &       93184 &  93196 &          0.993 &  0.992875 \\\\\n",
      "1  &          PLAIN &       60747 &  64764 &          0.999 &  0.994688 \\\\\n",
      "2  &          PUNCT &       20263 &  20264 &          1.000 &  0.999852 \\\\\n",
      "3  &           DATE &        1495 &   1495 &          0.976 &  0.973244 \\\\\n",
      "4  &        LETTERS &        1839 &   1840 &          0.991 &  0.990761 \\\\\n",
      "5  &       CARDINAL &        2387 &   2388 &          0.940 &  0.941792 \\\\\n",
      "6  &       VERBATIM &        1298 &   1344 &          1.000 &  0.998512 \\\\\n",
      "7  &        MEASURE &         409 &    411 &          0.883 &  0.897810 \\\\\n",
      "8  &        ORDINAL &         427 &    427 &          0.956 &  0.946136 \\\\\n",
      "9  &        DECIMAL &          60 &     60 &          0.867 &  0.900000 \\\\\n",
      "10 &     ELECTRONIC &           2 &     48 &          1.000 &  0.604167 \\\\\n",
      "11 &          DIGIT &          16 &     16 &          1.000 &  1.000000 \\\\\n",
      "12 &          MONEY &          19 &     19 &          0.842 &  0.894737 \\\\\n",
      "13 &       FRACTION &          23 &     23 &          0.826 &  0.608696 \\\\\n",
      "14 &           TIME &           8 &      8 &          0.750 &  0.750000 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(ru_compared[['semiotic-class', 'base count','count','base accuracy','accuracy']].to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Conclusion\n",
    "<a id=\"conclusion\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**English Normalization Accuracy: 99.4% **"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Russian Normalization Accuracy: 99.3% **"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}