Quora Challenge Using Classifiers and Hand Crafted Features - submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Duplicate Question Pairs Detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__In this notebook, we implement our solution to \"Duplicate Question Pairs Detection\" with classifiers using hand-crafted features.__\n",
    "\n",
    "_This notebook is divided as follows_:\n",
    "> 1. Data Loading\n",
    "> 2. Feature Extraction\n",
    "> 3. Model Fitting\n",
    "> 4. Evaluation of our Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Import Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shared/anaconda3/lib/python3.6/site-packages/fuzzywuzzy/fuzz.py:35: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from __future__ import unicode_literals\n",
    "import pandas as pd\n",
    "import pyemd as emd\n",
    "import numpy as np\n",
    "import os\n",
    "from gensim.models import KeyedVectors\n",
    "import gensim\n",
    "from nltk.tokenize import word_tokenize\n",
    "from scipy import spatial\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from nltk.corpus import wordnet as wn\n",
    "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
    "import spacy\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn import svm\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import preprocessing\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report\n",
    "import networkx as nx\n",
    "import pickle\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "from scipy.stats import skew, kurtosis\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pickle\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create Instances of PorterStemmer, Spacy, SentimentIntensityAnalyzer\n",
    "##### Create set of stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = PorterStemmer()\n",
    "nlp = spacy.load('en')\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "stop = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data Loading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Load Train and Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('data/test.csv', encoding=\"utf-8\").fillna(\"\")\n",
    "df_train = pd.read_csv('data/train.csv', encoding=\"utf-8\").fillna(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Load Word2Vec Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model = KeyedVectors.load_word2vec_format('google_news_word_vectors/GoogleNews-vectors-negative300.bin', binary = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__On inspecting the training and testing data, we observed that there is a qid associated with the questions in training data but they are not present in the testing data. Hence in this section we create a custom mapping of all train and test data to ID values__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_map = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Map Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df_train.iterrows():\n",
    "    \n",
    "    question_id_1 = row[\"qid1\"]\n",
    "    question_id_2 = row[\"qid2\"]\n",
    "    \n",
    "    q1 = row['question1'].strip(' ')\n",
    "    q2 = row['question2'].strip(' ')\n",
    "    \n",
    "    if q1 not in question_map:\n",
    "        question_map[q1] = question_id_1\n",
    "        \n",
    "    if q2 not in question_map:\n",
    "        question_map[q2] = question_id_2\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Map Testing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "count = max(question_map.values())\n",
    "\n",
    "for index, row in df_test.iterrows():\n",
    "    \n",
    "    q1 = row['question1'].strip(' ')\n",
    "    q2 = row['question2'].strip(' ')\n",
    "    \n",
    "    \n",
    "    if q1 not in question_map:\n",
    "        question_map[q1] = count + 1\n",
    "        count = count + 1\n",
    "        \n",
    "    if q2 not in question_map:\n",
    "        question_map[q2] = count + 1\n",
    "        count = count + 1\n",
    "        \n",
    "    if(index%500000 == 0):\n",
    "        print(index)\n",
    "        print(len(question_map))\n",
    "        print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__In this section, we extract the features for our dataset.__\n",
    "\n",
    "For our dataset, we extract three kinds of features:\n",
    "> 1. Graph Based Features\n",
    "> 2. Language Based Features\n",
    "> 3. Similarity Based Features\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Graph Based Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We extract the following graph based features from our data:\n",
    "> PageRank Scores of the Questions <br>\n",
    "> Common Neighbors of the Questions <br>\n",
    "> Degree of the Questions <br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  To represent our dataset as a graph, we create an adjecency list of all our questions as shown in the following code. <br>\n",
    "##### Degree of a node can be easily computed by computing the length of the adjacency list for any node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "#questions_map contains the mapping from question to its index in the train and test combined\n",
    "\n",
    "qid_graph = {}\n",
    "\n",
    "Data = [df_train, df_test]\n",
    "    \n",
    "for data in Data:\n",
    "    \n",
    "    for index, row in data.iterrows():\n",
    "        \n",
    "        q1 = row['question1'].strip(' ')\n",
    "        q2 = row['question2'].strip(' ')\n",
    "        \n",
    "        question_id_1 = question_map.get(q1)\n",
    "        question_id_2 = question_map.get(q2)\n",
    "\n",
    "        qid_graph.setdefault(question_id_1, []).append(question_id_2)\n",
    "        qid_graph.setdefault(question_id_2, []).append(question_id_1)\n",
    "     \n",
    "    print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pagerank(qid_graph):\n",
    "\n",
    "    MAX_ITER = 20 \n",
    "    d = 0.85\n",
    "    \n",
    "    #Initializing -- every node gets a uniform value!\n",
    "    pagerank_dict = {i:1/len(qid_graph) for i in qid_graph}\n",
    "    num_nodes = len(pagerank_dict)\n",
    "    \n",
    "    for iter in range(0, MAX_ITER):\n",
    "        \n",
    "        for node in qid_graph:    \n",
    "            local_pr = 0\n",
    "            \n",
    "            for neighbor in qid_graph[node]:\n",
    "                local_pr += pagerank_dict[neighbor]/len(qid_graph[neighbor])\n",
    "            \n",
    "            pagerank_dict[node] = (1-d)/num_nodes + d*local_pr\n",
    "\n",
    "    return pagerank_dict\n",
    "\n",
    "pagerank_dict = pagerank(qid_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def common_neighbours(id1,id2):\n",
    "    \n",
    "    try:\n",
    "        list1 = qid_graph[id1]\n",
    "        list2 = qid_graph[id2]\n",
    "    except:\n",
    "        return 0\n",
    "    \n",
    "    return len(list(set(list1).intersection(list2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2. Language Based Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We extract the following language based features from our dataset:\n",
    "\n",
    "> 1. Word2vec embedding representation of sentence\n",
    "> 2. Dot product of Tfidf vectors of question pair\n",
    "> 3. Common entities in question pairs\n",
    "> 4. Common nouns in question pairs\n",
    "> 5. Jaccard similarity between embeddigs of question pair\n",
    "> 6. Canberra similarity between embeddigs of question pair\n",
    "> 7. Cityblock similarity between embeddigs of question pair\n",
    "> 8. Eucledian similarity between embeddigs of question pair\n",
    "> 9. Cosine similarity between embeddigs of question pair\n",
    "> 10. Minkowski similarity between embeddigs of question pair\n",
    "> 11. Braycurtis similarity between embeddigs of question pair\n",
    "> 12. Wordmover distance between embeddings\n",
    "> 13. Skew and kurtosis\n",
    "> 14. Sentiment of questions\n",
    "> 15. Common number of bigrams\n",
    "> 16. Fuzz ratio, partial ratio, token sort ratio of question pairs\n",
    "> 17. Type of question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Average Word2vec title\n",
    "def vector_title(sentence, w2v_model):\n",
    "    \n",
    "    sentence = word_tokenize(sentence)\n",
    "    sentence_new = []\n",
    "    \n",
    "    #Apply stemming\n",
    "    for word in sentence:\n",
    "        sentence_new.append(ps.stem(word))  \n",
    "    sentence = sentence_new \n",
    "    \n",
    "    if (len(sentence) == 0) or (type(sentence) == float):\n",
    "        return \"Error\"\n",
    "    \n",
    "    vec = np.zeros(300)\n",
    "    for w in sentence:\n",
    "        try:\n",
    "            vec += w2v_model[w]\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    return vec/len(sentence)\n",
    "\n",
    "\n",
    "def tfidf(d1, d2):\n",
    "    intersect = set(d1).intersection(set(d2))\n",
    "    if len(intersect) == 0:\n",
    "        return 0.0\n",
    "\n",
    "    tid = TfidfVectorizer(norm='l2',min_df=0, use_idf=True, \n",
    "            smooth_idf=False, sublinear_tf=True, tokenizer=tokenize)\n",
    "    try:\n",
    "        represent = tid.fit_transform([d1, d2]).toarray()\n",
    "    except ValueError:\n",
    "        return 0.0\n",
    "    \n",
    "    \n",
    "    # Taking dot product\n",
    "    product = 0\n",
    "    sum1 = 0\n",
    "    sum2 = 0\n",
    "    for i in range(len(represent[0])):\n",
    "        product += represent[0][i] * represent[1][i]\n",
    "        sum1 += pow(represent[0][i], 2)\n",
    "        sum2 += pow(represent[1][i], 2)\n",
    "\n",
    "    denom = math.sqrt(sum1 * sum2)\n",
    "    if int(denom) == 0:\n",
    "        return 0.0\n",
    "\n",
    "    return float(product)/denom\n",
    "\n",
    "\n",
    "\n",
    "def tokenize(d):\n",
    "    clean = re.sub(r'[^\\w]', ' ', d)\n",
    "    tk = clean.lower().split()\n",
    "    return [word for word in tk if word not in stop]\n",
    "\n",
    "\n",
    "#Statistics to measure the common entities in both the sentence.\n",
    "def common_ne_score(q1,q2):\n",
    "\n",
    "    q1 = nlp(q1)\n",
    "    q2 = nlp(q2)\n",
    "\n",
    "    q1_ne = q1.ents\n",
    "    q2_ne = q2.ents\n",
    "\n",
    "    q1_ne = set([str(i) for i in q1_ne])\n",
    "    q2_ne = set([str(i) for i in q2_ne])\n",
    "\n",
    "    common_ne = len(q1_ne.intersection(q2_ne))\n",
    "\n",
    "    if len(q1_ne) + len(q2_ne) == 0:\n",
    "        return 0\n",
    "    \n",
    "    return common_ne/(float)(len(q1_ne) + len(q2_ne) - common_ne)\n",
    "\n",
    "\n",
    "\n",
    "#Statistics to measure the common nouns in both the sentence.\n",
    "def common_chunk_score(q1,q2):\n",
    "\n",
    "    q1 = nlp(q1)\n",
    "    q2 = nlp(q2)\n",
    "\n",
    "    q1_nc = q1.noun_chunks\n",
    "    q2_nc = q2.noun_chunks\n",
    "\n",
    "    q1_nc = set([str(i) for i in q1_nc])\n",
    "    q2_nc = set([str(i) for i in q2_nc])\n",
    "\n",
    "    common_nc = len(q1_nc.intersection(q2_nc))\n",
    "\n",
    "    if len(q1_nc) + len(q2_nc) == 0:\n",
    "        return 0\n",
    "    \n",
    "    return common_nc/(float)(len(q1_nc) + len(q2_nc) - common_nc)\n",
    "\n",
    "\n",
    "def common_bigrams(q1, q2):\n",
    "    text = [q1]\n",
    "    q1_bigrams = [b for l in text for b in zip(l.split(\" \")[:-1], l.split(\" \")[1:])]\n",
    "    \n",
    "    text = [q2]\n",
    "    q2_bigrams = [b for l in text for b in zip(l.split(\" \")[:-1], l.split(\" \")[1:])]\n",
    "    \n",
    "    #print(q1_bigrams)\n",
    "    #print(q2_bigrams)\n",
    "\n",
    "    count = 0\n",
    "    for pair in q1_bigrams:\n",
    "        if pair in q2_bigrams:\n",
    "            count = count + 1\n",
    "    \n",
    "    return count\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features(df, w2v_model, nolabels=False):\n",
    "    \n",
    "    features = []\n",
    "    labels = []\n",
    "    for index, row in df.iterrows():\n",
    "            \n",
    "        q1 = row['question1']\n",
    "        q2 = row['question2']\n",
    "        \n",
    "        q1_index = question_map.get(q1)\n",
    "        q2_index = question_map.get(q2)      \n",
    "        \n",
    "        if q1 == \"\" or q2 == \"\":\n",
    "            current_feature = [0]*2\n",
    "            features.append(current_feature)            \n",
    "            continue\n",
    "        \n",
    "        if not nolabels:\n",
    "            label = row['is_duplicate']\n",
    "        else:\n",
    "            label = None\n",
    "        \n",
    "        if type(q1) == float or type(q2) == float:\n",
    "            current_feature = [0]*2\n",
    "            features.append(current_feature)            \n",
    "            continue\n",
    "\n",
    "        q1 = re.sub(r'[^\\w]', ' ', q1)\n",
    "        q2 = re.sub(r'[^\\w]', ' ', q2)\n",
    "\n",
    "        q1 = q1.lower()\n",
    "        q2 = q2.lower()\n",
    "        \n",
    "        \n",
    "        current_feature = []\n",
    "\n",
    "        q1_afv = vector_title(q1, w2v_model)\n",
    "        q2_afv = vector_title(q2, w2v_model)\n",
    "        \n",
    "        if q1_afv == \"Error\" or q2_afv == \"Error\":\n",
    "            \n",
    "            print('Error')\n",
    "            print(index)\n",
    "            current_feature = [0]*30\n",
    "            features.append(current_feature)\n",
    "            continue\n",
    "        \n",
    "        #Jaccard Similarity\n",
    "        qjac_sim = jaccard(q1_afv,q2_afv)     \n",
    "        \n",
    "        #Canberra Similarity\n",
    "        qcan_sim = canberra(q1_afv,q2_afv)       \n",
    "\n",
    "        #CityBlock Similarity\n",
    "        qcity_sim = cityblock(q1_afv,q2_afv)        \n",
    "  \n",
    "        #Eucledian Similarity\n",
    "        qeuc_sim = euclidean(q1_afv,q2_afv)\n",
    "\n",
    "        #Cosine Similarity\n",
    "        qcos_sim = cosine(q1_afv, q2_afv)    \n",
    "        \n",
    "        #Minowski Similarity\n",
    "        qmin_sim = minkowski(q1_afv, q2_afv)    \n",
    "        \n",
    "        #braycurtis Similarity\n",
    "        qbay_sim = braycurtis(q1_afv, q2_afv)    \n",
    "        \n",
    "        #WordMover's distance\n",
    "        mover_distance = w2v_model.wmdistance(q1, q2)\n",
    "        \n",
    "        \n",
    "        #Fuzzy Features\n",
    "        fuzz_ratio = fuzz.ratio(q1,q2)\n",
    "        fuzz_partialratio = fuzz.partial_ratio(q1,q2) \n",
    "        fuzz_tokenratio = fuzz.token_sort_ratio(q1,q2) \n",
    "        \n",
    "        #Skew and Kurtosis\n",
    "        q1_skew = skew(q1_afv)\n",
    "        q2_skew = skew(q2_afv)\n",
    "        q1_kurtosis = kurtosis(q1_afv)\n",
    "        q2_kurtosis = kurtosis(q2_afv)\n",
    "        \n",
    "        #Common number of bigrams\n",
    "        common_bigrams_count = common_bigrams(q1,q2)\n",
    "        \n",
    "        #Common NE Score\n",
    "        q_ne_score = common_ne_score(q1,q2)\n",
    "        \n",
    "        #Common Noun Score\n",
    "        q_noun_score = common_chunk_score(q1,q2)\n",
    "        \n",
    "        #TF-IDF\n",
    "        q_tfidf = tfidf(q1, q2)\n",
    "        \n",
    "        #Get the sentiment\n",
    "        q1_sentiment = analyzer.polarity_scores(q1)\n",
    "        q2_sentiment = analyzer.polarity_scores(q2)\n",
    "        #ratio = float(q1_sentiment['compound'])/q2_sentiment['compound']\n",
    "        \n",
    "        \n",
    "        #Features to check for (what,which,does,when,Can,Who,Will,Did) and (why, how)\n",
    "        if 'why' in q1.lower() or 'how' in q1.lower():\n",
    "            q1_question_type = 1\n",
    "        else:\n",
    "            q1_question_type = 0\n",
    "            \n",
    "            \n",
    "        if 'why' in q2.lower() or 'how' in q2.lower():\n",
    "            q2_question_type = 1\n",
    "        else:\n",
    "            q2_question_type = 0\n",
    "        \n",
    "           \n",
    "    \n",
    "        #==== Graph Features (Magic Features) =====#\n",
    "        \n",
    "        #Number of Common Neighbours\n",
    "        q_common_neighbours = common_neighbours(row['qid1'],row['qid2'])\n",
    "        \n",
    "        #Degree of each node\n",
    "        deg_1 = qid_graph.get(q1_index)\n",
    "        deg_2 = qid_graph.get(q2_index)  \n",
    "        \n",
    "        #Get the pagerank scores for each node\n",
    "        pagerank_1 = pagerank_dict.get(q1_index)\n",
    "        pagerank_2 = pagerank_dict.get(q2_index)\n",
    "\n",
    "        \n",
    "        \n",
    "        current_feature.extend([row['id']])\n",
    "        current_feature.extend([q1_index])\n",
    "        current_feature.extend([q2_index])\n",
    "        \n",
    "        \n",
    "        current_feature.extend([len(tokenize(q1))])\n",
    "        current_feature.extend([len(tokenize(q1))])\n",
    "        current_feature.extend([abs(len(tokenize(q1))-len(tokenize(q2)))])\n",
    "        \n",
    "        \n",
    "        current_feature.extend([1-qjac_sim]) \n",
    "        current_feature.extend([qcan_sim]) \n",
    "        current_feature.extend([qcity_sim]) \n",
    "        current_feature.extend([qeuc_sim])\n",
    "        current_feature.extend([qcos_sim])\n",
    "        current_feature.extend([qmin_sim])\n",
    "        current_feature.extend([qbay_sim])\n",
    "        current_feature.extend([mover_distance])\n",
    "        \n",
    "        current_feature.extend([fuzz_ratio])\n",
    "        current_feature.extend([fuzz_partialratio])\n",
    "        current_feature.extend([fuzz_tokenratio])\n",
    "        \n",
    "        current_feature.extend([q1_skew])\n",
    "        current_feature.extend([q2_skew])\n",
    "        current_feature.extend([q1_kurtosis])\n",
    "        current_feature.extend([q2_kurtosis])\n",
    "        \n",
    "        current_feature.extend([common_bigrams_count])\n",
    "        \n",
    "        current_feature.extend([q_ne_score])\n",
    "        current_feature.extend([q_noun_score])\n",
    "        current_feature.extend([q_tfidf])\n",
    "        current_feature.extend([q_common_neighbours])\n",
    "        current_feature.extend([q1_sentiment['compound']])\n",
    "        current_feature.extend([q2_sentiment['compound']])\n",
    "        \n",
    "        current_feature.extend([q1_question_type])\n",
    "        current_feature.extend([q2_question_type])\n",
    "\n",
    "        current_feature.extend([pagerank_1])\n",
    "        current_feature.extend([pagerank_2])       \n",
    "        \n",
    "        if math.isnan(qjac_sim) or math.isnan(qcan_sim) or math.isnan(qcity_sim) or math.isnan(qeuc_sim) or math.isnan(qcos_sim) or math.isnan(q_ne_score) or math.isnan(q_noun_score) or math.isnan(q_tfidf):        \n",
    "            print(\"Found Nan\")\n",
    "            current_feature = [0]*30\n",
    "            features.append(current_feature)\n",
    "        \n",
    "        else:\n",
    "            #Prepare Features\n",
    "            features.append(current_feature)\n",
    "            labels.append(label)\n",
    "        \n",
    "        \n",
    "        if len(features)%20000 == 0:\n",
    "            print('Progress...',index)\n",
    "       \n",
    "    return np.asarray(features), labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Check for any nan indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "nanIndices = np.argwhere(np.isnan(features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for entry in nanIndices:\n",
    "    print(entry)\n",
    "    features[entry[0], entry[1]] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(x, open( \"baseline_pickles_new/finalFeatures.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "finalFeatures = features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(404290, 32)\n",
      "[[ 0.00000000e+00  1.00000000e+00  2.00000000e+00  7.00000000e+00\n",
      "   7.00000000e+00  1.00000000e+00  0.00000000e+00  7.53613668e+01\n",
      "   3.90879068e+00  2.82853518e-01  4.61144384e-02  2.82853518e-01\n",
      "   1.51061747e-01  2.93806295e-01  9.30000000e+01  1.00000000e+02\n",
      "   9.30000000e+01 -3.48890662e-02 -6.02397426e-02 -1.15610765e-01\n",
      "  -7.52179917e-02  1.10000000e+01  0.00000000e+00  8.00000000e-01\n",
      "   8.39926214e-01  0.00000000e+00  2.96000000e-01  2.96000000e-01\n",
      "   0.00000000e+00  0.00000000e+00  1.60851165e-07  3.04769172e-07]\n",
      " [ 1.00000000e+00  3.00000000e+00  4.00000000e+00  5.00000000e+00\n",
      "   5.00000000e+00  5.00000000e+00  0.00000000e+00  1.26539742e+02\n",
      "   7.72307764e+00  5.58875127e-01  1.80014401e-01  5.58875127e-01\n",
      "   3.15472520e-01  7.03441799e-01  6.60000000e+01  7.50000000e+01\n",
      "   6.30000000e+01 -1.01617794e-01  1.72510641e-02 -2.18512090e-01\n",
      "   4.13854600e-03  6.00000000e+00  0.00000000e+00  1.42857143e-01\n",
      "   3.31521398e-01  0.00000000e+00  3.40000000e-01  3.40000000e-01\n",
      "   0.00000000e+00  0.00000000e+00  8.19893828e-07  3.09732950e-07]\n",
      " [ 2.00000000e+00  5.00000000e+00  6.00000000e+00  6.00000000e+00\n",
      "   6.00000000e+00  1.00000000e+00  0.00000000e+00  1.32828897e+02\n",
      "   9.02722266e+00  6.53668204e-01  1.96136106e-01  6.53668204e-01\n",
      "   3.36755119e-01  6.07826025e-01  4.40000000e+01  4.70000000e+01\n",
      "   6.30000000e+01 -3.00739007e-02  3.39024286e-01 -1.77026835e-01\n",
      "  -1.43901277e-01  1.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   1.67392826e-01  0.00000000e+00  3.18200000e-01  2.73200000e-01\n",
      "   1.00000000e+00  1.00000000e+00  3.04926023e-07  1.60915751e-07]\n",
      " [ 3.00000000e+00  7.00000000e+00  8.00000000e+00  3.00000000e+00\n",
      "   3.00000000e+00  6.00000000e+00  0.00000000e+00  1.88376899e+02\n",
      "   1.37758132e+01  9.95100261e-01  6.00007125e-01  9.95100261e-01\n",
      "   6.78369937e-01  1.19694947e+00  1.00000000e+01  1.40000000e+01\n",
      "   2.40000000e+01 -7.33515884e-02  7.51269140e-02 -3.23568272e-01\n",
      "   9.32172931e-02  0.00000000e+00  2.00000000e-01  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00 -2.48400000e-01  0.00000000e+00\n",
      "   1.00000000e+00  0.00000000e+00  2.08814608e-07  2.08814608e-07]\n",
      " [ 4.00000000e+00  9.00000000e+00  1.00000000e+01  1.00000000e+01\n",
      "   1.00000000e+01  5.00000000e+00  0.00000000e+00  1.46317746e+02\n",
      "   1.04365021e+01  7.47352199e-01  2.43976772e-01  7.47352199e-01\n",
      "   4.09992057e-01  8.87436190e-01  3.70000000e+01  5.60000000e+01\n",
      "   4.70000000e+01 -2.79936009e-02  7.32817216e-02 -1.13034896e-01\n",
      "  -4.65949933e-01  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   1.23020166e-01  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  3.40677528e-07  1.27847491e-07]\n",
      " [ 5.00000000e+00  1.10000000e+01  1.20000000e+01  8.00000000e+00\n",
      "   8.00000000e+00  1.00000000e+00  0.00000000e+00  1.33948261e+02\n",
      "   7.43499621e+00  5.24894333e-01  1.73710984e-01  5.24894333e-01\n",
      "   3.14923704e-01  3.46734772e-01  7.30000000e+01  7.30000000e+01\n",
      "   7.40000000e+01 -1.09346482e-01 -3.83353048e-02 -2.34398174e-01\n",
      "  -3.36813330e-01  6.00000000e+00  0.00000000e+00  3.33333333e-01\n",
      "   3.25220138e-01  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  2.08814608e-07  2.08814608e-07]\n",
      " [ 6.00000000e+00  1.30000000e+01  1.40000000e+01  2.00000000e+00\n",
      "   2.00000000e+00  5.00000000e+00  0.00000000e+00  1.80753886e+02\n",
      "   1.54433777e+01  1.12698637e+00  5.79064558e-01  1.12698637e+00\n",
      "   6.44258958e-01  1.72537609e+00  2.00000000e+01  4.20000000e+01\n",
      "   2.30000000e+01  4.28718374e-02 -8.95937317e-03  4.34173487e-01\n",
      "  -3.71883837e-01  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  0.00000000e+00  4.01900000e-01\n",
      "   0.00000000e+00  0.00000000e+00  2.08814608e-07  2.08814608e-07]\n",
      " [ 7.00000000e+00  1.50000000e+01  1.60000000e+01  2.00000000e+00\n",
      "   2.00000000e+00  0.00000000e+00  0.00000000e+00  1.02851089e+02\n",
      "   5.72592916e+00  4.22258950e-01  7.63889841e-02  4.22258950e-01\n",
      "   2.04687096e-01  6.61923469e-01  6.20000000e+01  6.70000000e+01\n",
      "   6.10000000e+01  1.97980573e-01  2.21851931e-01 -4.97041008e-02\n",
      "   5.79759272e-02  2.00000000e+00  0.00000000e+00  2.50000000e-01\n",
      "   2.58615292e-01  0.00000000e+00  4.40400000e-01  6.24900000e-01\n",
      "   1.00000000e+00  0.00000000e+00  2.08814608e-07  2.08814608e-07]\n",
      " [ 8.00000000e+00  1.70000000e+01  1.80000000e+01  4.00000000e+00\n",
      "   4.00000000e+00  2.00000000e+00  0.00000000e+00  1.99333333e+01\n",
      "   1.58511053e+00  1.15282047e-01  0.00000000e+00  1.15282047e-01\n",
      "   6.66666667e-02  2.02635697e-01  8.50000000e+01  9.00000000e+01\n",
      "   8.70000000e+01  1.99295970e-01  1.99295970e-01  1.71027033e-01\n",
      "   1.71027033e-01  4.00000000e+00  0.00000000e+00  5.00000000e-01\n",
      "   0.00000000e+00  1.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  1.73025484e-07  2.49997970e-07]\n",
      " [ 9.00000000e+00  1.90000000e+01  2.00000000e+01  6.00000000e+00\n",
      "   6.00000000e+00  1.00000000e+00  0.00000000e+00  1.20851491e+02\n",
      "   9.13552682e+00  6.60544808e-01  1.75508744e-01  6.60544808e-01\n",
      "   3.06544630e-01  7.74556974e-01  5.00000000e+01  5.70000000e+01\n",
      "   4.40000000e+01 -3.61295216e-02  1.87833525e-01 -4.06127194e-01\n",
      "  -4.83520791e-01  1.00000000e+00  0.00000000e+00  2.00000000e-01\n",
      "   0.00000000e+00  0.00000000e+00  0.00000000e+00  5.10600000e-01\n",
      "   0.00000000e+00  1.00000000e+00  2.08814608e-07  2.08814608e-07]]\n"
     ]
    }
   ],
   "source": [
    "print(finalFeatures.shape)\n",
    "print(finalFeatures[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "features_normalized = min_max_scaler.fit_transform(finalFeatures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(features_normalized, open( \"baseline_pickles_new/featuresNormalized.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = df_train['is_duplicate'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Split into training, validation and testing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 304205\n",
    "valid_size = 50000\n",
    "test_size = 50000\n",
    "test_size_unseen = df_test.shape[0]\n",
    "\n",
    "X_train = finalFeatures[0:train_size]\n",
    "X_valid = finalFeatures[train_size:train_size+valid_size]\n",
    "X_test = finalFeatures[train_size+valid_size:train_size+valid_size+test_size]\n",
    "\n",
    "y_train = labels[0:train_size]\n",
    "y_valid = labels[train_size:train_size+valid_size]\n",
    "y_test = labels[train_size+valid_size:train_size+valid_size+test_size]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Run different models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RandomForestClassifier(max_depth=15, max_features=32, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### XGBoost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBClassifier()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Logistic Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(penalty = 'l1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Fit the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save the predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_train = model.predict_proba(X_train)\n",
    "predictions_xgboost_train = predictions_train[:,1]\n",
    "pickle.dump( predictions_xgboost_train, open( \"baseline_pickles_new/predictions_full_xgboost_train.p\", \"wb\" ) )\n",
    "\n",
    "predictions_valid = model.predict_proba(X_valid)\n",
    "predictions_xgboost_valid = predictions_valid[:,1]\n",
    "pickle.dump( predictions_xgboost_valid, open( \"baseline_pickles_new/predictions_full_xgboost_valid.p\", \"wb\" ) )\n",
    "\n",
    "predictions_test = model.predict_proba(X_test)\n",
    "predictions_xgboost_test = predictions_test[:,1]\n",
    "pickle.dump( predictions_xgboost_test, open( \"baseline_pickles_new/predictions_full_xgboost_test.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_test_bool = [prediction > 0.5 for prediction in predictions_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### LightGBM classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shared/anaconda3/lib/python3.6/site-packages/lightgbm/engine.py:99: UserWarning: Found `num_trees` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's binary_error: 0.14498\n",
      "[2]\tvalid_0's binary_error: 0.14308\n",
      "[3]\tvalid_0's binary_error: 0.14414\n",
      "[4]\tvalid_0's binary_error: 0.1452\n",
      "[5]\tvalid_0's binary_error: 0.14326\n",
      "[6]\tvalid_0's binary_error: 0.14282\n",
      "[7]\tvalid_0's binary_error: 0.14222\n",
      "[8]\tvalid_0's binary_error: 0.142\n",
      "[9]\tvalid_0's binary_error: 0.14164\n",
      "[10]\tvalid_0's binary_error: 0.1418\n",
      "[11]\tvalid_0's binary_error: 0.14162\n",
      "[12]\tvalid_0's binary_error: 0.14074\n",
      "[13]\tvalid_0's binary_error: 0.14084\n",
      "[14]\tvalid_0's binary_error: 0.13968\n",
      "[15]\tvalid_0's binary_error: 0.1395\n",
      "[16]\tvalid_0's binary_error: 0.13936\n",
      "[17]\tvalid_0's binary_error: 0.13896\n",
      "[18]\tvalid_0's binary_error: 0.1385\n",
      "[19]\tvalid_0's binary_error: 0.13792\n",
      "[20]\tvalid_0's binary_error: 0.1376\n",
      "[21]\tvalid_0's binary_error: 0.13752\n",
      "[22]\tvalid_0's binary_error: 0.13698\n",
      "[23]\tvalid_0's binary_error: 0.1367\n",
      "[24]\tvalid_0's binary_error: 0.1362\n",
      "[25]\tvalid_0's binary_error: 0.1359\n",
      "[26]\tvalid_0's binary_error: 0.13604\n",
      "[27]\tvalid_0's binary_error: 0.13528\n",
      "[28]\tvalid_0's binary_error: 0.1351\n",
      "[29]\tvalid_0's binary_error: 0.1349\n",
      "[30]\tvalid_0's binary_error: 0.1348\n",
      "[31]\tvalid_0's binary_error: 0.13476\n",
      "[32]\tvalid_0's binary_error: 0.13432\n",
      "[33]\tvalid_0's binary_error: 0.13414\n",
      "[34]\tvalid_0's binary_error: 0.134\n",
      "[35]\tvalid_0's binary_error: 0.1335\n",
      "[36]\tvalid_0's binary_error: 0.13352\n",
      "[37]\tvalid_0's binary_error: 0.13338\n",
      "[38]\tvalid_0's binary_error: 0.13348\n",
      "[39]\tvalid_0's binary_error: 0.13312\n",
      "[40]\tvalid_0's binary_error: 0.13292\n",
      "[41]\tvalid_0's binary_error: 0.13276\n",
      "[42]\tvalid_0's binary_error: 0.13228\n",
      "[43]\tvalid_0's binary_error: 0.13192\n",
      "[44]\tvalid_0's binary_error: 0.13184\n",
      "[45]\tvalid_0's binary_error: 0.1318\n",
      "[46]\tvalid_0's binary_error: 0.13146\n",
      "[47]\tvalid_0's binary_error: 0.13136\n",
      "[48]\tvalid_0's binary_error: 0.13124\n",
      "[49]\tvalid_0's binary_error: 0.13094\n",
      "[50]\tvalid_0's binary_error: 0.13088\n",
      "[51]\tvalid_0's binary_error: 0.13052\n",
      "[52]\tvalid_0's binary_error: 0.13048\n",
      "[53]\tvalid_0's binary_error: 0.13024\n",
      "[54]\tvalid_0's binary_error: 0.12986\n",
      "[55]\tvalid_0's binary_error: 0.12966\n",
      "[56]\tvalid_0's binary_error: 0.12952\n",
      "[57]\tvalid_0's binary_error: 0.1292\n",
      "[58]\tvalid_0's binary_error: 0.12916\n",
      "[59]\tvalid_0's binary_error: 0.12888\n",
      "[60]\tvalid_0's binary_error: 0.12882\n",
      "[61]\tvalid_0's binary_error: 0.12878\n",
      "[62]\tvalid_0's binary_error: 0.12856\n",
      "[63]\tvalid_0's binary_error: 0.12832\n",
      "[64]\tvalid_0's binary_error: 0.12826\n",
      "[65]\tvalid_0's binary_error: 0.12812\n",
      "[66]\tvalid_0's binary_error: 0.128\n",
      "[67]\tvalid_0's binary_error: 0.12788\n",
      "[68]\tvalid_0's binary_error: 0.12792\n",
      "[69]\tvalid_0's binary_error: 0.12782\n",
      "[70]\tvalid_0's binary_error: 0.12776\n",
      "[71]\tvalid_0's binary_error: 0.12756\n",
      "[72]\tvalid_0's binary_error: 0.12744\n",
      "[73]\tvalid_0's binary_error: 0.12738\n",
      "[74]\tvalid_0's binary_error: 0.12736\n",
      "[75]\tvalid_0's binary_error: 0.12732\n",
      "[76]\tvalid_0's binary_error: 0.1274\n",
      "[77]\tvalid_0's binary_error: 0.12714\n",
      "[78]\tvalid_0's binary_error: 0.12692\n",
      "[79]\tvalid_0's binary_error: 0.12678\n",
      "[80]\tvalid_0's binary_error: 0.12662\n",
      "[81]\tvalid_0's binary_error: 0.12674\n",
      "[82]\tvalid_0's binary_error: 0.12658\n",
      "[83]\tvalid_0's binary_error: 0.1261\n",
      "[84]\tvalid_0's binary_error: 0.1261\n",
      "[85]\tvalid_0's binary_error: 0.12608\n",
      "[86]\tvalid_0's binary_error: 0.1259\n",
      "[87]\tvalid_0's binary_error: 0.12592\n",
      "[88]\tvalid_0's binary_error: 0.1258\n",
      "[89]\tvalid_0's binary_error: 0.12574\n",
      "[90]\tvalid_0's binary_error: 0.12562\n",
      "[91]\tvalid_0's binary_error: 0.12546\n",
      "[92]\tvalid_0's binary_error: 0.12542\n",
      "[93]\tvalid_0's binary_error: 0.12514\n",
      "[94]\tvalid_0's binary_error: 0.12514\n",
      "[95]\tvalid_0's binary_error: 0.125\n",
      "[96]\tvalid_0's binary_error: 0.12502\n",
      "[97]\tvalid_0's binary_error: 0.12496\n",
      "[98]\tvalid_0's binary_error: 0.12504\n",
      "[99]\tvalid_0's binary_error: 0.12486\n",
      "[100]\tvalid_0's binary_error: 0.12466\n"
     ]
    }
   ],
   "source": [
    "param = {'num_leaves':31, 'num_trees':100, 'objective':'binary'}\n",
    "param['metric'] = [\"binary_error\"]\n",
    "lgbDataTrain = lgb.Dataset(X_train, label = y_train)\n",
    "lgbDataValid = lgb.Dataset(X_valid, label = y_valid)\n",
    "lgbDataTest = lgb.Dataset(X_test, label = y_test)\n",
    "num_round = 10\n",
    "lgbmodel = lgb.train(param, lgbDataTrain, num_round, valid_sets=[lgbDataValid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "predTrain = lgbmodel.predict(X_train)\n",
    "predValid = lgbmodel.predict(X_valid)\n",
    "predTest = lgbmodel.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump( predTrain, open( \"baseline_pickles_new/predictions_full_LGBM_train.p\", \"wb\" ) )\n",
    "pickle.dump( predValid, open( \"baseline_pickles_new/predictions_full_LGBM_valid.p\", \"wb\" ) )\n",
    "pickle.dump( predTest, open( \"baseline_pickles_new/predictions_full_LGBM_test.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_test_bool = [prediction > 0.5 for prediction in predTest]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Plot the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[30363  2030]\n",
      " [ 4080 13527]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAEmCAYAAABRfjp6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XecFEX6x/HPd3dJkhFBkoqKmAVBTKdiQDGCOWfFeJ7neXfGn5hO1DPHMyJmzogRwVMRTGQQEwgoSYIEAREFnt8fVbsOy+7M7Lq7M7P7vHn1a2eqq7urZ4ZnaqqrqmVmOOecyx55mS6Ac865tXlgds65LOOB2TnnsowHZuecyzIemJ1zLst4YHbOuSzjgbmKSaon6TVJSyT99w/s50RJ71Rk2TJF0h6Svs6W40naRJJJKqiqMuUKSdMl7RcfXyHpkUo4xoOSrq7o/eYSeT/mkkk6AbgE2BJYCowDbjSz4X9wvycDfwZ2M7NVf7igWU6SAR3MbEqmy1IaSdOBs8xsaHy+CTANqFXR75Gk/sBMM7uqIvdbVYq/VhWwv9Pi/v5UEfurLrzGXAJJlwB3Av8CWgIbAfcDvSpg9xsD39SEoJwOr5VWHn9tc5iZ+ZKwAI2BZcDRSfLUIQTu2XG5E6gT13UHZgJ/A+YBc4DT47prgV+B3+IxzgT6Ak8l7HsTwICC+Pw0YCqh1j4NODEhfXjCdrsBI4El8e9uCeveB64HRsT9vAM0L+XcCsv/j4Ty9wYOAr4BFgJXJOTvBnwMLI557wVqx3XD4rksj+d7bML+/wn8ADxZmBa32SweY8f4vDWwAOiexnv3BPC3+LhNPPb58fnmcb8qdrwngTXAiljGfyS8B6cC38fjX5nm+7/W+xLTLB6/T3zvf43Heq2U8zDgXGAysAi4j99/3eYBVwHfxfdnANC42GfnzFjuYQlppwMz4v7OBXYCJsT37d6EY28G/A/4MZ7300CThPXTgf3i477Ez25835clLKuAvnHdZcC3hM/eF8DhMX0r4BdgddxmcUzvD9yQcMyzgSnx/RsEtE7ntcrlJeMFyLYF6Bk/VAVJ8lwHfAK0ADYAPgKuj+u6x+2vA2oRAtrPQNO4vujDXMrzwv9IBUB94CegY1zXCtgmPj6NGACAZvFDeXLc7vj4fP24/v34H2MLoF583q+Ucyss///F8p8NzAeeARoC28T/TJvG/F2AXeJxNwG+BC5O2J8Bm5ew/5sJAa4eCYEy5jk77mc9YDDw7zTfuzOIwQ44IZ7z8wnrXk0oQ+LxphODTbH34OFYvh2AlcBWabz/Re9LSa8BxYJOKedhwOtAE8KvtflAz4TzmAJsCjQAXgKeLFbuAYTPTr2EtAeBusD+8f17JZa/DSHA7xX3sTnQI743GxCC+50lvVYU++wm5OkUy9w5Pj+a8AWbR/hyXg60SvJ6Fb1GwD6EL4gdY5nuAYal81rl8uJNGetaH1hgyZsaTgSuM7N5ZjafUBM+OWH9b3H9b2b2JqE20LGc5VkDbCupnpnNMbNJJeQ5GJhsZk+a2Sozexb4Cjg0Ic/jZvaNma0ABhL+85TmN0J7+m/Ac0Bz4C4zWxqPPwnYHsDMRpvZJ/G404H/AHulcU7XmNnKWJ61mNnDhBrQp4QvoytT7K/QB8AekvKAPYFbgN3jur3i+rK41sxWmNl4YDwhQEPq978i9DOzxWb2PfAev79fJwK3m9lUM1sGXA4cV6zZoq+ZLS/22l5vZr+Y2TuEwPhsLP8s4EOgM4CZTTGzIfG9mQ/cTur3s4ikDQhB/89mNjbu879mNtvM1pjZ84T3tluauzwReMzMxpjZyni+u8brAIVKe61ylgfmdf0INE/RPtea8FOy0HcxrWgfxQL7z4TaTZmY2XJCDeNcYI6kNyRtmUZ5CsvUJuH5D2Uoz49mtjo+LvzPPTdh/YrC7SVtIel1ST9I+onQLt88yb4B5pvZLynyPAxsC9wT/0OmZGbfEr4EOwF7EGpSsyV1pHyBubTXLNX7XxHKcuwCwrWQQjNK2F/x96+097OFpOckzYrv51Okfj+J29YCXgCeMbPnEtJPkTRO0mJJiwnva1r7pNj5xi+jHyn/ZzsneGBe18eEn3q9k+SZTbiIV2ijmFYeywk/2QttmLjSzAabWQ9CzfErQsBKVZ7CMs0qZ5nK4gFCuTqYWSPgCkI7bjJJuwJJakBot30U6CupWRnK8wFwFKGde1Z8fgrQlNCzpszlKUGy93+t91PSWu9nOY6VzrFXsXag/SPHuCluv318P08i9ftZ6B5CO3JRjxNJGxM+sxcSmtaaAJ8n7DNVWdc6X0n1Cb9qq+KznTEemIsxsyWE9tX7JPWWtJ6kWpIOlHRLzPYscJWkDSQ1j/mfKuchxwF7StpIUmPCTzUAJLWUdFj8MK4k1AZXl7CPN4EtJJ0gqUDSscDWhBpjZWtIaAdfFmvz5xVbP5fQHloWdwGjzews4A1C+ygAkvpKej/Jth8QgsCw+Px9QvfE4Qm/AooraxmTvf/jgW0kdZJUl9AO+0eOVdKx/yqpffwC+xehHb2ievk0JF6Ik9QG+Hs6G0k6h/Cr5AQzW5Owqj4h+M6P+U4n1JgLzQXaSqpdyq6fAU6Pr2cdwvl+GpvNqi0PzCUws9sJfZivInygZhD+s78Ss9wAjCJc1Z4IjIlp5TnWEOD5uK/RrB1M8wi9O2YTrkjvBZxfwj5+BA6JeX8k9Cw4xMwWlKdMZXQp4ULbUkLN6Pli6/sCT8Sfscek2pmkXoQLsOfGpEuAHSWdGJ+3I/QuKc0HhOBSGJiHE2qww0rdItQSr4plvDRVGUny/pvZN4SLg0MJbanF+70/Cmwdj/UKZfcYoSfJMEIvnV8IXzwV5VrChbYlhC/Fl9Lc7njCF85sScvicoWZfQHcRvglOhfYjrXfv/8Rrln8IGmdz6uZvQtcDbxI6PWzGXBceU4sl/gAE5dTJI0D9o1fRs5VSx6YnXMuy3hThnPOZRkPzM45l2U8MDvnXJbxSU4ySAX1TLUbZroYNVLnrTbKdBFqpO++m86CBQvS7RedVH6jjc1WrTNwdB22Yv5gM+tZ2vrYrXEYYch3AfCCmV0jqT1h5GszQs+bk83s19htbwBhOoIfgWMLu+9JupwwV8lq4CIzGxzTexK6geYDj5hZv2Rl9sCcQardkDodU/Ygc5VgxKf3ZroINdLuO3etsH3ZqhVp/f/5Zdx9qUYZrgT2MbNlcfTicElvEbpq3mFmz0l6kBBwH4h/F5nZ5pKOI8z7cqykrQld+bYhjFgcKmmLeIz7CHOQzARGShoUuxKWyJsynHO5SYK8/NRLChYsi09rxcUIEyi9ENOf4PfRwL3ic+L6fSUppj8X5xmZRphsqltcpsT5TX4l1MKTTiHsgdk5l7uUl3oJc9+MSlj6rLMbKT/2kZ8HDCHMTLg4YUTlTH6fn6MNcT6SuH4JYZh4UXqxbUpLL5U3ZTjncpfSaq5eYGZJ21DicP1OkpoALxPmil4nW+FRS1lXWnpJFeCkA0g8MDvncpTSaqooCzNbHOdi2QVoIqkg1orb8vtEVTMJUwPMjLNQNiZMmVCYXihxm9LSS+RNGc653CTSbcpIvpswGVWT+LgesB/hRg3vEWYqhHA3m1fj40HxOXH9/ywMoR5EmBu7TuzR0QH4jHBHoQ5x4qnahAuEg5KVyWvMzrkcpXSbMlJpRZhoK59QWR1oZq9L+gJ4TtINwFjCBFTEv09KKrzd1XEAZjZJ0kDC7bNWARcUzmgo6ULC3XjyCRP/l3TDiyIemJ1zuasCmjLMbALxDi7F0qdSwp1W4k0eji5lXzcCN5aQ/iZhet60eGB2zuUopdVUkYs8MDvncpOo8It/2cIDs3MuR3mN2Tnnsk9ehVz8yzoemJ1zucmbMpxzLtt4U4ZzzmWfiunHnHU8MDvncpMqfkh2tvDA7JzLXd6U4ZxzWcabMpxzLpt4U4ZzzmWXwtnlqiEPzM65HOXd5ZxzLvt4U4ZzzmUZv/jnnHNZRN6U4ZxzWUd5Hpidcy5rCJA3ZTjnXBaRkE/76Zxz2cVrzM45l2U8MDvnXDYR3pThnHPZRMhrzM45l22qa2Cunp0AnXM1Ql5eXsolFUntJL0n6UtJkyT9Jab3lTRL0ri4HJSwzeWSpkj6WtIBCek9Y9oUSZclpLeX9KmkyZKel1Q76XmV69VwzrlMU5pLaquAv5nZVsAuwAWSto7r7jCzTnF5EyCuOw7YBugJ3C8pX1I+cB9wILA1cHzCfm6O++oALALOTFYgD8zOuZwlKeWSipnNMbMx8fFS4EugTZJNegHPmdlKM5sGTAG6xWWKmU01s1+B54BeCoXYB3ghbv8E0DtZmTwwO+dyklC6TRnNJY1KWPqUuk9pE6Az8GlMulDSBEmPSWoa09oAMxI2mxnTSktfH1hsZquKpZfKA7NzLnel15SxwMy6JiwPlbgrqQHwInCxmf0EPABsBnQC5gC3JRy1OCtHeqm8V4ZzLjep4nplSKpFCMpPm9lLAGY2N2H9w8Dr8elMoF3C5m2B2fFxSekLgCaSCmKtOTF/ibzG7JzLWRXUK0PAo8CXZnZ7QnqrhGyHA5/Hx4OA4yTVkdQe6AB8BowEOsQeGLUJFwgHmZkB7wFHxe1PBV5NViavMddgdWoXMPTRi6ldu4CC/HxeHjqWGx58k41br8+T/U6naeP1GPflDM64agC/rVrNWUf9iXOO2ZPVa9aw/OeVXHDDs3w19QcAtu3QmnuvOp6G9euyZo3xp5NuYeWvq3j13vPZcINGFOTnM2Lst1x80/OsWZP0V1yNM2PGDM46/RTmzv2BvLw8zjizDxde9BcWLlzIySccy3ffTWfjjTfhqWcH0rRpU14b9CrXXXM1eXl5FBQUcMttd7L7n/4EwFMDnqDfTTcAcNnlV3HSKadm8tQqVQUOMNkdOBmYKGlcTLuC0KuiE6HZYTpwDoCZTZI0EPiC0KPjAjNbDSDpQmAwkA88ZmaT4v7+CTwn6QZgLOGLoPRzC8HcZULeei2sTsdjMlqG+vVqs3zFrxQU5PG/xy7h0ltf4KKT9uHV/43nv4NHc/eVxzHxm5k8/N/hNKxfl6XLfwHg4L22o8/Re9DrwvvJz8/j42f+yZlXD2DiN7No1rg+i5f+zJo1ttY2z/77LF4aMpb/Dh6dyVMGYNHIezNdhCJz5szhhzlz6LzjjixdupTddu7CwBde4ckB/WnarBl//8dl3HpLPxYvWsSNN93MsmXLqF+/PpKYOGECJ51wDOM//4qFCxey+y5dGfHJKCSx285d+OjT0TRt2jR1IarI7jt3ZfToURUSTWtvsLk1P/KWlPnm/OfI0WbWtSKOWVW8KaOGW77iVwBqFeRTUJCPmbHXTlvw0tCxADz92qcc2n0HgKIACyGgW7x+sd+uW/L55FlM/GYWAAuXLC+qFRduU1CQR624f7e2Vq1a0XnHHQFo2LAhW265FbNnz+L1117lpJNDjfekk0/ltUGvANCgQYOimuLy5cuLHg95ZzD77tuDZs2a0bRpU/bdtwfvDH47A2dURVQxTRnZyJsyari8PPHRM/9ks3Yb8J/nhzF15gKWLF3B6tVrAJg1dxGtWzQuyn/OMXty0Ul7U7tWAT3PuRuADhu1wAwG3XcBzZs24IXBo7n9iaFF2wy67wK6brsx74z4oijgu5J9N30648aNZaduOzNv7lxatQrNnK1atWL+vHlF+V595WX+76rLmT9vHi+9+gYAs2fPom273689tWnbltmzZ1XtCVQxH5JdRpJM0m0Jzy+V1DfFNr0TRsqk2v+yP1C2vpIujY+vk7RfOfaxiaQTyluGbLFmjbHLcf3Y/ICr6LrtxmzZfsN18iRWcv8zcBjbHHYtV931Kped1ROAgvx8duu8Kadf2Z99z7idw/bZge7dtija5rAL7qN9jyuoU7uA7jt1rPRzylXLli3j+GOO5Nbb7qRRo0ZJ8/bqfTjjP/+KgS++wnV9rwYo8ddIdQ1chZSnlEsuqsx6/krgCEnNy7BNb8JQxipjZv9nZkNT51zHJkDOB+ZCS5atYNioyXTbbhMaN6xHfn74aLRp2ZQ585esk3/g4NEc2n17AGbNW8yHo6fw4+LlrPjlN94ePonOW7ZbK//KX1fx+gcTObT7dpV/Mjnot99+4/hjjuTY40+k9+FHANCiZUvmzJkDhHboDVq0WGe7P+2xJ1OnfsuCBQto06YtM2f8Pr5h1syZtGrVumpOIAPSGfWXq19MlRmYVwEPAX8tvkLSxpLejSNq3pW0kaTdgMOAW+OEIZsV26a9pI8ljZR0fUJ6d0mvJzy/V9Jp8fF0STdL+iwum5dQlv6SjoqPd5L0kaTxMX/DWDP+UNKYuOwWN+0H7BHL+tc4Vv7WWL4Jks75w69gJWvetAGNG9QDoG6dWuyzc0e+mjaXYaO+4Yj9OgNw4qE78/r7EwDYbKMNirY9cI9tmDJjPgBDPvqCbTu0oV7dWuTn57FHl835cuoP1K9Xmw2bh5pffn4ePXffmq+nz8Wtzcw49+wz6bjlVvzlr5cUpR98yGE89eQTADz15BMccmgvAL6dMqWodjx2zBh+/fVX1l9/fXrsfwBDh77DokWLWLRoEUOHvkOP/Q9Y94DVSHUNzJXdxnwfMEFS8Uun9wIDzOwJSWcAd5tZb0mDgNfN7IV19gR3AQ+Y2QBJF5ShDD+ZWTdJpwB3AoeUlCn2O3weONbMRkpqBKwA5gE9zOwXSR2AZ4GuwGXApWZ2SNy+D7DEzHaSVAcYIemdOJY+K23YvBEPX3cy+Xl55OWJF4eM4a0PP+fLqXN4st/pXHP+IYz/egb9X/kYgPOO3ZO9d96S31atZvFPP3P21QMAWLx0BXc/9T+GP/UPzIzBwyfx9vBJtGjWkBfuPIfatQrIz8/jg5Hf8PALwzN5ylnpoxEjeObpJ9l22+3YuUsnAK694V9c+o/LOOn4Y3ji8Udp124jnn7uvwC8/PKLPPPUAGoV1KJuvXo8+fTzSKJZs2ZcfsXV/GnXnQC44sr/o1mzZhk7r6qQq00VqVRadzlJy8ysgaTrgN8IQa6BmfWVtABoZWa/xRE3c8ysuaT+lBKYJf0IbBi3aQTMjvvvztoB8l5glJn1lzQd2MfMpsbj/GBm68e27mVm9u/CYwJfAw+a2e7FjtuY8EXSCVgNbGFm65Vw3BeA7YGf46aNgXPM7J1i++sDhLH6tRp0qbtN9e1nms2yqbtcTVKR3eXqtOxgbU68K2W+aXccnHPd5aqiV8adwBjg8SR50v12KCnfKtZukqmbZJtkx1Ep6/8KzAV2iMf5pYQ8hdv/2cwGJzkGcZz+QxD6MSfL65xLogKHZGebSu/kZ2YLgYGsPf/oR4ThigAnAoW/b5cCDUvZ1Yhi2xT6Dtg6Do9sDOxbbLtjE/5+nKSoXwGtJe0EENuXCwg13zlmtoYwOii/lLIOBs6LNXMkbSGpfpLjOef+gDC7XOolF1VV7+vbgMTeGRcBp0uaQAh2f4npzwF/lzS2+MW/mOcCSSMJwRIAM5tBCPwTgKcJwx0T1ZH0adx+nQuRCfv5lRC875E0HhhCqH3fD5wq6RNgC2B53GQCsCpeKPwr8AhhiOYYSZ8D/8H7iTtXqaTUSy6q1kOyYxtzVzNbkOmylCQbhmTXVN7GnBkV2cZcd8MtbONT70mZ75tbenobs3POVQUJ8vNztEqcQrUOzGa2SabL4JyrPLnaVJFKtQ7Mzrnqrbr2yvDA7JzLSRI52+siFQ/MzrkclbtDrlPxwOycy1leY3bOuWySw/2UU/HA7JzLScIv/jnnXNbxpgznnMsy1bTC7IHZOZejqvHsch6YnXM5qXB2uerIA7NzLmdV0wpzlU376ZxzFa4i7vknqZ2k9yR9KWmSpL/E9GaShkiaHP82jemSdLekKfH+njsm7OvUmH+ypFMT0rtImhi3uVspCuaB2TmXkwqHZFfARPmrgL+Z2VbALoR537cm3NfzXTPrALwbnwMcCHSISx/ggVAeNQOuAXYGugHXFAbzmKdPwnY9kxXIA7NzLmdVRI3ZzOaY2Zj4eCnwJdAG6AU8EbM9AfSOj3sRbiZtZvYJ0ERSK+AAYIiZLTSzRYSbbfSM6xqZ2ccWJsAfkLCvEnkbs3MuZ1V0G7OkTYDOwKdASzObAyF4S2oRs7UBZiRsNjOmJUufWUJ6qTwwO+dyU/qzyzWXNCrh+UPxpshr705qALwIXGxmPyWpbZe0wsqRXioPzM65nKT0Z5dbkOrWUvEmyi8CT5vZSzF5rqRWsbbcCpgX02cC7RI2bwvMjundi6W/H9PblpC/VKW2MUtqlGxJtlPnnKsKFXEz1thD4lHgSzO7PWHVIKCwZ8WpwKsJ6afE3hm7AEtik8dgYH9JTeNFv/2BwXHdUkm7xGOdkrCvEiWrMU9i3Wp44XMDNkp5xs45V4nyK2aAye7AycBESeNi2hVAP2CgpDOB74Gj47o3gYOAKcDPwOkAZrZQ0vXAyJjvOjNbGB+fB/QH6gFvxaVUpQZmM2tX2jrnnMs0VdCQbDMbTsntwAD7lpDfgAtK2ddjwGMlpI8Ctk23TGl1l5N0nKQr4uO2krqkewDnnKss+XlKueSilIFZ0r3A3oSqPoSq+4OVWSjnnEtHRbQxZ6N0emXsZmY7ShoLRe0otSu5XM45l5QIPTOqo3QC82+S8oj97iStD6yp1FI551wqyt2milTSaWO+j9C/bwNJ1wLDgZsrtVTOOZeGGtuUYWYDJI0G9otJR5vZ55VbLOecS05AXq5G3hTSHfmXD/xGaM7wiY+cc1mhuk6Un06vjCuBZ4HWhKGEz0i6vLIL5pxzyaTTjJGrFep0aswnAV3M7GcASTcCo4GbKrNgzjmXSk1uyviuWL4CYGrlFMc559JX4wKzpDsIbco/A5MkDY7P9yf0zHDOuYwJF/8yXYrKkazGXNjzYhLwRkL6J5VXHOecS1OadyjJRckmMXq0KgvinHNlVV17ZaRsY5a0GXAjsDVQtzDdzLaoxHI551xS1bkpI50+yf2Bxwmvw4HAQOC5SiyTc86lpSJuxpqN0gnM65nZYAAz+9bMriLMNueccxkjQb6UcslF6XSXWxlvh/KtpHOBWUCLFNs451yly9G4m1I6gfmvQAPgIkJbc2PgjMoslHPOpaPGXvwzs0/jw6X8Plm+c85llFCNHGDyMnEO5pKY2RGVUqIaZLuO7XjngzsyXYwa6eFPpmW6CDXS/OUrK25nOTwXRirJasz3VlkpnHOuHHL14l4qyQaYvFuVBXHOubIQFXOX7GyU7nzMzjmXdarptT8PzM653CRRbe/5l3ZgllTHzCqw5d455/6YahqX07qDSTdJE4HJ8fkOku6p9JI551wKFXEHE0mPSZon6fOEtL6SZkkaF5eDEtZdLmmKpK8lHZCQ3jOmTZF0WUJ6e0mfSpos6XlJtVOVKZ0h2XcDhwA/ApjZeHxItnMuwwQUSCmXNPQHepaQfoeZdYrLmwCStgaOA7aJ29wvKV9SPnAfYT6hrYHjY16Am+O+OgCLgDNTFSidwJxnZt8VS1udxnbOOVepKqLGbGbDgIVpHrIX8JyZrTSzacAUoFtcppjZVDP7lTDRW684ncU+wAtx+yeA3qkOkk5gniGpG2Dxm+Fi4Js0T8I55yqFFEb+pVqA5pJGJSx90jzEhZImxKaOpjGtDTAjIc/MmFZa+vrAYjNbVSw9qXQu/p1HaM7YCJgLDI1pzjmXUfnpVC1hgZl1LeOuHwCuJ4x+vh64jTBHUEl1cKPkSq4lyZ9UOnNlzCO0qTjnXNYIE+VXTrcMM5tbdBzpYeD1+HQm0C4ha1tgdnxcUvoCoImkglhrTsxfqnTuYPIwJUR4M0v354BzzlU8pV1jLvuupVZmNic+PZzf74E6CHhG0u1Aa6AD8FkoDR0ktSdMjXwccIKZmaT3gKMI7c6nAq+mOn46TRlDEx7XjYWcUUpe55yrMiqxpaCM+5CeBboT2qJnAtcA3SV1IlRKpwPnAJjZJEkDgS+AVcAFZrY67udCYDCQDzxmZpPiIf4JPCfpBmAskPJ+quk0ZTxf7CSeBIak2s455ypTRd3zz8yOLyG51OBpZjcS5qYvnv4m8GYJ6VMJvTbSVp4h2e2BjcuxnXPOVagaOyRb0iJ+b2POI/T3u6z0LZxzrvJV57tkJw3MsXP0DoTGbIA1Zpayq4dzzlW6GjpRPvGK4stm1qWqCuScc+kQUFBNq8zpdDb5TNKOlV4S55wro4oYkp2Nkt3zr7BD9J+AsyV9CywnfFGZmXmwds5lkMirgO5y2ShZU8ZnwI6kMeGGc85VNVXiAJNMSxaYBWBm31ZRWZxzrkwqa0h2piULzBtIuqS0lWZ2eyWUxznn0hJuxprpUlSOZIE5H2hAybMjOedcxtXEASZzzOy6KiuJc86VgUivW1kuStnG7JxzWUlhsvzqKFlg3rfKSuGcc2UkIL+mBWYzS/ceWM45lxHVMyyXb3Y555zLAiKvBl78c865rFVTL/4551xWq4kX/5xzLnupZo78c865rOVNGc45l4W8KcM557JMNe2U4YHZOZebQlNG9YzMHpidczmrmrZkeGB2zuUqVdteGdX1oqZzrporbMpItaTcj/SYpHmSPk9IayZpiKTJ8W/TmC5Jd0uaImlC4v1QJZ0a80+WdGpCehdJE+M2dyuNK5YemJ1zuSmNG7GmWaHuD/QslnYZ8K6ZdQDejc8BDgQ6xKUP8ACEQA5cA+wMdAOuKQzmMU+fhO2KH2sd3pTh1rJ69WoO2GsXNmzdhqcGvsJ306dx7hknsXjRIrbboRP3PtSf2rVrM3PG91x03pn8tHgxq9es5sq+N7Lf/gcCcPdtN/PMk/3Jz8/jhpvvYO/99s/wWWWfp2/6B5M+eo+GTdfn8gFvA/DGI7cz8cMhKC+PBk3X56QrbqVx85ZMHvsJD1/eh/VbtQNg+z0P4MDTL2LR3Nk8eeOlLF04HymP3Q47ju5Hnw7A49f8mXnfTwVgxbKfqNegEf98/I3MnGwlqoimDDMbJmmTYsm9gO7x8RPA+8A/Y/oAMzPgE0lNJLWKeYcUTv5/sfs2AAAYeUlEQVQmaQjQU9L7QCMz+zimDyDcR/WtZGXywOzW8vAD99Ch45YsXboUgBuuuYJzzr+I3kcdyz8uvoBnBjzOaWedw5233sRhvY/itLPO4euvvuDEo3sxauJkvv7qC155aSAffDqOH+bM5pheB/LRmEnk5+dn+Myyy84HHsWeR5zCUzdeWpS2z/Fnc/BZ4W5uH7zQn7f7382xl94IwGbb78Q5tzy61j7y8gs4/IIraNdxW375eRm3nnkYHbv+iVbtO3D6tfcU5Xv53hupW79hFZxV1RJpd5drLmlUwvOHzOyhFNu0NLM5AGY2R1KLmN4GmJGQb2ZMS5Y+s4T0pLwpwxWZPWsmQwe/xYmnnAGAmTFi2Psc0vtIAI454WTefmMQEDr2L136EwBLf/qJDTdsBcDgN16j9xHHUKdOHTbepD3tN92MsaNHZuBsstvmnbqxXqMma6XVSwieK1f8TKpJLRs3b0G7jtsCUHe9BrTcZHOWLPhhrTxmxtj33qTLfodWTMGzjNL4Bywws64JS6qgnPyQ67JypCflNWZX5OrL/sbV193EsmWhtrxw4Y80atyEgoLwMWnVug1z5swC4NLLr+bYww/isYfu5+flyxn4avg5PmfObLrs1K1on61at2HO7FlVfCa56/WH/s1ng1+mXv2GXHjX00Xp0yaNpd9pB9G4eUt6X3A5rdpvsdZ2P86ZyaxvJrHx1p3WSv92/EgaNl2fFu3aV0n5q1ol9sqYK6lVrC23AubF9JlAu4R8bYHZMb17sfT3Y3rbEvInlRM1ZkmrJY2TNEnSeEmXSCp32SX1l3RUfPyIpK3LsY9Okg4qbxmyzTtvv0HzDVqwQ+eii8yEZrS1FV5QfvmF5zn2hFMY++U0nn5hEBeecxpr1qxJuo1L7ZA+l3LdiyPo0uMwPnxpAABtt9iGa//7IZf1f5M9jzyFR644Z61tVv68nEevOp8jLrp6rVo3wOihg+iy32FVVv6qVNiUkWopp0FAYc+KU4FXE9JPib0zdgGWxCaPwcD+kprGi377A4PjuqWSdom9MU5J2FepciIwAyvMrJOZbQP0AA4iXAH9w8zsLDP7ohybdorlqBZGfvIR77z1Ol2368C5Z5zEiGHv8X+X/Y2flixm1apVAMyZPYsNN2wNwDNPPs5hhx8FQNduu7Dyl5X8+OMCWrduw+yZvzepzZk9iw1bta76E8pxXXv0YvwHg4HQxFFnvfoAbLPr3qxetYpli8MNhlav+o1Hrzqfrj0OY4e91r7Yv3rVKiYMG0znfQ6u2sJXFYV+zKmW1LvRs8DHQEdJMyWdCfQDekiaTIg5/WL2N4GpwBTgYeB8KLrj0/XAyLhcl3AXqPOAR+I235Liwh/kTmAuYmbzCF1PLozfWqdJurdwvaTXJXWPj5dJuk3SGEnvStqg+P4kvS+pa3zcM+YdL+ndmNZN0keSxsa/HSXVBq4Djo01+WMl1Y/9IUfGvL2q4OWoMFf2vZGxX05j1MTJPPjYU+y+597c/8gAdttjL15/5UUABj7zJAccFNoq27TdiA8/eA+Ab77+kpUrf6F58w3Y/6BDeOWlgaxcuZLvpk9j6rdT6Nxlp4ydVy6ZN2Na0eOJw4fSYqNNAfjpx/lFv0S++2I8tmYN9Rs3xcx4pt9ltNxkM/Y57qx19vf16BG02GgzmrZoVTUnkAFKY0nFzI43s1ZmVsvM2prZo2b2o5nta2Yd4t+FMa+Z2QVmtpmZbWdmoxL285iZbR6XxxPSR5nZtnGbC62kn5XF5GQbs5lNjU0ZLVJkrQ+MMbO/Sfo/Qi37wpIyxqD9MLCnmU2L/RIBvoppqyTtB/zLzI6M++tqZhfG7f8F/M/MzpDUBPhM0lAzW17sOH0IXyy0bbdReU6/Sl197b8454yT6HdDX7bdfgdOOCV0x+p7481cetF5PHT/XUjirvsfQRJbbrUNh/U+ij277UBBQT433XaX98goQf++FzFl7KcsW7KIq4/YjYPO+AtffPI+876fhiSabtiGYy+9AYBx77/F8FeeJi8/n1p16nJq37uRxLcTRjJy8Mu03rQjN58easWH9LmUbXbdG4AxQ1+vthf9oLApo3o2kymN4J1xkpaZWYNiaYuBjoQO34kB8nXg32b2vqTVQJ0YVDcFXjKzTpL6A6+b2Quxn+GlQCvgODM7sdhx2gF3EzqGG1DLzLaUdFqx444C6gKr4qbNgAPM7MvSzmuHzl3snQ8+Kf8L48pt4ISZqTO5CnfrWYfx/VcTKySabrVdZ3v8lfdS5tt186ajzaxrRRyzquRkjTkG2dWEK6WrWLtJpm6STZN9C6mU9dcD75nZ4bET+vtJtj/SzL5OcgznXAVSNZ1dLufamGOTw4PAvbGtZjrQSVJerN12S8ieBxwVH58ADE+y64+BvSS1j8cpbMpoDBT29zotIf9SIPES+GDgz4Xj4CV1LtuZOefKqoKGZGedXKkx15M0DqhFqCE/Cdwe140ApgETgc+BMQnbLQe2kTQaWAIcW9oBzGx+bP99KbZfzyNcjb0FeELSJcD/EjZ5D7gslusmQs36TmBCDM7TgUP+yEk755LL1cCbSk4EZjMr9epRrDWfmGT91cDVxdJOS3jcPeHxWxTryhLHuCf25r86pi8Einc3OAfnXJUIvS6qZ2TOicDsnHPryOGmilSqdWAu3pPDOVe9eGB2zrmsIm/KcM65bOM1ZuecyyLCA7NzzmUdb8pwzrks8wem9cxqHpidc7kp3enjcpAHZudczvKmDOecyyJluBlrzvHA7JzLXR6YnXMuu3hThnPOZRlvynDOuWzjgdk557KHT/vpnHPZRt6U4Zxz2ccDs3POZZPqO+1nzt2M1Tnn4PcBJqmWtPYlTZc0UdI4SaNiWjNJQyRNjn+bxnRJulvSFEkTJO2YsJ9TY/7Jkk4t77l5YHbO5S6lsaRvbzPrZGZd4/PLgHfNrAPwbnwOcCDQIS59gAcgBHLgGmBnoBtwTWEwLysPzM65nKU0/v0BvYAn4uMngN4J6QMs+ARoIqkVcAAwxMwWmtkiYAjQszwH9sDsnMtZFdWUARjwjqTRkvrEtJZmNgcg/m0R09sAMxK2nRnTSksvM7/455zLTenfJbt5Ybtx9JCZPVQsz+5mNltSC2CIpK+SH3kdliS9zDwwO+dyUri1VFqReUFCu3GJzGx2/DtP0suENuK5klqZ2ZzYVDEvZp8JtEvYvC0wO6Z3L5b+fjoFLM6bMpxzOasirv1Jqi+pYeFjYH/gc2AQUNiz4lTg1fh4EHBK7J2xC7AkNnUMBvaX1DRe9Ns/ppWZ15idczmrgm7G2hJ4Oda+C4BnzOxtSSOBgZLOBL4Hjo753wQOAqYAPwOnA5jZQknXAyNjvuvMbGF5CuSB2TmXs9JsykjKzKYCO5SQ/iOwbwnpBlxQyr4eAx77o2XywOycy1nVc9yfB2bnXI5S+r0yco4HZudczqqIpoxs5IHZOZezqmdY9sDsnMth1bTC7IHZOZebhMirppHZB5g451yW8Rqzcy5nVdMKswdm51yOEtW2KcMDs3MuJ5V9Hvzc4YHZOZe7qmlk9sDsnMtZ3pThnHNZpnqGZQ/MzrkcVl2HZCvMYOcyQdJ84LtMl6OcmgMLMl2IGiqXX/uNzWyDitiRpLcJr0UqC8ysXDdFzRQPzK5cJI1KdbseVzn8ta/+fOSfc85lGQ/MzjmXZTwwu/Iqfvt3V3X8ta/mvI3ZOeeyjNeYnXMuy3hgds65LOOB2TnnsowHZuecyzIemJ1zLst4YHYZozjRgaTtJfWQ1ELVdfKDDEl4jVtJ2ijT5XHp8cDsMsbMTNLBwHNAD2AYsHdmS1W9xNf4MOAV4A5Jz0taP9Plcsl5YHZVLqEWtyFwLrAP8DbwGzCheD5XfpK2A/4OHAy8AWxLeJ0L1/trnIV8gImrMpLqEipxKyW1AH4E/gE0BroDx5vZNEmHACPMbFHmSpubJNUys9/i4wKgLdATWAWcxe+v8c5m9mkGi+qS8PmYXZWQlAd0A/aS9CVwENAP2IDQfHF6DBi7AP8GTgA8MJeBpFrAnpJWxaTOwEjgKKAhcLiZzZa0N3CzpKPM7PsMFdcl4YHZVQkzWyNpAnA5cDFwkpl9I+kZYHPgjPizem/g72Y2JoPFzVV5hGaK6wiv6WFmNkbSa8BxQHdJjYELgMs8KGcvb8pwVUZSPvAfoBnwA3C1mf0oaVNgS2B9YLKZfSJJ5h/OMpPUHhgKfA08bGYvx/TTgdZAC+A1Mxvqr3H28sDsKlXhf35J7YD5ZvaLpHrArUADMzstXgTcwsyGZba0uSnhNV7PzH6W1AjYAzgC+NjMHpHUDMg3s/mZLa1Lh/fKcJUqoUvca8B9kp42sxXADcAKScOAd4E1mSxnrkoIygcDAyXdBfQwszeA94DdJN1PeP3TuQ2TywLexuwqlaQ/ATcBvYB9gYckNTezA4Dz4k/sqWY2PJPlzFUxKB9A+KI7GzgFuEdSEzN7VNIMwoXUm8zsy0yW1aXPA7OrcMXaLpcDJwJbAGcQam0fShpiZj3M7PEStnFl057wGrcHuhAu7v07vqaPAB+Av8a5xJsyXIVK+Gm9n6Q9gXHAJKA38JCZLQQeB3aQtGPhdh4w0pcwQKdLbK/vD8wBLgIuiRf8xgHXSdo4dlX01ziHeGB2FSoG5QOBe4F6FqwBZgGbSToV2BXY07vElV3CF9+BwFPAjmb2C6Gb3CyglqS9gLnA3mb2XXz9XQ7xwOz+MEn146g+JNUn9FM+z8wGJwz5HRH/ngI8aWZfZaCoOSux1hsnI7oZOMPMRsT0ZcBswhD3R4G3zezrTJXX/THexuz+EElNCIH4Hkm/AiuB2oTaG0Ad4BdgpJm9J6mema3w9s70SWoLHCHpgTjcuhYwy8w+juvrmtkvZnZVHP3X0sxm+mucu7zG7MpNUn0zWwzcA9QHDjazVcCHQD9JTWO/5b2AdyRtQAjS3t6ZpviLYwmhS2FzSc2B74ACSecDxNe4p6QHgDVmNjOm+2ucozwwu3KJQ3uvjm3Gi4ADgdNi2+djwERghKS/A/cRumvN92CRvjjw5mJgAzObBDxAGG7dBLgL2F7SI5IOBW4B3jCz1RkrsKswPvLPlYukOoRuWW2BD83sZUlnAbsBLwODCX2XVwELzOxD/2ldNpK2IQTiT4GHCRWpBwjDrfsTmjT+CswjjPB701/j6sEDsyszSXlxUqK6wOnAdsCQGJzPAXYGXgUGxx4DrowSel/sQKg1TyX88jDCL5IvgJvN7Kfi22SkwK5CeVOGK5P4n3+NpPXjBacHgM+AAyQdbmb/AUYDRwONMlrYHJUQlPPNbDxhVN+mwIWACF+GOwJ9JdUu3M6DcvXhNWZXZnFehmuBr4AfzOxSSacQasofmNlASe3MbEZGC5rDJPUETiY0W7wb/94KTAEeJMwt0t77gldPXmN2ZSKpI3AkcAUhOO8qaYCZDSCM8NtbUksPyuUnaSvCKL4RwGJCE8auwGWEZqMLgKUelKsvrzG7tElqDYwC3jezExLShxMmKnqX0If2uwwVMefFNuUhwHVmdm9M2xO4htA81I7QajGh9L24XOc1ZpcWSRua2WzgamAXSd0SVg8DmsQ2Zw/K5SSpdmxTngD8OWHVCMKNBTYws/EelKs/D8wuJUltgH9JOtXMHiW0dT4p6SJJPQj3lJuT0ULmOIW7Wd8evwD3A2ZL+lDSJoTmi92AehksoqtCHpjdWhJmLiv6bJjZLMJNPXeTdELsiXEroZ35AMJNPv+XMC+GK7uZQEvgn5I2MLO9CaMkxwKHA8eZ2bhMFtBVHQ/MrrgGUHTz1C6Szo3PHyAMdNhX0rFxnt8rCCP+XBklfAFuJWl7M1sE9AGaAtfEie57ENrt9zGzTxO3c9WbB2ZXRNJ6wARJZ8akNsDRks4GMLPHCE0W10o6MT5/DngsTqTjV5JTSAishf/3TgQukrRdDM5/BboSmopamdlRQD1Jr4D3Va4pfHY5BxQNavg59kd+RdI8MxskaTXQJ472+w9huHUnYDyAmV0v6T4f4ZeapC2Ak+M8I5J0I/AIob/y2ZIeNbPxku4hDCZpDMwxs65xqk9XQ3h3OQesNcy6FWFCnCOBE8zslTig5HrCMOBOwJ/jFJ75ZrbahwKnFvt/v0iYK3kN4VZbvYFDgR8Jo/k6E26g2gu4wsw+llQQZ+xzNYgHZldE0r7A3cClwE7x71lxJN9mwCHAqMLJ2V16JG0NPA1cY2aDEtKvBM4jjJhcQuin3AMYYGZvZ6KsLjt4YHZF4oW+lmZ2bXy+P/A84W4kz2W0cDlM4U7hw8wsLz6vZ2Yr4uPbgRbAaWa2SlItM/vNf4XUbH7xzyVaDexQ+MTM3iH8tH5CUitJ+RkrWQ4zs+HAwZK+jZM/rYgz80Ho6aLC5op4hxK/yFfDeWCugUrrcmVmDwNtJT0lqWkcCvwN0NnM5vgk7OVnZm8RLuh9JqlZwsXSlcBiSbW8K5wr5IG5hok1tfbxccfYQ4DC2rCZdQMaAncA9wMjzOyLmMcDxx+QEJxHQdEFwX7Aa2b2m9eSXSFvY65hJHUmDAqpC5xEGLwwPa4r6gEgqR7Q2Mx+8PbOiqVw+60XgWnA383szQwXyWUZD8w1ULzgdD6hl8DNxdZ5EK4CsQdMIzN7OdNlcdnHA3MNkRhw49SSBxPu1/c2YRrPn7zPbNXzL0JXEh/5VwMk3KqoO7Al4Rb3/5J0ImFmuKWS6gOdJPXz4Fx1PCi7kvjFvxogBuUehFverw/sI2k08F9gKHAscCfwhQdl5zLPmzKqKUnNgdaFk6rH+RfGmNnj8fn9QBsz6yWpJVDHzL73n9bOZZ7XmKshhTsnn0mYfKhLTP4ZaJaQ7TJC/9n1zGyumX0P/tPauWzggbkaMrNfgfeB+cCxkjYnTM95haReMdv2wDZAk4wU0jlXKm/KqEYktQO2ikOpibclOhuoDfwb2Bx4mHAPuV2Bf5rZGxkprHOuVB6Yq4nYfDGZcBflh4F5wEuEgSR7EOb2vYPwK2k9YD0z+8LblJ3LPh6Yq5HYP3kQ4T5xjwFXAp8BWxPm/P0ZuNHMJmeskM65lLyNuRoxs/HAYcBehC/dnQnd4KYQ7iV3ClAncyV0zqXDa8zVkKRuwDvA5Wb2QMLdSdqb2bRMl885l5yP/KuGzOwzSfsBb0qqY2Z3xlXTwYcBO5ftvMZcjUnamTCybxtghgdj53KDB+ZqTlIjM/sp0+VwzqXPL/5Vf0vBJ7l3Lpd4jdk557KM15idcy7LeGB2zrks44HZOeeyjAdmV+kkrZY0TtLnkv4rab0/sK/ukl6Pjw+TdFmSvE0knV+OY/SVdGm66cXy9Jd0VBmOtYmkz8taRle9eWB2VWGFmXUys22BX4FzE1cqKPNn0cwGmVm/JFmaEG4661xO8cDsqtqHwOaxpvhlvJPKGKCdpP0lfSxpTKxZNwCQ1FPSV5KGA0cU7kjSaZLujY9bSnpZ0vi47Ab0AzaLtfVbY76/SxopaYKkaxP2daWkryUNBTqmOglJZ8f9jJf0YrFfAftJ+lDSN5IOifnzJd2acOxz/ugL6aovD8yuykgqAA4EJsakjsAAM+sMLAeuAvYzsx2BUcAlkuoSpjE9lDB96Yal7P5u4AMz2wHYEZhEuEvLt7G2/ndJ+wMdgG5AJ6CLpD3jXV6OAzoTAv9OaZzOS2a2Uzzel4Q7xhTahDCR1MHAg/EczgSWmNlOcf9nS2qfxnFcDeRzZbiqUE/SuPj4Q+BRoDXwnZl9EtN3IUxPOiKOhakNfEy4q/e0wqlKJT0F9CnhGPsQZs/DzFYDSyQ1LZZn/7iMjc8bEAJ1Q+BlM/s5HmNQGue0raQbCM0lDYDBCesGmtkaYLKkqfEc9ge2T2h/bhyP/U0ax3I1jAdmVxVWmFmnxIQYfJcnJgFDzOz4Yvk6ARU1CkrATWb2n2LHuLgcx+gP9Daz8ZJOA7onrCu+L4vH/rOZJQbwwrvMOLcWb8pw2eITYPd4f0IkrSdpC+AroL2kzWK+40vZ/l3gvLhtvqRGhOHoDRPyDAbOSGi7biOpBTAMOFxSPUkNCc0mqTQE5kiqBZxYbN3RkvJimTcFvo7HPi/mR9IWkuqncRxXA3mN2WUFM5sfa57PSiqczP8qM/tGUh/gDUkLgOHAtiXs4i/AQ5LOBFYD55nZx5JGxO5ob8V25q2Aj2ONfRlwkpmNkfQ8MA74jtDcksrVwKcx/0TW/gL4GvgAaAmca2a/SHqE0PY8Js5bMh/ond6r42oanyvDOeeyjDdlOOdclvHA7JxzWcYDs3POZRkPzM45l2U8MDvnXJbxwOycc1nGA7NzzmWZ/wd+y+Ebo3ClNwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2015e1cf98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "\n",
    "                          normalize=False,\n",
    "\n",
    "                          title='Confusion matrix',\n",
    "\n",
    "                          cmap=plt.cm.Blues):\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    This function prints and plots the confusion matrix.\n",
    "\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    if normalize:\n",
    "\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "        print(\"Normalized confusion matrix\")\n",
    "\n",
    "    else:\n",
    "\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "\n",
    "    plt.title(title)\n",
    "\n",
    "    plt.colorbar()\n",
    "\n",
    "    tick_marks = np.arange(len(classes))\n",
    "\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "\n",
    "                 horizontalalignment=\"center\",\n",
    "\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    plt.ylabel('True label')\n",
    "\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "\n",
    "\n",
    "# Compute confusion matrix\n",
    "\n",
    "y_test = y_test\n",
    "\n",
    "y_pred = predictions_test_bool\n",
    "\n",
    "\n",
    "\n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "\n",
    "\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plot_confusion_matrix(cnf_matrix, classes=['Not duplicate','Duplicate'],\n",
    "\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8778\n",
      "Recall:  0.7682739819390015\n",
      "Precision:  0.8695121167320178\n",
      "F1 score:  0.8157640815341938\n",
      "ROC AUC score:  0.8528030607994023\n",
      "Average precision-recall score: 0.92\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5,1,'2-class Precision-Recall curve: AP=0.92')"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xm8VeV97/HPlzMAAkIENWEQVNCIqKBHxDphNFZtxNw0SbUZNLUa29okt0na9HaIsc1Nk9w0N21Mq72miRMy1EZiHWKMY4wViEoEgxIQOIAKyCCTTL/7x7O2Z7s5Z519hn32Gb7v12u9zt5rr73Ws9c5Z3338zxrPUsRgZmZWUv6VbsAZmbWvTkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDogeTdKWkJ6tdjs4mabGk6a0sc4SkbZJquqhYFSfpFUnnZ4+vl3R7tctkBg6KLiepv6RbJK2U9KakZyVdVO1ylSM7kO3MDtCvSfp3SYM7ezsRcXxEPNrKMqsiYnBE7Ovs7WcH6T3Z59ws6SlJp3f2dvoKST+QtFfSyJL5nbKfJf1+9v+0XdKPJB2Ss+wlkl7ItvmUpIlFr10haaGkrZIaJX1DUm1by9MbOSi6Xi2wGjgHGAr8DTBb0rgqlqktLomIwcDJwKnAX5cuoKSn/23Nyj7nCOARYE6Vy9PpuuIgKGkQ8LvAFuBjzSxS2M+HAk8Cd0tSG9Z/PHAT8AngcGAH8L0Wlp0A3AFcCwwDfgzMK9oPBwGfI/3OTwPOA75Qbll6s57+z9zjRMT2iLg+Il6JiP0RcS+wAjilpfdIGiPpbknrJW2U9N0WlvuOpNXZN6KFks4qem2qpAXZa69J+sds/gBJt2fr3SxpvqTDy/gca4D7gUnZeh6V9FVJPyf9sx4laWhWe1onaY2kvy9uKpJ0taQXs5rVEkknZ/OLm2BaKvc4SVH4J5c0UtI8SW9IWibp6qLtXC9ptqRbs20tltTQ2mfMPude0sFllKRDi9b5AUnPFX0TPrHotWZ/X5KOlvSzbN4GSXdIGlZOOUpJujTb/lZJv5F0Yem+K/rst5fss6skrQJ+JukBSdeVrPt5SR/KHr9X0kPZfl0q6aNtLOrvApuBG4ArWlooIvYAPwTeDQxvw/o/Bvw4Ih6PiG2kL14fkjSkmWV/G3giIp7Mfq9fB0aRvrQREf8SEU9ExO7s7/sO4Iw2lKXXclBUWXZQPgZY3MLrNcC9wEpgHOkP+64WVjcfmAwcAtwJzJE0IHvtO8B3IuJg4Ghgdjb/ClLNZgzpH/RaYGcZ5R4DXAw8WzT7E8A1wJCsvD8E9gLjgSnABcAfZu//CHA98EngYGAGsLGZTbVU7lIzgUZgJPBh4H9LOq/o9Rmk/TYMmAc0G7bNfM76rIwbgU3ZvJOB7wOfJu2zm0jfTPu38vsS8LWsjMeR9vn15ZSjpExTgVuBL2af52zglTas4pxs+79N+ju5vGjdE4GxwH9ltYGHsmUOy5b7XvYtvtDks6iVbV1B+t3cBby38GWgmc/UH7gSaIyIDZLOzEK4penM7K3HA88X1hMRvwF2k/6nDthMNpU+n9RC2c+mhf/LPiciPFVpAuqAnwI35SxzOrAeqG3mtSuBJ3Peuwk4KXv8OPAVYETJMn8APAWcWEZ5XwG2kb4hriRV8Qdmrz0K3FC07OHAW4XXs3mXA49kjx8EPpuznfNbKfc4IEhNeWOAfcCQote/Bvwge3w98NOi1yYCO3M+5/Wkg83mbL0bgelFr/8L8Hcl71lKOgC3+PtqZjsfBJ5t4XNfD9zewvtuAr7d2r4rXU/RPjuq6PUhwHZgbPb8q8D3s8e/R/oGXrrtL5f5930EsB+YXPQ7/04L+/l14GfAKW38H3oYuLZk3pri31fR/Pdmn3U6UE+qfewH/rKZZT9F+uIxoi3l6a2TaxRVotSGfxvpH+W6ovn3K3W0bZP0MdJBcGWkqnJr6/x81pSzRdJmUk1hRPbyVaRvWb/Ompc+kM2/jfQPfJektUodeHU5m/lgRAyLiLER8ccRUVz7WF30eCwpCNcVvgWSDjKHZa+PAX7T2mfKKXexkcAbEfFm0byVpG/zBa8WPd4BDJBUK+ljRfv7/qJlZkfEMFLgvcA7mwbHAp8v/oabfZ6R5Py+JB0m6a6sGW4rcDtNv5+2KHffteTt31O2z/4LuCybdRmpyQXS5zyt5HN+jNQ8VI5PAC9GxHPZ8zuA3y/5+5qd/T0dFhHvi4iFbfws20g10mIHA2+WLhgRvybVcL4LrCPt+yWkQHibpA8C/wBcFBEb2lieXsk9+lUgScAtpIPQxZHaZwGIiItKlj0dOEJSbV5YKPVH/AWpA25xROyXtImsqh0RLwOXZwH1IWCupOERsZ30jf0rSh3q95G+Hd/Sjo9WPBTxalKNYkQL5V5NakrKX2EL5S5ZbC1wiKQhRWFxBOmbZWvrv4OmA2Nzr2+Q9GlgvqQ7I2JdVvavRsRXS5dv5ff1NdI+OjEiNmYHpLKawErk7bvtpE7ZguYO6qVDRs8EvizpcWAgqfO+sJ3HIuL97SgjpCa7IyQVQrqW1FR3Ean5r0XZ3/P9OYtcFBFPkJqGTip631FAf+Cl5t4UEXOBudmyw0g16vlF778Q+DfgdyLiV3ll7Etco6iOfyG1EV9S8o28Oc+Qvv38g6RBSp3PzXWwDSH1B6wHaiX9LUXftCR9XNKhEbGfVNUH2CfpXEknZG3rW4E9pOaWDskOqD8BviXpYEn9ss7cc7JF/h/wBUmnKBkvaWzpeloqd8m2VpOaz76W7Z8TSTWRFgOgjZ/l16Ra159ns/4NuFbSaVnZB0n6nawDNe/3NYSs6U7SKFIfQ3vcAnxK0nnZfh0l6b3Za88Bl0mqU+qw/3AZ67uPVHu4gXQW0v5s/r3AMZI+ka2vTtKpko5rbYVZYB4NTCX1m00m9QXcSU6ndkGkTuXBOdMT2aJ3AJdIOivrU7kBuLukdllcrlMk1SidmHATqSP819lr78vW97sR8UxrZexLHBRdLDsYfpr0j/NqSTPTASJdJ3AJqUN4Fama/HvNLPog6RvYS6Rml128synoQmCxpG2kDuLLImIX6RvnXFJIvAg8RmoS6QyfJLUFLyH1l8wF3pN9rjmk9vA7Sc0EPyJ1wpdqqdylLie1wa8F/pPUjv5QJ30OgG8C10g6LCIWAFeTagObgGWk/qLWfl9fIZ1WvIXU3HN3ewqSHcQ+BXw7W9djpAM9pHb3o7NyfYW0f1tb31tZWc4vXj472F5Aao5aS2q++zrpGztZs11Lnb1XAPdExK8i4tXCRPodfkA51zq0RUQsJp2AcQepn2MI8MeF17Om3P9V9JbvkL5wLM1+Xl302t+Qmmvva6E5ss9ShG9cZGZmLXONwszMcjkozMwsl4PCzMxyOSjMzCxXj7uOYsSIETFu3LhqF8PMrEdZuHDhhog4tPUlD9TjgmLcuHEsWLCg2sUwM+tRJK1s73vd9GRmZrkcFGZmlstBYWZmuRwUZmaWy0FhZma5HBRmZparYkEh6fuSXpf0QguvS9I/Kd3feJFauEWimZlVVyWvo/gBaRjmW1t4/SJgQjadRrpHw2nlrHj//taX6Uv6uV5oZhVUsaCIiMezO6a15FLg1kjjnD8taZik92Q3vGnRtm3wxBN5S/Q9q1eD1PpybdHW9RXCu64O+vdPzyPSekpHsi88L7znoIPS+wYNSqEXAe96F+zcCQcfDFu3wqGHQm1tWl9hevPN9J76+qZ5u3al9dXUpPXU1TW91q/fO3+aWXmqeWX2KN55Y53GbN4BQSHpGuAagBEjxrF6tb9FF2zfDjt2pANmqfbeaqQ9B9H9+9OB++DSuxeXUabN2X3r+vWDPXtg9+4UNvv2pXmlB/bCwb7wuHgqrKfczyClUBo5EgYOTO/dujX9rK+HoUNTYA0fnub1758m//1ZX1LNoGjuX7nZQ1tE3AzcDHDssQ0xYUL6dmm921tvpRrk6683zSsETWmtJCI93rOnqTaxa1c62BdqIhEpBAoH+m3bUrBt3w4rV6baR0QKqkJA1dQ0hVBNTZoK+veHsWPTvEKoDB6cpkJNxqw3qObhthEYU/R8NOl2i2ZA07f34cO7bpv79sHevbB+faohFQJo374UIHv2pMe7dqUAevXV9LMQIoXHkILmiCNSgEip+awQWIVlhwx5Z/iYdUfVDIp5wHWS7iJ1Ym9prX/CrNIKB/zRo8tbft++1Ee0Y0cKkV27UlNVRKpZLFqUHhcCpNBs1lyT2ogRKUxGjIDDD4cBAyrzGc3aqmJBIWkmMB0YIakR+DJQBxAR/wrcB1xMujH9DtLN4s16lJoaaG3U+82bYe3aFBgRTTWSbdtSLWX48BQyGzfC8uVNNSlINY7Ro9PP/v3hkEOaOuvNukolz3q6vJXXA/iTSm3frLsYNixN5diwAVatSsFSU5OeNzamPo9CX0ihFjJ+PIwaBYcd1hQsZpXgLmGzbmTEiDQV27QJ1q1Lnfs7dqQ+lF27Uk1l8eKmkDjmmBQkhX4R1zqsszgozLq5d70rTaU2boQXX0wd6gMGwJYtKSief74pJMaNS01bQ4ak/g/XPKw9HBRmPdTw4XDmmU3P16xJ07ZtqfO8rg5++csUHv37N50mPGFC6iwfNsxnXVl5HBRmvcSoUWkqtn9/qnXs3p36PQYMSE1WhfAohMSpp8JRR6VwMSvloDDrxfr1g+OPf+e8Xbvg5ZfThYaF60YeewyeeSbVOI47LtVWRo5MgWLmoDDrYwYMgBNOaHq+fz8891zq66ivh6efTsvU16fphBNSE9V73uMREfoq/9rN+rh+/eDkokH+X38dVqxIwTF4cGqqqqtrql28732pc33gwOqU17qeg8LM3uGww9IEqVlq6dJ0tfkbb6TAuO++pqvGjz8+1TTe/e7qldcqz0FhZi2qrX1nH8eePbBkSTo1d/9+eOqppmHizzoLxozxyLq9kYPCzMpWVwcnndT0fMUKeOmlFBY//WnTdRpnnZWGHvGpt72Dg8LM2u3II9P05puwYEHT0OwPPZSap8aNg0mTyh/CxLonB4WZddiQIXDuuenx3r3w7LPpuo3Nm2HZstSENX58asYaMqS6ZbW2c1CYWaeqrU0X8AG88EK6UnzHjhQaS5emGse556brNHxzp57BQWFmFTNpUtPjtWvTMOoADzzQNJz6BRekQQyt+3JQmFmXGDkyTVu2pGFFNmxI/Rj/8R+pM3zGjPTTuh8HhZl1qaFDYdq09HjLFli4MHWGz5mT+jEmT4ZBg6pbRnsnB4WZVc3QoelK7+eeSzWMbdtS53dNDXzoQ65hdBcOCjOrusmT089Fi1Jg1NWlGkZNDVx6qc+UqjYHhZl1GyeemH6+8ELTDZnmzk3B4RpG9fhiezPrdiZNgvPPT0Oeb9qUxpmaMwd+/nOIqHbp+h7XKMys2xo+HM45J/VhNDame2msWJEu3Jsypdql6zscFGbW7RX6MObPb7rd6+LFbo7qKm56MrMe49RTU5PU1q2wfn1qjlq8uNql6v0cFGbW45x3XjoTas0a+MUv4Pbb0w2XrDLc9GRmPdKUKemeGI8/ni7c+/GP0x35PvIR3xOjs3l3mlmP1a8fTJ+e7n/x+uvphkozZ6ZOb+s8Dgoz6/FqatLggv36pcEHZ82CBx+sdql6DweFmfUap54KRx8N69bBypWp72LnzmqXqudzUJhZrzJ2bDozavt2eO01mD0b7rgD9u2rdsl6LgeFmfU6Epx9droFa2NjumnSzJnpp7Wdg8LMeq3Jk+HCC9PjNWvgnnvSvb2tbXx6rJn1etOmpbvrvfwyvPVW6r+45BKor692yXqGitYoJF0oaamkZZK+1MzrR0h6RNKzkhZJuriS5TGzvuuoo9KZUZs3p+aoWbNg9epql6pnqFhQSKoBbgQuAiYCl0uaWLLYXwOzI2IKcBnwvUqVx8xMSld1DxyYwuInP0md3e7ozlfJGsVUYFlELI+I3cBdwKUlywRwcPZ4KLC2guUxMwPglFPShXqvvprGjJo5M/VhWPMqGRSjgOKKXWM2r9j1wMclNQL3AX/a3IokXSNpgaQFW7asr0RZzayPqa9PHd1SCokHHvB1Fy2pZFComXmltxy5HPhBRIwGLgZuk3RAmSLi5ohoiIiGoUMPrUBRzayvOu00OPNMePPNNATI7NnpsTWpZFA0AmOKno/mwKalq4DZABHxC2AAMKKCZTIzO8CAAWm8qFGj0lXdd9+dBhm0pJJBMR+YIOlISfWkzup5JcusAs4DkHQcKSjctmRmVXHkkTByZDobas2adL8Ld3RXMCgiYi9wHfAg8CLp7KbFkm6QNCNb7PPA1ZKeB2YCV0b4jrhmVj3HH5/6LnbsSAMM3nln6vTuy9TTjsvHHtsQt966gFpfKmhmFbZgQeqvGDYs3Sjpgx+E/v2rXar2kbQwIhra814P4WFm1oKGhjRm1Jtvpr6LWbNg1apql6rrOSjMzHLU1KSwGDw49Vs8/HDfCwsHhZlZGaZMgalTU3/FT37St86KclCYmZVp2DB43/tSWKxdm06j7QscFGZmbVBbm86K2rAhDf8xaxbs2VPtUlWWg8LMrI2kNBLtzp2pk/uuu2DTpmqXqnIcFGZm7dCvX7qau74+dXLPmwdLl1a7VJXhoDAz64DTToNjjkl9Fk8+CS+9VO0SdT4HhZlZB40dC2ecAa+9Bo89Bo8+Wu0SdS4HhZlZJxg0KJ0R9dprsGJFCozewkFhZtZJ6upSv8WGDam/YtasapeoczgozMw60eDB6UruN95I0+239/wRaB0UZmadrLYW3v/+dPrs+vVwxx2wa1e1S9V+Dgozswo555x0U6RXX03NUD31znkOCjOzCpo6FYYOTWFx992wfXu1S9R2DgozswpraEjjRL36Ksydmzq7exIHhZlZF2hogHe/O4XFPffAAw9Uu0Tlc1CYmXWRSZOaTp9dubLnXJjnoDAz60L9+8P06Sksli/vGWdDOSjMzLpYbS2cfnrTMOXdvYPbQWFmVgVDh6bptddSB3d3vr2qg8LMrEpOPTUNKLh2LTzyCOzfX+0SNc9BYWZWRePHw5gxqRlqzhyIqHaJDuSgMDOrsuOOS/0WhbvldbexoRwUZmbdwFlnQU1Naoa6555ql+adHBRmZt3Eb/1WU1jMmgV79lS7RImDwsysGznrLJCgsRFmzuwezVAOCjOzbmbaNDjooDTcx5w51T8bykFhZtYNTZ2abq/aHTq4HRRmZt3UtGmpNrFuXboor1ocFGZm3dj06amDe906uP/+6pSh7KCQNErSb0k6uzBVsmBmZpacdVZqelqxAhYu7PrtlxUUkr4O/Bz4a+CL2fSFMt53oaSlkpZJ+lILy3xU0hJJiyXd2Yaym5n1GWeeCZs2waJFsGNH1267tszlPggcGxFvlbtiSTXAjcD7gUZgvqR5EbGkaJkJwF8CZ0TEJkmHlV90M7O+ozDi7DPPwL33wkc+kk6j7QrlNj0tB+rauO6pwLKIWB4Ru4G7gEtLlrkauDEiNgFExOtt3IaZWZ8xdCgMGQKrV8O8eV233XKDYgfwnKSbJP1TYWrlPaOA1UXPG7N5xY4BjpH0c0lPS7qwzPKYmfVJ06ZBXR288krX3SGv3KanednUFs1VikrHRawFJgDTgdHAE5ImRcTmd6xIuga4BuDww49oYzHMzHqX00+Hhx5KYfHWW+mueZVUVo0iIn4IzAQWZtOd2bw8jcCYouejgbXNLHNPROyJiBXAUlJwlG7/5ohoiIiGoUMPLafIZma9Vr9+cNpp6crtrjhlttyznqYDL5M6p78HvFTG6bHzgQmSjpRUD1zGgbWSHwHnZtsYQWqKWl526c3M+qhhw6C+Po0J9fDDld1WuX0U3wIuiIhzIuJs4LeBb+e9ISL2AtcBDwIvArMjYrGkGyTNyBZ7ENgoaQnwCPDFiNjYng9iZtbXnHEGbN2abqNaySE+FGXcTknSoog4sbV5XeHYYxvi1lsXUFtu74qZWS+2dSvMnw+TJsHFF7e8nKSFEdHQnm2Ue7hdIOkW4Lbs+cdIfRVmZlZFBx+c+ixWrYING2DEiM7fRrlNT38ELAY+A3wWWAJc2/nFMTOztpoyBTZvThfjVUJZNYrsiux/zCYzM+tGhg1Lo8yuXAkvvwwTDjh3tGNyaxSSZmc/fyVpUenUuUUxM7P2OvdceOONNBZUZ2utRvHZ7OcHOn/TZmbWWerrYeDAdG1FZ/dV5NYoImJd9nADsDoiVgL9gZM48OI5MzOromnT0llQnd1XUW5n9uPAAEmjgIeBTwE/6NyimJlZR/Tvn0aZXbUKyrjyoWzlBoUiYgfwIeCfI+J/ABM7rxhmZtYZTjwRtm+HJ5/svHWWHRSSTiddP/Ff2Txf8mZm1s0MGwY7d6ZaRWcpNyg+R7rB0H9mw3AcRRpyw8zMupHa2hQWmze3vmy5yh099rGImBERX8+eL4+Iz3ReMczMrLMcfTTs2gVPPdU568ttPpL0fyPic5J+zIH3kiAiZjTzNjMzq6JDDknNTxs7aYjV1voZCmM7/Z/O2ZyZmVVabW26pmLtWti7lw4Popr79ogoDPy3ANgZEfsBJNWQrqcwM7NuaPJk+OUv09lP06d3bF3ldmY/DBxU9Hwg8NOObdrMzCpl8OB0m9Tt2zu+rnKDYkBEbCs8yR4flLO8mZlVUW1tuuhu/fqOr6vcoNgu6eTCE0mnADs7vnkzM6uUUaNg27Y0smxHlNvF8TlgjqTC+E7vAX6vY5s2M7NKGj48XU+xbl3ry+Yp934U8yW9FzgWEPDriNjTsU2bmVklDR4Me/bAK690bD1lNT1JOgj4C+CzEfErYJwkDz1uZtaN1dSkoNi1q2PrKbeP4t+B3cDp2fNG4O87tmkzM6ukgQNTh/bWrR1bT7lBcXREfAPYAxARO0lNUGZm1k3V1oIEW7Z0bD3lBsVuSQPJhvGQdDTwVsc2bWZmlTZ+fKFGoXZ/uS/3rKcvAw8AYyTdAZwBXNnejZqZWdeoq0v9FFBf1951tBoUkgT8mnTTommkJqfPRsSG9m7UzMy6xqBBsG8fQF27R3xq9Y0REZJ+FBGn0HTTIjMz6wEGDkwDA3ZEuX0UT0s6tWObMjOzrlZbW6hRdGAdZS53LnCtpFeA7aTmp4iIEzu2eTMzq6R+b1cH2n+iarlBcVG7t2BmZlXTFBTtT4rW7nA3ALgWGA/8CrglIjrY2mVmZl1FgoM6ONZ3a30UPwQaSCFxEfCtjm3OzMy6klSoVVSu6WliRJyQNqZbgGfavSUzM+ty/fqlMZ/oQFK0VqN4e4RYNzmZmfVMde2+1C5pLShOkrQ1m94ETiw8ltTqMFOSLpS0VNIySV/KWe7DkkJSQ1s/gJmZ5Uunx1ao6Skiatq7Ykk1wI3A+0mjzc6XNC8ilpQsNwT4DPDf7d2WmZm1bNAgoIJNTx0xFVgWEcsjYjdwF3BpM8v9HfANoIMjppuZWXMOOaRj769kUIwCVhc9b8zmvU3SFGBMRNybtyJJ10haIGnBli2dcKdwM7M+5PDDId2Zon0qGRTNVXPeLqikfsC3gc+3tqKIuDkiGiKiYejQQzuxiGZm1ppKBkUjMKbo+WhgbdHzIcAk4NFsaJBpwDx3aJuZdS+VDIr5wARJR0qqBy4D5hVejIgtETEiIsZFxDjgaWBGRCyoYJnMzKyNKhYU2XUX1wEPAi8CsyNisaQbJM2o1HbNzKxztftGFuWIiPuA+0rm/W0Ly06vZFnMzKx9Ktn0ZGZmvYCDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMclU0KCRdKGmppGWSvtTM638maYmkRZIeljS2kuUxM7O2q1hQSKoBbgQuAiYCl0uaWLLYs0BDRJwIzAW+UanymJlZ+1SyRjEVWBYRyyNiN3AXcGnxAhHxSETsyJ4+DYyuYHnMzKwdKhkUo4DVRc8bs3ktuQq4v7kXJF0jaYGkBVu2rO/EIpqZWWsqGRRqZl40u6D0caAB+GZzr0fEzRHREBENQ4ce2olFNDOz1tRWcN2NwJii56OBtaULSTof+CvgnIh4q4LlMTOzdqhkjWI+MEHSkZLqgcuAecULSJoC3ATMiIjXK1gWMzNrp4oFRUTsBa4DHgReBGZHxGJJN0iakS32TWAwMEfSc5LmtbA6MzOrkko2PRER9wH3lcz726LH51dy+2Zm1nG+MtvMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSjMzCyXg8LMzHI5KMzMLJeDwszMclU0KCRdKGmppGWSvtTM6/0lzcpe/29J4ypZHjMza7uKBYWkGuBG4CJgInC5pIkli10FbIqI8cC3ga9XqjxmZtY+tRVc91RgWUQsB5B0F3ApsKRomUuB67PHc4HvSlJEREsrjYBdu6C2kiU3M+tFdu8GULvfX8nD7ShgddHzRuC0lpaJiL2StgDDgQ3FC0m6Brgme7Z7+vSDfwMtZkkfsuddULep2qXoHrwvmnhfNPG+SCTYdkR7313JoGguvkqP7uUsQ0TcDNwMIGlBxNaGjhev50v7Ypf3Bd4XxbwvmnhfNJG0oL3vrWRndiMwpuj5aGBtS8tIqgWGAm9UsExmZtZGlQyK+cAESUdKqgcuA+aVLDMPuCJ7/GHgZ3n9E2Zm1vUq1vSU9TlcBzwI1ADfj4jFkm4AFkTEPOAW4DZJy0g1icvKWPXOs6AXAAAEzklEQVTNlSpzD+R90cT7oon3RRPviybt3hfyF3gzM8vjK7PNzCyXg8LMzHJ126Dw8B9NytgXfyZpiaRFkh6WNLYa5ewKre2LouU+LCkk9dpTI8vZF5I+mv1tLJZ0Z1eXsauU8T9yhKRHJD2b/Z9cXI1yVpqk70t6XdILLbwuSf+U7adFkk4ua8UR0e0mUuf3b4CjgHrgeWBiyTJ/DPxr9vgyYFa1y13FfXEucFD2+I/68r7IlhsCPA48DTRUu9xV/LuYADwLvCt7fli1y13FfXEz8EfZ44nAK9Uud4X2xdnAycALLbx+MXA/6Rq2acB/l7Pe7lqjeHv4j4jYDRSG/yh2KfDD7PFc4DxJ7b9GvftqdV9ExCMRsSN7+jTpmpXeqJy/C4C/A74B7OrKwnWxcvbF1cCNEbEJICJe7+IydpVy9kUAB2ePh3LgNV29QkQ8Tv61aJcCt0byNDBM0ntaW293DYrmhv8Y1dIyEbEXKAz/0duUsy+KXUX6xtAbtbovJE0BxkTEvV1ZsCoo5+/iGOAYST+X9LSkC7usdF2rnH1xPfBxSY3AfcCfdk3Rup22Hk+Ayg7h0RGdNvxHL1D255T0caABOKeiJaqe3H0hqR9pFOIru6pAVVTO30UtqflpOqmW+YSkSRGxucJl62rl7IvLgR9ExLcknU66fmtSROyvfPG6lXYdN7trjcLDfzQpZ18g6Xzgr4AZEfFWF5Wtq7W2L4YAk4BHJb1CaoOd10s7tMv9H7knIvZExApgKSk4epty9sVVwGyAiPgFMAAY0SWl617KOp6U6q5B4eE/mrS6L7LmlptIIdFb26GhlX0REVsiYkREjIuIcaT+mhkR0e7B0Lqxcv5HfkQ60QFJI0hNUcu7tJRdo5x9sQo4D0DScaSgWN+lpewe5gGfzM5+mgZsiYh1rb2pWzY9ReWG/+hxytwX3wQGA3Oy/vxVETGjaoWukDL3RZ9Q5r54ELhA0hJgH/DFiNhYvVJXRpn74vPAv0n6n6Smlit74xdLSTNJTY0jsv6YLwN1ABHxr6T+mYuBZcAO4FNlrbcX7iszM+tE3bXpyczMugkHhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4VZCUn7JD0n6QVJP5Y0rJPXf6Wk72aPr5f0hc5cv1lnc1CYHWhnREyOiEmka3T+pNoFMqsmB4VZvl9QNGiapC9Kmp+N5f+VovmfzOY9L+m2bN4l2b1SnpX0U0mHV6H8Zh3WLa/MNusOJNWQhn24JXt+AWmspKmkwdXmSTob2EgaZ+uMiNgg6ZBsFU8C0yIiJP0h8OekK4TNehQHhdmBBkp6DhgHLAQeyuZfkE3PZs8Hk4LjJGBuRGwAiIjC4JSjgVnZeP/1wIouKb1ZJ3PTk9mBdkbEZGAs6QBf6KMQ8LWs/2JyRIyPiFuy+c2NhfPPwHcj4gTg06SB6Mx6HAeFWQsiYgvwGeALkupIg879gaTBAJJGSToMeBj4qKTh2fxC09NQYE32+ArMeig3PZnliIhnJT0PXBYRt2VDVP8iG6V3G/DxbKTSrwKPSdpHapq6knRXtTmS1pCGPD+yGp/BrKM8eqyZmeVy05OZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeX6/0dap5vT5hpLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f200510e518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_test = y_test\n",
    "\n",
    "y_pred = predictions_test_bool\n",
    "\n",
    "y_pred_prob = predictions_test[:,1]\n",
    "\n",
    "\n",
    "\n",
    "rec = recall_score(y_test, y_pred)\n",
    "\n",
    "prec = precision_score(y_test, y_pred)\n",
    "\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "\n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "\n",
    "roc_auc = roc_auc_score(y_test, y_pred)\n",
    "\n",
    "\n",
    "\n",
    "print(\"Accuracy: \", acc)\n",
    "\n",
    "print(\"Recall: \", rec)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "\n",
    "print(\"F1 score: \", f1)\n",
    "\n",
    "print(\"ROC AUC score: \", roc_auc)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "average_precision = average_precision_score(y_test, predTest)\n",
    "\n",
    "\n",
    "\n",
    "print('Average precision-recall score: {0:0.2f}'.format(\n",
    "\n",
    "      average_precision))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "precision, recall, _ = precision_recall_curve(y_test, predTest)\n",
    "\n",
    "\n",
    "\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "\n",
    "         where='post')\n",
    "\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2,\n",
    "\n",
    "                 color='b')\n",
    "\n",
    "\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "\n",
    "plt.ylabel('Precision')\n",
    "\n",
    "plt.ylim([0.0, 1.05])\n",
    "\n",
    "plt.xlim([0.0, 1.0])\n",
    "\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}