text_nb_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "26734f92",
   "metadata": {},
   "source": [
    "# Text classification with Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43c84a0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism',\n",
       " 'comp.graphics',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.windows.x',\n",
       " 'misc.forsale',\n",
       " 'rec.autos',\n",
       " 'rec.motorcycles',\n",
       " 'rec.sport.baseball',\n",
       " 'rec.sport.hockey',\n",
       " 'sci.crypt',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'sci.space',\n",
       " 'soc.religion.christian',\n",
       " 'talk.politics.guns',\n",
       " 'talk.politics.mideast',\n",
       " 'talk.politics.misc',\n",
       " 'talk.religion.misc']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the 20newsgroups corpus to classify short documents to categories\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "# Fetch data and print the categories\n",
    "data = fetch_20newsgroups()\n",
    "data.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4a2ddc25",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use a few of these categories and get the corresponding training and test sets\n",
    "categories = ['talk.religion.misc', 'soc.religion.christian', 'sci.space', 'comp.graphics']\n",
    "\n",
    "train = fetch_20newsgroups(subset='train', categories=categories)\n",
    "test = fetch_20newsgroups(subset='test', categories=categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f81ba3fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: jono@mac-ak-24.rtsg.mot.com (Jon Ogden)\n",
      "Subject: Re: Losing your temper is not a Christian trait\n",
      "Organization: Motorola LPA Development\n",
      "Lines: 26\n",
      "\n",
      "In article <Apr.23.02.55.47.1993.3138@geneva.rutgers.edu>, jcj@tellabs.com\n",
      "(jcj) wrote:\n",
      "\n",
      "> I'd like to remind people of the withering of the fig tree and Jesus\n",
      "> driving the money changers et. al. out of the temple.  I think those\n",
      "> were two instances of Christ showing anger (as part of His human side).\n",
      "> \n",
      "Yes, and what about Paul saying:\n",
      "\n",
      "26 Be ye angry, and sin not: let not the sun go down upon your wrath:\n",
      "(Ephesians 4:26).\n",
      "\n",
      "Obviously then, we can be angry w/o sinning.\n",
      "\n",
      "Jon\n",
      "\n",
      "------------------------------------------------\n",
      "Jon Ogden         - jono@mac-ak-24.rtsg.mot.com\n",
      "Motorola Cellular - Advanced Products Division\n",
      "Voice: 708-632-2521      Data: 708-632-6086\n",
      "------------------------------------------------\n",
      "\n",
      "They drew a circle and shut him out.\n",
      "Heretic, Rebel, a thing to flout.\n",
      "But Love and I had the wit to win;\n",
      "We drew a circle and took him in.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print a sample\n",
    "print(train.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d1a84519",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need to convert text to vectors\n",
    "# Use the TFIDF vectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "\n",
    "# Create a pipeline to train our model: vectorize text -> naive bayes\n",
    "model = make_pipeline(TfidfVectorizer(), MultinomialNB())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d92cc208",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model using the training set (and the corresponding labels/categories)\n",
    "model.fit(train.data, train.target)\n",
    "\n",
    "# Predfict on test set\n",
    "labels = model.predict(test.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "54d18130",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(89.133125, 0.5, 'predicted label')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAFjCAYAAADowmrhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABIsElEQVR4nO3de1zP9///8du7d6UcmxwzLKeYzZmYDZUhhEJCsc35kGNOk5jDEOYrdsCYmFOItBxyKDbHHHIuNuehHCIddHq/f3/4eX/WlLZV79e7PK6Xyy7zfr3eh/vL4d7z/XydVFqtVosQQghFGSkdQAghhJSxEEIYBCljIYQwAFLGQghhAKSMhRDCAEgZCyGEATBWOoAwHMkHVigdIV9YdJqldIR8kaHRKB0hXxipVEpHyDepKXezXScjYyGEMABSxkIIYQCkjIUQwgBIGQshhAGQMhZCCAMgZSyEEAZAylgIIQyAlLEQQhgAKWMhhDAAUsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwAFLGQghhAKSMhRDCAEgZCyGEAZAyFkIIAyBlLPLcpvCzuMxaQ/dZaxjzww6ePE/KtH7c8iDmbj7w2uvik17QadqP7DtzVV9R81TdurUJDQ3gxPHdHD0SQsOGHyodKU+t+nEx48YOUTpGnuvSpT2PHl5ROoaUsaGysbHJcvmgQYOIiYnRc5p/7vLtGPz3n8Lfqzfbpn1GlbIWfBt8RLf+p9CTnP3jz9dep9Vq8V6zm4QXKfqMm2fMzc0I+eVnvln0PbbNHZk7dwn+a/yUjpUnateuwb69AfTs4aR0lDxXo4Y18+ZNw8hI+So0VjqA+HdWrlypdIQ3er9KeXZ+9QUmajUpaenEPkvAyrIUABHRtzl6+SY9PqlPfNKLTK9bufs4NSuVJTElVYnYufZp29Zcv3GLPXvDAAj+JZQbN28rnCpvDBv6GWvWbub2ndd/iBZk5uZmrPnJj4kTv2Kt/zKl4xSeMtZqtSxcuJD9+/ejVqvp1asXrVq1wsfHh6dPn1K0aFGmTp1KvXr1mDx5Mubm5pw+fZrnz5/z5ZdfEhQURFRUFG3btmXy5MkEBgYSGhrKs2fPePz4MXZ2dkyePBmVSpXpc9euXcvPP/9MiRIlqFatGlWqVMHT05PmzZtTt25dHj16xNatW/nqq6+4du0ajx49wtrammXLlvHo0SOGDRtG5cqVuXXrFlZWVixYsAALCwsAfHx8iIyMBGDp0qVUrVoVe3t71q5dS9myZfnqq684ffo0JiYmDB8+nI4dOzJ//nyOHDmCWq3GwcGBkSNH6vlPAkzUag5GXmPm+lBMjI0Z1rklsU8T8N0Sxnee3dn66/lMzz96+Sanr93lO8/uDF6yRe9580LNmtbEPHjIDz8soN6H7/Ps2TOmfPm10rHyxOgx3gDY232scJK89d2381n5489cuKD8FAUUommKPXv2cObMGYKDg9myZQuBgYEMHToUDw8PgoODmTJlCqNHjyY19eXIKzY2lp07dzJq1CimTJnCV199xY4dOwgICOD58+cAXLx4kaVLl/LLL79w7tw59u3bl+kzo6KiWL9+PYGBgWzYsIFbt27p1sXFxTF48GCCgoKIjIzExMSEzZs3s2/fPlJSUjh06BAAV69epX///oSEhFC9enWWLfvfT+iPPvqInTt30rJlSzZt2pTps9etW0dSUhK7d+/mp59+4ttvv+XPP//k8OHD7Ny5k02bNnHz5k1SUpT52m/foCbhC0YwtFMLhi3dyuTVvzChpx1lSxXP9Lz7T+L5JvAQcz7riNoAvir+V8YmJnToYM+qVev5qGUnvvtuDUE7/DE1NVU6msjCkCH9SE9Px99/s9JRdArNyDgiIgJHR0dMTU0xNTVlw4YN2NnZ0a5dOwAaNGhAqVKluH79OgCtWrUCwMrKipo1a2JpaQmAhYUFz549A8De3p4yZcoA0LFjR44fP657P4Bjx45hZ2dH8eIvC6ZTp07Ex8fr1tevXx+Apk2bYmFhwfr167l+/To3b94kKenlTq333nsPW1tbALp164aXl5fu9W3btgWgRo0anDp16rXtdXV1xcjIiLJlyxISEkJ6ejpFihTBzc0NOzs7xowZQ5EiRXL9e/tv3I6N43F8Ig1rvAtAt48+YM7G/TxNSGbhtnAAHscnotFoSU1Lx7qCJS9S0xj+7TYA7jx8yuLth3iakEzPVvX1mj037t+PITr6dyIiIoGX0xTff+9LNesqREX/rmw48Zp+Hq4ULWpGxMm9mJqaYG7+8tdduvbj/n1l9skUmjI2Ns68KXfu3EGr1WZaptVqycjIAMDExCTb176iVqt1v9ZoNJkeAxgZGaHRaLLNZGZmBsCBAwfw8/OjX79+uLi4EBcXp8v218/WarWZPuPVOpVK9dq2/D3zrVu3qFixIlu2bOHkyZMcPnwYNzc31q1bh7W1dbYZ89qj+EQmrw5h85cevFO8KLtOXqGGVRkCpvbTPef7X47yNDGZKb0cAOjXtolu3YDFm3Fr3ZBPG9XSW+a8sHdvGPPnTaNhww85e/YCH39si1ar5cbNO0pHE1lo+XFn3a+rVn2Xs2cO0LRZewUTFaJpiqZNm7Jv3z7S0tJITk5mzJgxqFQqQkNDAYiMjOTRo0fUrFnzH7/n4cOHef78OSkpKYSEhOhG06+0aNGCQ4cOkZCQQGpqKqGhoa/NKcPLEbSjoyPdu3enTJkyRERE6H4o3LhxgytXXs5Zbdu27bXPeNP27t69G61Wy+PHj3F3d+f8+fO4u7vTtGlTJk2aRPXq1blx48Y/3t680KjGuwzsYMvAxQG4fr2WPaejWTykq14zKCEm5iE9XQfit2QOZ07vZ4GvD73cBis2TSQKnkIzMv7000+5ePEiLi4uaDQa+vXrh62tLTNmzGDp0qWYmJiwdOnSfzWHZ2lpyaBBg4iLi6Nr16588sknAHTt2pUVK1ZQq1Yt+vXrR69evShatCjvvPNOltMCPXv2xMvLiz179mBqakqDBg24e/cuAKVKlcLPz4/bt29jY2PD7Nmz/1G2Pn36MHv2bLp06QLAtGnTaNKkCQ0aNKBz586Ym5tTp06df1zuecm1VQNcWzXIdv2wzh9lu27V2F75kEg/fvvtBJ+06qJ0jHwzYOBYpSPki1u37lLaMutDSfVJpf37918BQGBgICdPnmTevHnZPufGjRscOnSIzz77DIBhw4bRs2dP7O3t/9Fn3L17l379+nHw4MG8iJxryQdWKB0hX1h0mqV0hHyR8YYpsoLMKItvl4VFasrdbNcVmpGxEipVqsSFCxfo3LkzKpWKjz/+GDs7O6VjCSEKIBkZCx0ZGRcsMjIueN40Mi40O/CEEKIgkzIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwAFLGQghhAKSMhRDCAEgZCyGEAZAyFkIIAyBlLIQQBkDKWAghDICUsRBCGAApYyGEMABSxkIIYQCkjIUQwgBIGQshhAGQMhZCCAMgZSyEEAZAylgIIQyA3JBU6JQsVk3pCPki9ugypSPki1K2Q5SOkC9MjArvTesTkm5ku05GxkIIYQCkjIUQwgBIGQshhAGQMhZCCAMgZSyEEAZAylgIIQyAlLEQQhgAKWMhhDAAUsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwANlexfnSpUtvfGHdunXzPIwQQrytsi1jT0/PbF+kUqk4cOBAvgQSQoi3UbZlfPDgQX3mEEKIt1qOc8aJiYnMnDmT/v378/TpU3x8fEhMTNRHNiGEeGvkWMazZ8+mRIkSPH78mCJFipCQkICPj48+sgkhxFsjxzK+cuUKY8eOxdjYGHNzcxYuXMiVK1f0kU0IId4aOd4T28goc19nZGS8tkyIf+L75b5cvnyVpUt+pGTJEiz7bh61bKphpDJiw4ZA/u+b5UpHzNHGPb8SEHoElUpF5fJl8BniimWpEmze+xuBB4+TkppGHevKfDXMDVOT//3zik9Iwm3yIsa6O/Fp8wbKbcB/MH/eNFy6dyLuyVMArl69jrvHcGVD5dIPyxdw+fJV/JasxMysCN/830waN6qHkZEREaciGTfGhxcvUvSaKcdWbdq0KQsWLODFixf8+uuveHp6Ymtrq49sBdqBAwdYsmSJ0jEMQi2b6gTv+hlnl066Zd7TxnLv3gOaN3WkTatuDBjYl2bNGiqYMmeXr99hbXAYa2ePJnDRJKpUKMO3m3ez/8R5Nu75lRXThhG4aBIpqWmsCwnXvU6r1TJ12XoSkl4oFz4XmjdvjIfHCJrZdqCZbYcCXcQ2NtUJ2bUel+7/+7s4cdJIjNXGNLftiG0zR8zNzPCaoP9tzHFk7OXlxYoVKyhRogSLFy/mk08+YfjwgvuHoS8ODg44ODgoHcMgDB7swfp1W7l7555u2cQJM1Gr1QBUqFCOIkVMeRb/XKmI/8j71Sqzc8lUTIzVpKSmEfvkGZXKWfLLoQj6dbajVPFiAHgP7klaerrudSu27aNWVSuS9DzSygumpqY0aFCXsWOGUK1aVf744yYTJn7Fnb/8WRYkg4f0Y926rZny//bbSW7fuotWq0Wr1XLu3GXq1Kmp92w5lrGJiQkjRoygf//+mJiYUKRIEX3kMlgPHjzAy8uLpKQkjIyM8Pb2JikpiXnz5qHVarGysmLRokWEhoZy8uRJ5s2bl+n1P/30E9u3b8fIyIh69eoxc+ZMAgMDCQ0N5dmzZzx+/Bg7OzsmT55MRkYGM2bM4Nq1azx69Ahra2uWLVuGmZkZa9asYePGjajVauzs7JgwYQKPHj3Cx8eHBw8eoFKpGD9+PB999JFCv1P/4zV+BgCt22TOkpGRwcpV39C1myO/BIdy7ep1BdL9OybGag6evMBXyzdjYmzM8F6OjPFdxZNnzxk2ZzkP457RqHY1xrg7AXD0XBSnr/zO91OHMnjmdwqn//esrMoTHn6UadPmcfXadcaNHcLWLauwbe6odLT/ZPy46QC0+cvfxYMHftX9unLlSowY+TmeI7/Ue7Ycpylu3ryJq6srtra2NG7cmH79+nH//n19ZDNIW7dupU2bNgQGBjJhwgROnjyJl5cX8+fPJzg4GBsbG7Zv357la9PT01m+fDnbtm0jMDAQlUpFTEwMABcvXmTp0qX88ssvnDt3jn379nH27FlMTEzYvHkz+/btIyUlhUOHDnH+/Hk2bNjA1q1b2blzJ5cuXeLixYvMmTOH7t27ExgYyPfff4+Pjw8JCQn6/O351wYNGId1lca8804pJk8ZpXScf8S+2YccWjWbYT3bM2zOD6RnZHDswlUWjO3PxnnjeJaQxLJNu7j/KI5Fa3fytac76gK6n+XmzTt07dafq9de/qD8ZvFyqlWrynvvVVY4Wd5r0PADQvdvZvkPa9mzW//nWeQ4Mvbx8aFHjx6sX78erVbL5s2b8fb2ZtWqVfrIZ3BatGiBp6cnV65coXXr1jRq1Ijdu3dTp04dAMaNGwdAYGDga681NjamYcOG9OjRAwcHB/r27Uv58uUBsLe3p0yZMgB07NiR48eP4+Pjg4WFBevXr+f69evcvHmTpKQkIiIisLOzo0SJEgCsWbMGgKNHj3L9+nX8/PyAl+V/584dXTZD4tD2Ey5djObBg1gSE5PYsiWYrl07KB3rjW4/eMijp89pVLsaAN3sbZm9cgvV3q2AfdMPKV7UDIBOrRqzfGso5Y9F8iI1leFzlv//1z/im3XBxMUn4tqupWLb8W988EFt6tV7nw0b/vf3WaVSkZaWpmCqvNejR2e++b9ZjB83nS0BOxXJkGMZx8fH4+rqqnvs4eHB1q1b8zWUIWvcuDEhISGEh4eza9eu106Aef78+RtPivnuu++IjIzk8OHDDBw4kIULFwLo5k8BNBoNarWaAwcO4OfnR79+/XBxcSEuLg6tVouxceY/tpiYGMzNzdFoNPj7+2NhYaFb/qrgDY2zSyecurRnzChvTE1NcXHpRNjB35SO9UaP4uKZtGQdAb5evFOyOLt+PU2NKhVxtrcl9Fgk3ds2p4iJCWEnL1K3ehX6O9nR38lO9/oBM5bh1uHjAnU0hUaj5ZtFX3H0aAQ3b95hyOB+XLh4hT//fKB0tDzTrZsjCxZOp2uXfpw9c0GxHDl+d6pSpQrnzp3TPY6KiqJKlSr5GsqQ+fr6EhQUhLOzMz4+Ply9epUnT57w+++/A/Djjz+ycePGLF/75MkTHB0dqVWrFqNHj6Zly5ZER0cDcPjwYZ4/f05KSgohISG0atWKY8eO4ejoSPfu3SlTpgwRERFkZGTQpEkTDh8+TGJiIunp6YwfP56LFy/SvHlzNmzYAMDvv/9Oly5dSE5O1s9vzL80dcocSpUswfGI3Rz6LYjIsxf47tuflI71Ro3qVGeQy6cM+OpbXCcsYM/Rsyye8AW92n9M8w9r0XvSN3QdM5ekFymM6t0p5zcsAC5fjmbsOB8Ct/3EuciDdO3ann79RiodK0/NmDkBlUrFt9/N4+jxEI4eD+GbxTP1nkOl1Wq1Wa1wcnq5AyIxMZGYmBhsbGwwMjIiKiqK6tWrExQUpNeghuL+/fuMHz+exMRE1Go1AwcOpEyZMsybN4+0tDSqVKmCr68ve/fu1e3AW7JkCeXKlaN3796sWbOGzZs3Y25uTsWKFZk/fz6hoaFs2rQJIyMj4uLi6Nq1K8OHDyc6OhovLy/UajWmpqaUL1+eatWqMXbsWNavX8+mTZvQaDR8+umnjBkzhpiYGHx8fLh37+WeYi8vL1q3bv2Pt61ksWr59dumqNijy5SOkC9K2Q5ROkK+MDHK8Qt7gZWQdCPbddmW8cmTJ9/4ps2aNctdKqETGBiY5ZEX+iZlXLBIGRc8byrjbLf6r2X79OlTkpOT0Wq1ZGRkcPv27bxNKIQQb7lsR8avLFmyhBUrVgAvdzKlpaVRo0YNgoOD9RJQ6I+MjAsWGRkXPG8aGee4Ay8oKIiwsDDat29PaGgo8+bNo0aNGnkaUAgh3nY5lnHp0qUpV64c1apVIyoqiq5du3Lr1i19ZBNCiLdGjmVsbGzM7du3qVatGqdOnSI9PZ34+Hh9ZBNCiLdGjmU8ZMgQpk2bRps2bQgNDaVNmzZy1TYhhMhjOe7A+6vk5GRu3bpF7dq18zOTUIjswCtYZAdewfOfDm2bPXv2G9/U29v7vycSQgiRSbZl/Or6BkIIIfJftmU8cmThOv9cCCEMWcG8yKoQQhQyUsZCCGEApIyFEMIAZDtnvGzZmw8HkjllIYTIO9mWcVxcHADXr1/nxo0btG3bFmNjYw4cOICNjY3eAgohxNsg2zKeNm0aAP369SMwMJDSpUsDMGzYMIYPH66fdEII8ZbIcc744cOHuiIGKFmyJI8fP87XUEII8bbJ8bxDGxsbpkyZQteuXdFqtWzdupX69evrI5sQQrw1crw2RUJCAn5+fhw7dgyVSsUnn3yCp6cnZmZm+soo9ESuTVGwyLUpCp7/dG2KV4oXL864ceO4efMmtWrVIjU1VYpYCCHyWI5zxpGRkbRt25ahQ4cSGxtL69atOXPmjD6yCSHEWyPHkbGvry9r1qzBy8uLChUq4Ovry5w5c9i2bZs+8gk9esesuNIR8sV7rccrHSFfJNw9pHSEfHH8g4lKR1BEjiPjFy9eZLrnXevWrcnIyMjXUEII8bb5R7ddevbsGSqVCnh5EogQQoi8leM0xdChQ3F3d+fRo0eMGzeOI0eOMHPmTH1kE0KIt0aOZWxvb0/16tU5cuQIGo2G4cOHZ5q2EEIIkXs5TlN8+eWXVK1alT59+uDu7k6NGjXw9PTURzYhhHhrZDsynj59OjExMZw+fZonT57olqelpXHjRvYHLgshhPj3si3jHj16cO3aNaKjo2nfvr1uuVqtpmHDhnoJJ4QQb4tsy/jDDz/kww8/5KOPPuLOnTs0bdqUp0+fcurUKSpXrqzPjEIIUejlOGe8ceNG/Pz8gJfHHK9YsYLvvvsu34MJIcTbJMcyPnDgAKtXrwagQoUK/Pzzz+zatSvfgwkhxNskxzJOS0vDxMRE99jExER3AogQQoi8keNxxo0aNWL8+PH06NEDlUrFjh075HrGQgiRx3IcGU+bNo0yZcowd+5cfH19sbS0ZOrUqfrIJoQQb40cR8ZFixZlypQp+sgihBBvrWzLePTo0SxZsgQnJ6cs1wcHB+dbKCGEeNtkW8aDBg0C/neXaCGEEPkn2zIuXbo09+7d491339VnHiGEeCtlW8adOnVCpVKh1Wp58eIFxYoVQ61WEx8fj6WlJb/99ps+cwohRKGWbRmfPXsWAB8fH2xtbenUqRPw8iSQ/fv36yedEEK8JXI8tO3ixYu6IgZwcHAgKioqX0MJIcTbJscy1mg0nDhxQvf48OHDcgaeEELksRyPM/b29mbMmDGYmJig1WrRarV8++23+sgmhBBvjRzLuEmTJoSFhXH16lVUKhW1atXC2DjHlwmh03+gG+6fu6LVwq2bd5g85isSEhKZ7fsl9Rp+gJGRisjTF/Ce+DUpL1KUjvuPdXd1YvioL9BqtSQnvcB70hzORV7Srf/q68lUq1YVD7dhCqb8ZzZs3cnm7SGoVCoqV6rIjMmjMVarmblgGdHX/sDc3IxuHT+lb8+uAFy4Es38JStITn6BRqPhC/eeOLW3V3grsldryQgSo+7w5/c7wciIGnMHUKrF+wA8OXCWG1+tpWitd7H5brTuNSq1EcXqVOXyFwt4vOtEdm+dZ3KcpkhMTNSdCm1lZcXMmTNJTEzM92D/xYkTJ/Dw8ABg6tSpXLhwIdvnxsTE6I6lzg8eHh6ZpneyM2jQIGJiYrJdP2XKFP78889/9FxD9EH9Ogwa0R+XDv1o97ELN/+4zfgpI/AcNwi1sTEdWvWg/Sc9KGJmxogxA5SO+49Vr/EePjMn0Lv7YNp+4sL/LfyBVev8dOu7dOtAD9esT5gyNJeirrFm4zZ+Xv4NO37+gSqVrVi2ci3zlyynaFEzgtYvZ8OKxfx2/BThR06g1WoZO3UOIwa4s83/W75fNJMFfiu4dedPpTflNeY1K/Hh1umUcfpIt6xcz1aYV7fidJvxnLH3olSL9ynj1IKkq3c523aC7r+4Q+eIDfxVL0UM/6CMZ8+eTYkSJXj8+DFFihQhISEBHx8ffWTLlTlz5vDhhx9mu758+fKsXLlSj4mytnLlSsqXL5/t+hMnXv7l/yfPNUQXz12hTVMnnj9PoEgRU8pXLEdc3DNOHD3N0kUr0Gq1aDQaLl2IolJlK6Xj/mOpqamMHzWN2JiHAJw7e5Fy5ctgYmJCzVrVGDF6AN/4FozrftetXZOQzasoUbwYKSmpxD58TKmSJbkc/TtO7R1Qq9WYmJjQ6qNm7Av7jdTUNIZ93pcWTV/e8adCubJYWJQkJvaRwlvyOqvPOxCzKYxHwUd1y1RGRhgVNcOoiDEqUxNUJsZoXqRmel1J2zqU6dyC3yeu0FvWHMv4ypUrjB07FmNjY8zNzVm4cCFXrlzJ8Y0fPHiAu7s7Li4u9OjRg8jISCIjI+nZsyddunShf//+3Lp1S/cZPXv2xMnJCXd3dx48eJDpve7evUuHDh3o3bs3n332GRkZGcydOxdnZ2e6dOnCmjVrXvv8v45MFy1aRLt27ejVqxcjR44kMDCQu3fvYm//8mvVo0ePGDJkCE5OTjg7O3P48GEAli5dire3Nx4eHtjb2/P999+/9jlarZYFCxbQvn17OnbsiL+/v27dli1bcHFxwcHBgYMHDwIwefJkhg4diqOjIwcPHsTe3p67d+8SFRWFq6srLi4u9O7dm5s3b7JixQpiY2MZPHgwcXFxuucmJCQwatQoevXqhZ2dHRMmTECr1XLixAm++OILhg8fTvv27Rk1ahSpqamvZda39PR02nW04/iFfdh+1IgtG3bwa/gxbvzx8s+/0rsVGTC0LyFBoQon/efu3L7H/tBDuscz5kwidHcYJqYmLFs+n9HDvyQhwTC/QWbFxNiYA4eP4uDswenIizh3+pQP69oQvPcAaenpJCUlsy/8CA8fP6FIEVO6O/3vVmxbgnaRlPyCeh/UVnALsvbHl6uI3Xo407KYzeGkP02g2dkV2J5fyYsbD3iy73Sm51hP78fNuRvISEjWW9Ycy9jIKPNTMjIyXluWla1bt9KmTRsCAwOZMGECERERjBs3jmnTprFz507c3NwYN24cAF5eXgwfPpzg4ODXCu2VGzdusGDBAtasWUNAQAAA27dvZ+vWrRw4cIBTp05lmePgwYOcPn2aX375hRUrVnD58uXXnjNr1iyaN29OcHAwfn5+fPnllzx69PKnfHR0NKtWrWLLli2sWLGC+Pj4TK/ds2cPZ86cITg4mC1bthAYGMjDhy9HSyVLliQwMBBvb+9MOz0tLCzYvXu37ocBgL+/P59//jmBgYF4eHgQGRnJ4MGDKVeuHCtWrOCdd97RPTc8PJw6deqwefNm9u7dS2RkJJcuvZyrPHv2LD4+PuzevZt79+4ZzMk5obvCaFirNYvn/8C6LT/ojsj5oH4dtoSswf/HTRwMPZzDuxieokXNWblmMdbVqjJu1DQWL53NqhU/E3XlmtLR/jWHVh/x267NDB/QlyHjvPEaMRAVKnp+NpJRU2bRomlDTEwy7y/6cV0A3676mWXzZ2BWpIhCyf+dql49SXscz4kPB3Ky4RCM3ylOpaH/m1Iq0cQGk9IleBio3387Oe6Ja9q0KQsWLODFixf8+uuvrF+/Hltb2xzfuEWLFnh6enLlyhVat25N69atCQkJoV69egA4Ojri4+PDn3/+ycOHD7GzswOgT58+Wb6fpaWl7tTsY8eOceXKFY4fPw5AUlIS0dHR1KhR47XXHT16FEdHR0xNTTE1NaVt27avPef48ePMnj0bgMqVK1O/fn3OnTsHgK2tLaamplhaWmJhYcHz588pWbKk7rURERGZ3j8oKEi37tVn1ahRg7i4ON3yV78Hf9W6dWtmzpzJr7/+ip2dXaabwP5d586dOX/+PGvWrOH69es8ffqUpKQkAGrWrEmFChUAqF69Os+ePcv2ffShqnVlypYrw6kTL08iCli/na8XeVPKoiSftGnB7AVT8Zk0l6BtBe/uMZXercjaTd9xLfo63Z36805pC2xbNKZ6zfcYPLw/FhalKFmyBOsDltPXdYjScbN1++49Hj1+QqP6HwDg3KkdMxcsIzEpifEjBlCqZAkAVv0cQJVKL6eSUlNTmTrnG/64eZv1yxdTqWLBmT6z7GjLH1NXoU1LJyMtnZiAcMp0bsGfP7y8+FnZrh8Ru+UQ/P/pQX3JcYjr5eVF0aJFKVGiBIsXL8bGxoaJEyfm+MaNGzcmJCSEjz/+mF27djFr1qzXnqPNYmNTUlK4c+fOa8vNzMx0v87IyGDChAkEBQURFBTE5s2b6d69e5Y5jIyM0Gg0b8z69xxarZaMjAwAivzlp/2r08P/6u9Hlty9e1dXjGq1Wve67LbllQ4dOrB9+3bq1auHv78/06dPzzbvunXr8PX1pXTp0ri7u1O9enVdrpzy6lu58mVZ9qMv75S2AKBbz05EX/mdjz5uxoy5k3HvMaRAFrGFRSm2h6xlV/A+hg4Yz4sXKdy/F0ODOq1p+4kLbT9xYcHcpZw4dtqgixjg4aMnTJg+j7inL39w/xIaRo1qVQnYsZtlK9cB8OhJHFt37qFjuzYAjPP+moTEJH7+4ZsCVcQACRduULbLyx16KmM1lu2a8vz0Vd36Ui3e5+mv2e/8zy85lrGfnx8jRozQfQUfO3Zspn/w2fH19SUoKAhnZ2d8fHyIiori6dOnnD9/HoBdu3ZhZWVFpUqVqFChAkeOHAEgKCiIJUuWvPG9mzdvTkBAAGlpaSQmJtKnTx/dSPbvWrZsSWhoKKmpqSQkJBAeHv5aOTZv3pytW7cCcOfOHc6cOUODBg1y3EZ4+c1h3759pKWlkZyczMCBA//TEQ9jxozh/PnzuLm5MXr0aN10ilqt1v1geOXIkSP06tWLLl26oFKpiIqKyvEHjlIijp9h2Tcr2bxzNbvCA+ji3IHBHmOYOG0UKhXM/78Z7AoPYFd4ALN8v1Q67j/Wf4Abld6tiGPntuz/NVD33zvvWCgd7V9r3OADBvV34/ORk+jefwR79h/Cb64PgzxciXn4iG7uQxngOZnhA9z5sI4NZ85fIvzICe7cvYfH0PF07z+C7v1HcOTE6Zw/zABc9/kJdYmiNP51CQ0PLCTl/mPuLtuhW29erSIv7sTqPVeO0xTh4eGMHz/+X7+xh4cH48ePZ/v27ajVar766isqVqzIrFmzSE5OplSpUixevBiABQsWMGPGDHx9fXnnnXfw9fUFoGvXrqxY8freTDc3N27duoWzszPp6em4uLhga2ub5aFkrVu35syZMzg7O1OqVCnKlSv32g+TqVOn4uPjQ2BgIPDyCJJy5cplu20xMTEMHjyYoKAgPv30Uy5evIiLiwsajYZ+/fphbW39r3+/hg4dytSpU/nuu+9Qq9VMnjwZgDZt2jB48GB+/PFH3XP79+/PjBkzWL16NcWKFaNhw4bcvXuXKlWq/OvP1Yeffwrg558CMi1r06xgHPaVnSWLlrNk0fI3Pmfzhh1s3rBDP4Fyyc25M27OnV9b7jfv9SOnGtWry8Uju/URK89cHf2/fTbpcQlED89+wHe0mrs+Ir1Gpc3he+ywYcNISUmhUaNGFCtWTLf8888/z/dweeHs2bPcvHkTZ2dn0tLS6NWrF19//TW1axvenl+lVbV8fS67MEjJSFM6Qr6483uI0hHyxfEPcp4GLag+ebA123U5jowtLCwAdCceFDTW1tYsW7aMn376Ca1WS7du3aSIhRAGJ8eR8SvPnj1DrVZTvHjx/M4kFCIj44JFRsYFz5tGxjnuwLt+/Trdu3fno48+olmzZri7u3Pv3r08DSiEEG+7HMt4ypQp9OzZU3cGXfv27Zk6dao+sgkhxFsjxzJOTk7Gzc0NExMTTE1N8fDw0J2dJoQQIm/kWMaVK1fmzJkzusdXr16Vm5QKIUQey/FoipiYGDw8PLCxscHY2JjLly9TtmxZnJxeHicaHByc7yGFEKKwy7GM/8mpz0IIIXInxzJu1qyZPnIIIcRbLedrYQohhMh3UsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwAFLGQghhAKSMhRDCAEgZCyGEAfjH98AThZ+xaSWlIwhB3dJVlY6Qb849OJrtOhkZCyGEAZAyFkIIAyBlLIQQBkDKWAghDICUsRBCGAApYyGEMABSxkIIYQCkjIUQwgBIGQshhAGQMhZCCAMgZSyEEAZAylgIIQyAlLEQQhgAKWMhhDAAUsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwAFLGQq86Ojpw5vQ+Ll08zKaNyylRorjSkfJMYd22wrhddh1aceTaPt3joV4D2H54A9vCf2aWnzemRUz1nsngy/j58+cMHz78jc+xsbEBYOnSpSxdujRPPvfVe27cuJGNGze+8bmDBg0iJiYmTz43J1OnTuXChQt6+ay8VqZMaX5c+Q2uvQZT94NW3Lhxi6/nfKl0rDxRWLetMG5XFet3GTd9JEZGKgCafNSQDl3b4tbuM7q3cad48WL0HtBD77kMvoyfPXtGVFSUYp/fu3dvevfu/cbnrFy5kvLly+slz5w5c/jwww/18ll57dNPW3Pq1Dl+//0GAD8sX0uf3s4Kp8obhXXbCtt2mZkX4etvp7Nwup9umZGREaZmphQxK4KxiTGmZqakpqTqPZux3j/xX5o9ezaxsbGMGDGCGjVqcOzYMZ49e8Y777zD0qVLKVu27GuvycjIYOzYsbz77rtMnDgx0zp7e3vq1avHlStX2LBhA7/++iv+/v5oNBrq1q3L9OnTKVKkiO75r0banp6e7Nq1Cz8/P8zNzXn//ffJyMhg3rx52Nvbs3btWqysrPj66685duwYKpWKLl26MHjwYE6cOMHy5csxMzPjjz/+wMbGhoULF2JqmvmrUMuWLbGzs+PUqVOULVuWPn36sG7dOh48eMC8efNo1qwZHh4ejBw5kqpVq+Ll5UVSUhJGRkZ4e3vToEEDjh49yrx589BqtVhZWbFo0SKKFzeMr5WV37Xizt17usd3796nVKmSlChRnOfPExRMlnuFddsK23ZN853E1nU7uHbld92yk7+d5vihCPae3k5aajo3/7jNlrU79J7N4EfG3t7elCtXjokTJ3L9+nU2bdrE3r17qVKlCsHBwa89X6vV4u3tTYUKFV4r4ldatWrF3r17efLkCQEBAWzatImgoCAsLS1ZtWpVlq958uQJX3/9Nf7+/mzbto1nz5699pyNGzdy//59du7cyZYtWwgNDSU8PByAs2fP4uPjw+7du7l37x6//fbba69/9OgRbdq0Yc+ePQDs37+fDRs24Onpib+/f6bnbt26lTZt2hAYGMiECRM4ffo0qampeHl5MX/+fIKDg7GxsWH79u1v/P3VJyOjrP+6ZWRk6DlJ3ius21aYtsv1MxcyMjLYsTEk0/JuvTtRqUpFHOp1waG+E3/evofXDE+95zP4kfErVatWZdKkSWzZsoUbN24QGRlJlSpVXnvepk2beP78OQcOHMj2verXrw/AiRMnuHXrFq6urgCkpaXx/vvvZ/maU6dO0bBhQ910RLdu3di/f3+m55w4cQJnZ2fUajXm5uY4OTlx7Ngx7O3tqVmzJhUqVACgevXqWZY5vPxBAVCpUiUaN24MgJWVFfHx8Zme16JFCzw9Pbly5QqtW7fG3d2d6OhoypcvT506dQAYN25ctr8HSrh950+aNWuoe1ypUgWePIkjKSlZwVR5o7BuW2Harq69OmJmbsbm/WswMTWhiFkRNu9fQ/zT5/yydQ9JiUkAbPs5iClfj9d7PoMfGb9y8eJFBgwYgEajoX379rRt2xatVvva8xo2bMjQoUOZPXt2tu/1ahoiIyMDR0dHgoKCCAoKYsuWLfj4+GT5GiMjIzQazRsz/n29VqvVjSD+OvWhUqmyzA5kmrpQq9XZflbjxo0JCQnh448/ZteuXQwdOhQTE5NMz3n+/DkPHjx4Y2Z92rfvELbNGlGjhjUAQwZ7sDM4VOFUeaOwblth2q6+jgPp3sadXm0/Y2Tf8aS8SKFX2884e/IcDh1b6/69OXRsw/nTF/Wez+DL2NjYmPT0dCIiImjWrBm9e/emRo0aHDlyJMuvSrVr12bQoEFcu3aNsLCwN763ra0t+/bt4/Hjx2i1WmbMmPHadMArjRo14sKFC8TGxqLVatm1axcqlSrTc5o3b86OHTvIyMggOTmZ4OBgbG1t//vGv4Gvry9BQUE4Ozvj4+PD5cuXsba25smTJ/z++8v5sB9//DHHI0H06eHDxwwcNI7Nm1Zw4Xw4H9Stw4SJM5WOlScK67YV1u36qx+XrOXBvVi2H17P1rB1lLIoyaIZeXNU1r9h8NMUlpaWWFlZcfDgQV68eIGTkxMmJibY2Nhw9+7dLF9jamrKjBkzmDx5Ms2aNePrr7/G3t4eBweHTM+rXbs2I0eOpH///mg0GurUqcPgwYOzfM/SpUvj7e3NF198gampKe+++y4lS5bM9JxevXpx8+ZNunbtSlpaGl26dOHTTz/lxIkT2W7foEGDGDVq1L8+QsLDw4Px48ezfft21Gq1bsfjggULmDhxImlpaVSpUgVfX99/9b75bfeeg+zec1DpGPmisG5bYdyue3ce0KJ6WwBSU1L5evJChROBSpvd92WRSVxcHOvWrWPkyJEYGRkxe/ZsqlatioeHh9LR8oyxaSWlIwhB3dJVlY6Qb849OJrtOoMfGRsKCwsL4uPj6dy5M2q1mrp16+p2/AkhRG7JyFjoyMhYGIK3dWRs8DvwhBDibSBlLIQQBkDKWAghDICUsRBCGAApYyGEMABSxkIIYQCkjIUQwgBIGQshhAGQMhZCCAMgZSyEEAZAylgIIQyAlLEQQhgAKWMhhDAAUsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYALkhqRBCGAAZGQshhAGQMhZCCAMgZSyEEAZAylgIIQyAlLEQQhgAKWMhhDAAUsZCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpY6E1qaioAt27dIjw8HI1Go3CivJOUlERUVBRarZakpCSl4+SJ9PR0wsPD2bFjR6b/CrrExETWr18PQExMDEuWLCE5OVnhVFLGQk+WLVuGt7c39+7do2/fvqxZswYfHx+lY+WJY8eO0bVrV4YPH87Dhw+xt7fnt99+UzpWro0fP55vv/2W48ePc+LECd1/Bd348eOJjY0FoFixYmg0GiZOnKhwKkArhB44OztrU1JStMuXL9fOnz9ft6ww6NGjhzY2NlbbtWtXrVar1V67dk3r5OSkbKg80L59e6Uj5Ius/my6dOmiQJLMZGQs9EKj0WBqakpYWBitW7dGo9EYxFfDvKDRaChbtqzucY0aNRRMk3eqV6+uG0EWJiqViujoaN3jP/74A2NjYwUTvaR8AvFWaNGiBZ07d8bMzIymTZvi7u6Ovb290rHyRIUKFQgLC0OlUhEfH8/69euxsrJSOlauvXjxgg4dOlCrVi1MTU11y9euXatgqtybNGkSX3zxBeXLlwcgLi6OBQsWKJxKLi4v9OjevXuUL18ejUbD+fPnady4sdKR8sTjx4+ZM2cOR48eRavVYmtri7e3N+XKlVM6Wq6cPHkyy+XNmjXTc5K8l5qaytWrVzE2NqZatWqZftgoRaYphF7s2rWLIUOGoFaruX//PmPHjmX//v1Kx8oTlpaWDBw4kOPHj7N//3569+5d4IsYXpZu8eLFMTIyQqVSodFouH37ttKxcu38+fOsX7+eWrVq4evrS6tWrdi7d6/SsaSMhX58//33/PTTTwBUqVKF7du3s3TpUoVT5Y2FCxeycOFCAJKTk/nuu+8KxbZNmjSJMWPGMGLECL755huGDRvG7t27lY6Va7Nnz6Zu3brs3bsXMzMzAgMDWbFihdKxpIyFfqSlpVGmTBndY0tLSwrLDFl4eDgrV64EoFy5cvz000+EhoYqnCr3IiIiCAkJoX379syaNYuAgADdseIFmUajoVmzZoSHh9OuXTusrKzIyMhQOpaUsdCPxo0bM27cOMLCwggLC2PixIk0aNBA6Vh5Ij09nRcvXugep6WlKZgm75QrVw4TExOqV69OdHQ0NWvWJDExUelYuWZubs7q1as5fvw4dnZ2+Pv7U6xYMaVjydEUQj+mT5/OunXr2Lx5M8bGxjRp0oQ+ffooHStPuLm54eLiojs65PDhw4Vi28qXL8/y5ctp0aKF7miDwnB24cKFC9myZQtLly6lVKlSxMbG8s033ygdS46mEPnr4cOHlC1blnv37mW5vjAcAgYvdwqdOnVK94Pm/fffVzpSriUkJHDo0CE6derEunXrOHr0KP3796d58+ZKR/tPLl26RN26dYmIiMhyfdOmTfWcKDMpY5GvhgwZwvLly7G3t0elUqHVajP9/8CBA0pHzLXU1FQOHTqk+wqfkZHB3bt3GT16tMLJ/pvC+gN02rRpzJo1Cw8Pj9fWqVQqxY+fljIWIpcGDx5McnIyt2/fpkmTJkRERNCgQQP8/PyUjvafvA0/QA2RlLHQizt37rBp0ybi4uIyHUUxd+5cBVPljU8//ZTQ0FDmzJlD9+7dKV26NKNHj2bTpk1KR8uVp0+fYmFhkWnZ3bt3effdd5UJlEdOnTqFv78/z549y7Rc6ZGx7MATeuHp6UmLFi1o0qQJKpVK6Th5ytLSEpVKhbW1NdHR0XTr1q1AHwJ2//59tFotgwcPZuXKlbofnhkZGQwaNIg9e/YonDB3Jk+ezMiRIw1uukXKWOiFVqtl0qRJSsfIFzVr1mTWrFn07t0bLy8vYmNjC/ThbX5+fpw4cYLY2Fj69u2rW25sbEybNm2UC5ZHypcvT7du3ZSO8RqZphB6MWPGDFq2bImDgwNGRoXr8PaMjAzOnj1LkyZNOHjwIEePHsXV1ZVatWopHS1XVqxYweDBg5WOkef27NnD/v37ad68eaartSld0FLGIl/Vrl1bt/Pn71QqFVeuXFEgVd7bv38/x48fR61W06pVK1q2bKl0pFy7desW586dw8nJienTp3Pp0iWmTJlCkyZNlI6WK/369QOgUqVKmZYrvf9CyliIXJo3bx6RkZF06tQJjUZDSEgIDg4ODBkyROloudK3b1/c3d0xMTHB39+f0aNH4+vrS0BAgNLRcsXR0dEgr7Ehc8ZCL+Lj41m6dCnHjx/H2NiYVq1aMWzYMMzMzJSOlmthYWGEhITovvK6ubnRrVu3Al/GKSkpODo6MnXqVJycnGjSpAnp6elKx8q1Jk2aEBYWxieffGIQF5V/pXBN3gmDNWHCBIyNjVm4cCFz584lKSmJqVOnKh0rT1haWhIfH697nJaWxjvvvKNgoryhVqvZu3cv4eHhtGnThv379xeK+f6wsDCGDRvGhx9+SJ06dahduzZ16tRROpZMUwj96Ny5M7/88kuOywqiYcOGcfHiRezt7TE2Nubw4cNYWlpibW0NKD8X+V9FR0ezZs0a2rRpQ/v27Rk7dixDhgyhdu3aSkcrlAxnjC4KtapVq3Lq1Cndzp+oqCiqVq2qcKq80a5dO9q1a6d7/MEHHyiYJvdenQ5dokQJPD09gZd3aZkwYYLCyfLPq+tWKElGxkIvnJycuHbtGtbW1qjVam7cuEGpUqUwMzMrFKfYxsbGUq5cOU6dOkV0dDTOzs4ULVpU6Vj/SVanQ79SGP6ssjJ48GDFLzAvZSz04s8//3zj+r8fZlSQTJ8+HSMjI/r27cuAAQNo2bIlz58/L/B3+wgLC8POzk7pGG8NKWOhF4XtymZ/5eLiwrZt21i2bBnw8tRvFxcXAgMDFU6WO506dSIkJETpGHkuPT2d3377jadPn2ZarvRJHzJnLPRi5MiRWV7ZrDDIyMhAo9Fw4MABvvrqK5KTkzPd+aOgqly5MlOmTKF+/fqZDkFUurRya/z48dy7d4/q1atnuk6K0tslZSz04saNG5mubDZx4sRCMSqGl/+IP/74Yxo1akT9+vVxdHTEzc1N6Vi59urwvHPnzmVarnRp5VZ0dLRBXuxIyljoRWG7stlfff755/Tr1w+1Wg3A+vXrKV26tMKpcq+gHpKXk+rVq+t2uBoSKWOhF4XtymZ/96qIAUqXLs3mzZvp1auXgolyLzw8nG+//fa1a1AX9KMpXrx4QYcOHahVqxampqa65XI9Y/FWmD59OpGRkdSoUQNPT0+OHTvGokWLlI6Vb2JiYpSOkGtz5sxh6tSp1KhRo1Bdg9pQT1OXoymEXjg7O7N9+3alY4h/oTAcEZKdQ4cOcfz4cdLT07G1taVt27ZKR5KRsdAPS0tLTp06Rb169TJ9NSzIli5diqenJ1OmTMlyfUGdc3119+Tq1asze/ZsHBwcMl1QR+m7KOfWypUrCQ0NxcnJCa1Wyw8//MDvv//O0KFDFc0lI2OhF82bN3/tuM6Cfj3jgwcPYm9vn+2I39nZWc+J8kZWd09+xRDuopxbTk5ObNmyRXe4XnJyMi4uLopfVlNGxkIvjh8/rnSEPGdvbw+8vCFpUFAQffv2JSYmhk2bNhXoO2SsW7dO9+vHjx9jaWlJcnIysbGxheJ6IlqtNtNx00WKFDGIS2kqn0C8FV6dnfaKSqXCzMyM6tWrF/j7qnl5eWFjYwNAsWLF0Gg0TJw4scCfDr1u3ToCAwPZvn07T548YejQoXz22WcF/iiR5s2b4+npqfvmsmPHDmxtbRVOJdMUQk8mTpzIrVu36NSpEwChoaEUL14cIyMj3nvvPSZOnKhwwv+uS5cu7Ny5M9Oyrl27EhQUpFCivNG5c2cCAgJ0FzxKTk7G1dWV4OBghZPljlarZePGjRw/fhytVkvz5s3p1auX4qNjGRkLvbhx4wbr16/X7bxzc3PDw8ODzZs306VLlwJdxiqViujoaN3o+I8//lD8H3ZeSEtLy7Sz1cTERME0uffq0qD379+nTZs2mb6RxcbGYmVlpVw4pIyFnsTHx5Oenq77x52WlkZSUhJAljcrLUgmTZrEF198Qfny5QGIi4tjwYIFCqfKvbZt29K/f38cHR2Bl99mHBwcFE7133l7e7N8+XLc3d11lwb96/+VPplFpimEXqxdu5aNGzfSpk0btFothw4dwt3dnbS0NC5cuFCgTwA5f/48J06coFGjRixZsoTo6GhmzpxJ+/btlY6Wa3v27CEiIgJjY2OaNm1qEMfjFlZSxkJvoqOjOXbsGEZGRrRo0YKaNWty8+ZNrKysCvSxx66urkyYMIEHDx6we/dupk2bxsiRI9m2bZvS0UQW/n5c+F93Jvfs2VOxv4sF/+6CosCwsbHhs88+o1+/ftSsWZOHDx/y3nvvFegiBtBoNDRt2pTw8HDatWtHxYoVycjIUDpWviiox07/lVqtJiEhgbZt29K2bVtSUlJ4/PgxN27cYPr06YrlkjIWihk0aJDSEfKEubk5q1ev5sSJE9jZ2eHv70+xYsWUjpUvli9frnSEXLt8+TJLly7FwcEBBwcHFi5cyJ9//om3tzeXLl1SLJeUsVDMjh07lI6QJxYuXEhSUhJ+fn6UKlWK2NjYAj0H/lcJCQncv3+fe/fuce/ePdLT05WOlGvJyck8fPhQ9/jx48ekpKQAKPqNRuaMhd7s37+f48ePo1aradWqFS1btlQ6kniDH374gRUrVmBhYaFbZghHHeTWrl27mDt3Lg0bNkSj0XDx4kWmTp1KVFQU8fHxTJ06VZFcUsZCL+bPn8/Zs2fp1KkTGo2GkJAQHBwcDPZyhuLloW0BAQGF4kL5f/fkyRNOnz6NkZERDRs2pHTp0jx9+jTTDx59k+OMhV4cPHiQkJAQ3ckQbm5udOvWTcrYgFWsWJFSpUopHSPPvLrg/99PzX91saqRI0cqEUtHyljohaWlJfHx8bpRVlpamu4ea8Iwvffee/Tp0wdbW9tMR7woXVr/laFPAsg0hdCLYcOGcfHiRezt7TE2Nubw4cNYWlpibW0NFNxr/xZmfx9BvlJQy9jQSRkLvcjpLh+F4fjVwujJkyecO3eOjIwMGjRoQJkyZZSO9J/Vrl07y9tHvTodWulra0sZC71JSEggPj4+0zKlL84isvfrr7/y5Zdf0qBBAzQaDWfPnmXOnDnY2dkpHa1QkjljoRfz588nICBAt7faUC7OIrK3ePFiNmzYQOXKlQG4c+cOI0eOLPBlnJqayurVq7lx4wbTpk1jzZo1DB48WPEzQaWMhV4cOHCAw4cPF9oz0wqj9PR0XREDVK5cGY1Go2CivDFz5kxKly7NpUuXUKvV3L59m6lTpyp+pT05A0/ohY2NDampqUrHEP+ClZUVa9asISEhgYSEBNasWUOlSpWUjpVrly5dYty4cRgbG2Nubs78+fMVny8GGRkLPenatSvt2rWjVq1aqNVq3fKCfnPLwmzOnDnMmjWLH374QXdHjJkzZyodK9dUKhWpqam6nXlxcXFZ7tjTN9mBJ/TC3t6eUaNGvbbDrlmzZgolEm+rHTt2sGXLFm7duoWjoyP79+9nxIgR9OjRQ9FcUsZCLwrDPeHeFkOGDGH58uXY29tnOWIs6Dtdnzx5wpMnTzhx4gQZGRk0a9aM2rVrKx1Lyljox8yZM3n48CGtWrXKdC+1bt26KRdKZCk2NpZy5crx559/Zrm+oM8bOzo6snv3bqVjvEbmjIVeJCcnU7x4cc6cOZNpuZSx4SlXrhwAERERr60zMzMjMTGRWrVq6TtWnqlduzY7duygXr16mJmZ6ZYrfcy7jIyF3qSlpXHjxg0yMjKoWbNmobiDcmHm6enJ5cuXdfe9Cw8Pp1y5ciQlJeHk5MRnn32mbMD/yN7e/rVlhnDMu5Sx0IuLFy8yatQoLCws0Gg0PHr0iG+//Zb69esrHU1kw83NjRUrVlCyZEng5RmUQ4cOZc2aNbi4uLBz506FExYuMjQRejF79mwWL16sK9/IyEhmzZrF1q1bFU4mshMXF5fpJJ0iRYrw7NkzjI2NDeJQsLwUFham+JmFUsZCL5KSkjKNghs0aKC71Y0wTO3ataN///44Ojqi0WgIDQ3FwcGBHTt2ULZsWaXj5akDBw4oXsYyTSH0wsPDg/79++vmH/fv34+/vz/r1q1TOJl4k7CwMI4cOYJareajjz6idevWREZGYm1tXaguPJ+cnIy5ubmiGaSMhV7cvHmTIUOG8PTpU92yTZs26a5nLAzHpUuXqFu3bpZHUwA0bdpUz4ny1sKFC/Hy8tI9Dg8PZ+bMmRw8eFDBVDJNIfTk8OHDmJubs337dm7fvs3YsWM5efKklLEB2rhxI7Nnz8bPz++1dSqVqsCfwn779m3mzZvHwIEDmTVrFr///jvz5s1TOpaMjIV+dO7cmS1btui+CiYnJ+Pq6kpwcLDCycTbRqPR4O3tTUhICEOHDmXgwIGZTkRSioyMhV6kpaVl+gtvCH/5RdY8PDzeeLREQR0Z//U2UhUrVqR48eJcvnyZ5cuXA8rfTkrKWOhF27ZtdXvmAd2eeWF4PD09lY6Q71QqFb1791Y6RiYyTSH0Zs+ePURERGBsbEzTpk11R1YIw3X69GmuXr1K9+7dOXfuXIHfeWfIpIyFEFny9/dn//79xMbGsmnTJvr06UOPHj0YMGCA0tH+E7khqRCiQOrWrRsBAQG4urqyY8cOEhMT6dmzJ7t27VI6WqEkc8ZCiCwZGRlluklnkSJFMt2lpaB6/PgxwcHBJCYmotVq0Wg03L17F19fX0VzyT3whBBZatasGfPnzyc5OZn9+/czbNgwmjdvrnSsXBs5ciRXrlxh586dJCcnc/DgQYyMlK9CmaYQQmRJo9EQEBDA0aNH0Wg0NG/eHDc3twJ/6dMOHTqwZ88e5s+fT4cOHahWrRqfffYZ27ZtUzRXwf5dFULkm4EDB7J69Wrc3NyUjpKnXl1Tw9ramqioKOrXr09aWprCqWSaQgiRjRcvXnD//n2lY+S55s2bM2rUKFq2bMnq1avx8fFR/CJBICNjIUQ24uLisLe3x9LSkiJFiugOAVP6jhi5df36dSZOnEilSpX45ptviIiI4NatW0rHkjIWQmTtxx9/VDpCnhoxYgRRUVHExsZy+fJl3fL09HTF738HsgNPCPEvPHz4sMBeWD4hIYGnT58yZ84cvL29dcuNjY2xtLRUfMeklLEQ4h/r1q0bO3bsUDpGoSRlLIQQBkCOphBCZCkmJoYFCxYAcOfOHSZOnMjjx48VTlV4SRkLIbLk5eVF5cqVAShfvjxNmjRhwoQJCqcqvKSMhRBZevbsme6ED1NTU1xdXYmLi1M4VeElZSyEyJKZmRmHDh3SPT527JhBnBxRWMkOPCFElq5cucKECRN4+PAh8PJWRb6+vtSqVUvhZIWTlLEQ4o3u3buHSqWiYsWKSkcp1OQMPCFElm7fvs24ceO4c+cOWq0WKysrFi9ejLW1tdLRCiUZGQshsvT555/Tq1cvOnToAMCuXbvYuHEj69atUzhZ4SQ78IQQWYqLi9MVMUDHjh15+vSpcoEKOSljIUSWTE1NuXTpku7xhQsX5GiKfCTTFEKILJ07d46xY8diYWEBwNOnT1m8eDH169dXNlghJSNjIUSWVCoVffr0Yf78+RQvXpyEhAQePHigdKxCS8pYCJGl2bNnU79+faKioihevDhBQUGsWLFC6ViFlpSxECJLGo2Gpk2bEh4eTrt27ahYsSIZGRlKxyq0pIyFEFkyNzdn9erVnDhxAjs7O/z9/SlWrJjSsQotKWMhRJYWLlxIUlISfn5+lCpVitjYWBYtWqR0rEJLjqYQQggDICNjIYQwAFLGQghhAKSMhchHX3zxBU+ePMm397exscnx/T08PNizZ8+/et/AwECGDBmSm2jiX5IyFiIfHTlyROkIooCQMhYin0yZMgWA/v37c//+fezt7RkzZgyOjo7s27cPe3t7Lly4oHv+Xx+fOXOGPn364OzsjIuLC2FhYW/8rKSkJCZOnIirqyvt27fHxcWF69ev69bv27cPFxcXOnbsyPfff69b/m8/R+QfuZ6xEPlk7ty5BAYG4u/vT+nSpQGoWbMm//d//6dbn5Vnz54xZcoUVq1axbvvvktMTAyurq7Y2NhgZWWV5WsOHz5MyZIlCQgIAMDHx4f169czbdo0ABITEwkICODFixf07NmT999/nwYNGmT7OUL/pIyF0KMmTZrk+JzIyEgePnzIiBEjdMtUKhXR0dHZlnGHDh2oXLky69at49atW5w8eZKGDRvq1vfo0QNjY2OKFy9O+/btOXr0KEC2nyP0T8pYCD0qWrRopsd/Pcw/NTUVgIyMDKpXr86WLVt062JiYnSj66xs2LCBgIAA+vbti5OTExYWFty9e1e3Xq1WZ/pMY2PjN35OcHDwf99I8Z/InLEQ+UitVpOenp7lutKlS3Px4kXgf6NhgAYNGnDr1i0iIiKAlzcGbd++PbGxsdl+zm+//YazszM9e/bE2tqagwcPZrqOxI4dO9BqtTx79ozdu3fTqlWr//Q5Iv/IyFiIfPTpp5/Sp08fvvvuu9fWeXl5MWPGDDZv3kzdunWpW7cu8LKk/fz88PX1JSUlBa1Wi6+vL5UqVcr2c7744gt8fHwIDAxErVZTt25drl69qltfokQJXFxcePHiBe7u7tja2gL8688R+UdOhxZCCAMg0xRCCGEApIyFEMIASBkLIYQBkDIWQggDIGUshBAGQMpYCCEMgJSxEEIYACljIYQwAP8PMaYELMM8218AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get an idea of the performance using the confusion matrix\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns; sns.set()\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "matrix = confusion_matrix(test.target, labels)\n",
    "sns.heatmap(matrix.T, square=True, annot=True, fmt='d', cbar=False,\n",
    "           xticklabels=train.target_names, yticklabels=train.target_names)\n",
    "plt.xlabel('true label')\n",
    "plt.ylabel('predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dddf9a06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        precision    recall  f1-score   support\n",
      "\n",
      "         comp.graphics      0.969     0.884     0.925       389\n",
      "             sci.space      0.924     0.924     0.924       394\n",
      "soc.religion.christian      0.617     0.985     0.759       398\n",
      "    talk.religion.misc      1.000     0.191     0.321       251\n",
      "\n",
      "              accuracy                          0.802      1432\n",
      "             macro avg      0.878     0.746     0.732      1432\n",
      "          weighted avg      0.864     0.802     0.773      1432\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(\n",
    "    test.target,\n",
    "    labels,  \n",
    "    target_names=train.target_names,\n",
    "    digits=3\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7e19d49",
   "metadata": {},
   "source": [
    "*It seems that we succesfully separated space talk from computer talk, but there is a confusion between religion talk and christianity talk, which is probably expected. There are very similar areas.*"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "348416c8",
   "metadata": {},
   "source": [
    "## Let's remember some basics to explain the above confusion matrix\n",
    "\n",
    "### Precision (P) = TP / (TP+FP)\n",
    "### Recall (R) = TP / (TP+FN)\n",
    "### F1 = 2*P*R/(P+R) = 2TP/(2TP+FP+FN)\n",
    "\n",
    "#### How many are the TP, FN, FP for this multi-class setting?\n",
    "\n",
    "| Label |      TP |  FN |  FP |\n",
    "|-------|---------|-----|-----|\n",
    "| graphics  | 344 |  45 |  11 |\n",
    "| space     | 364 |  30 |  30 |\n",
    "| christian | 392 |   6 | 243 |\n",
    "| misc      |  48 | 203 |   0 |\n",
    "|           |     |     |     |\n",
    "| Total    | 1148 | 284 | 284 |\n",
    "\n",
    "#### The Macro-avg F1 is the average F1 of all classes\n",
    "Macro-avg F1 = (0.925+0.924+0.759+0.321)/4 = 0.732\n",
    "\n",
    "#### The Micro-avg F1 is the result of the pooling of all TP, FN, FP (thus, it is the Accuracy)\n",
    "Micro-avg F1 = 2 * 1148 / ( 2 * 1148 + 284 + 284) = 0.802\n",
    "\n",
    "Accuracy = Correct / Total = 1148 / 1432 = 0.802\n",
    "\n",
    "#### Why the total examples is 1432? \n",
    "See Support column above\n",
    "\n",
    "#### The weighted avg is the average F1 of all classes weighted by the support of each class\n",
    "Weighted avg F1 = (389/1432)*0.925 + (394/1432)*0.924 + (398/1432)*0.759 + (251/1432)*0.321 = 0.773"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "50393cd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's try to predict the category of new texts\n",
    "\n",
    "# We will define a simple method that given a text string \"s\", it asks the model to predict \n",
    "# the category for that text\n",
    "def predict_category(s, train=train, model=model):\n",
    "    pred = model.predict([s])\n",
    "    return train.target_names[pred[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "42334983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sci.space'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_category('Huston we are sending a payload to the ISS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c7c95e28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'soc.religion.christian'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_category('I am in a discussion between islam and christianity')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "010dd9d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'comp.graphics'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_category('the screen resolution is very low for this setting')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fdd0fbe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'comp.graphics'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_category('this new card should be very fast and I cannot wait to see the new unreal graphic engine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "694c25c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sci.space'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_category('We are ready to launch in the countdown to 10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8221d74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "local-venv",
   "language": "python",
   "name": "local-venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}