Texmin_1_Information_Retrieval.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Texmin 1 - Information Retrieval.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abuwildanm/Text-Mining/blob/master/Texmin_1_Information_Retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "itI3qhHI6mcu",
        "colab_type": "text"
      },
      "source": [
        "# Tugas Text Mining"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "typhPxl-6p5h",
        "colab_type": "text"
      },
      "source": [
        "Nama : Abu Wildan Mucholladin\n",
        "\n",
        "NIM : 165150200111002"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e-ajrtlx5uGh",
        "colab_type": "code",
        "outputId": "6ef28b7c-7b34-4f3f-e4dc-eb8c8837f83b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Import library standard\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "# Import library for indonesian stemming\n",
        "!pip install Sastrawi\n",
        "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
        "from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: Sastrawi in /usr/local/lib/python3.6/dist-packages (1.0.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S31xCZNw8-St",
        "colab_type": "text"
      },
      "source": [
        "## Parsing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZ_v3qHX7N4d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Document\n",
        "d1 = 'Kupu-kupu terbang di atas pohon'\n",
        "d2 = 'Dia terbang sambil mencari pohon untuk bertelur'\n",
        "d3 = 'Pohon tempat kupu-kupu bertelur adalah pohon mangga'\n",
        "d4 = 'Kupu-kupu bertelur untuk berkembang biak'\n",
        "document = np.array([d1, d2, d3, d4])\n",
        "\n",
        "# Query\n",
        "q1 = 'Kupu pohon'\n",
        "q2 = 'Kupu-kupu pohon'\n",
        "# query = np.array([q1])\n",
        "query = np.array([q1, q2])\n",
        "\n",
        "# Document & Query Combination\n",
        "document_query = np.append(document, query)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSbAkGnG82Fi",
        "colab_type": "text"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7xFhAB55955",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tokenizing(text):\n",
        "    # Case folding\n",
        "    text = text.lower()\n",
        "    \n",
        "    # Tokenization\n",
        "    token = re.findall('[A-Za-z]+', text)\n",
        "    token = np.array(token)\n",
        "    \n",
        "    return token\n",
        "\n",
        "def filtering(token):\n",
        "    # Stoplist\n",
        "    stopWordRemoverFactory = StopWordRemoverFactory()\n",
        "    stopwords = stopWordRemoverFactory.get_stop_words()\n",
        "\n",
        "    # Remove stopwords\n",
        "    stopwordRemover = ~np.isin(token, stopwords)\n",
        "    term = token[stopwordRemover]\n",
        "    \n",
        "    return term\n",
        "  \n",
        "def stemming(term):\n",
        "    # Create stemmer\n",
        "    stemmerFactory = StemmerFactory()\n",
        "    stemmer = stemmerFactory.create_stemmer()\n",
        "\n",
        "    # Stem process\n",
        "    stem = np.vectorize(lambda t: stemmer.stem(t))\n",
        "    term = stem(term)\n",
        "    \n",
        "    return term\n",
        "\n",
        "def preprocessing(text):\n",
        "    # Lexical Analysis\n",
        "    token = tokenizing(text)\n",
        "    # Filtering\n",
        "    term = filtering(token)\n",
        "    # Stemming\n",
        "    term = stemming(term)\n",
        "    \n",
        "    return term\n",
        "  \n",
        "def getFeatures(document):\n",
        "    feature = [preprocessing(text) for text in document]\n",
        "    feature = [term for listTerm in feature for term in listTerm]\n",
        "    feature = np.unique(feature)\n",
        "    return feature"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QKcLxfNYLLCm",
        "colab_type": "code",
        "outputId": "767f377b-1bc9-4bf0-f01d-6f20d96c57ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "feature = getFeatures(document_query)\n",
        "feature"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['atas', 'biak', 'cari', 'kembang', 'kupu', 'mangga', 'pohon',\n",
              "       'telur', 'tempat', 'terbang'], dtype='<U7')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 262
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ufZLmEHg9CZA",
        "colab_type": "text"
      },
      "source": [
        "## Term Weighting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IsoHm2eiot38",
        "colab_type": "text"
      },
      "source": [
        "### Term Frequency (tf) Weighting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TjWQQ597keA3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tfWeighting(document_query, normalize=True):\n",
        "    # Extract feature\n",
        "    feature = getFeatures(document_query)\n",
        "    \n",
        "    # Term Frequency Table\n",
        "    zero_data = np.zeros((document_query.shape[0], feature.shape[0]))\n",
        "    tf = pd.DataFrame(zero_data, columns=feature)\n",
        "    for i, text in enumerate(document_query):\n",
        "        word = preprocessing(text)\n",
        "        # Weighting process\n",
        "        term, frequency = np.unique(word, return_counts=True)\n",
        "        tf.loc[i, term] = frequency\n",
        "    \n",
        "    # Log Normalization\n",
        "    if normalize:\n",
        "       tf[tf != 0] = 1 + np.log10(tf)\n",
        "    \n",
        "    return tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "461KhZ77nauO",
        "colab_type": "code",
        "outputId": "14da6c91-c100-4339-9edb-bb74fc7d12fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        }
      },
      "source": [
        "tf = tfWeighting(document_query)\n",
        "tf"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: RuntimeWarning: divide by zero encountered in log10\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>atas</th>\n",
              "      <th>biak</th>\n",
              "      <th>cari</th>\n",
              "      <th>kembang</th>\n",
              "      <th>kupu</th>\n",
              "      <th>mangga</th>\n",
              "      <th>pohon</th>\n",
              "      <th>telur</th>\n",
              "      <th>tempat</th>\n",
              "      <th>terbang</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.30103</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.30103</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.30103</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.30103</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.30103</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   atas  biak  cari  kembang     kupu  mangga    pohon  telur  tempat  terbang\n",
              "0   1.0   0.0   0.0      0.0  1.30103     0.0  1.00000    0.0     0.0      1.0\n",
              "1   0.0   0.0   1.0      0.0  0.00000     0.0  1.00000    1.0     0.0      1.0\n",
              "2   0.0   0.0   0.0      0.0  1.30103     1.0  1.30103    1.0     1.0      0.0\n",
              "3   0.0   1.0   0.0      1.0  1.30103     0.0  0.00000    1.0     0.0      0.0\n",
              "4   0.0   0.0   0.0      0.0  1.00000     0.0  1.00000    0.0     0.0      0.0\n",
              "5   0.0   0.0   0.0      0.0  1.30103     0.0  1.00000    0.0     0.0      0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 264
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y1mNywGkqXXB",
        "colab_type": "text"
      },
      "source": [
        "### Inverse Document Frequency (idf) Weighting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoPDojECv7me",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def idfWeighting(term_frequency_document):\n",
        "    doc_freq = pd.DataFrame({'Document Frequency' : (term_frequency_document != 0).sum(axis=0)})\n",
        "    doc_freq['Inverse Document Frequency'] = np.log10(term_frequency_document.shape[0] / doc_freq)\n",
        "    return doc_freq"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kzC-2EK_o_vj",
        "colab_type": "code",
        "outputId": "c9eb4dd6-4576-4c83-819f-7cc2a0b4f749",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "tf_document = tf.iloc[:-query.shape[0]]\n",
        "doc_freq = idfWeighting(tf_document)\n",
        "doc_freq"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document Frequency</th>\n",
              "      <th>Inverse Document Frequency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>atas</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>biak</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cari</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>kembang</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>kupu</th>\n",
              "      <td>3</td>\n",
              "      <td>0.124939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mangga</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>pohon</th>\n",
              "      <td>3</td>\n",
              "      <td>0.124939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>telur</th>\n",
              "      <td>3</td>\n",
              "      <td>0.124939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tempat</th>\n",
              "      <td>1</td>\n",
              "      <td>0.602060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>terbang</th>\n",
              "      <td>2</td>\n",
              "      <td>0.301030</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         Document Frequency  Inverse Document Frequency\n",
              "atas                      1                    0.602060\n",
              "biak                      1                    0.602060\n",
              "cari                      1                    0.602060\n",
              "kembang                   1                    0.602060\n",
              "kupu                      3                    0.124939\n",
              "mangga                    1                    0.602060\n",
              "pohon                     3                    0.124939\n",
              "telur                     3                    0.124939\n",
              "tempat                    1                    0.602060\n",
              "terbang                   2                    0.301030"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 266
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cA6EhGfVwUtJ",
        "colab_type": "text"
      },
      "source": [
        "### Tf-Idf Weighting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7l4Edvuz-t1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tfIdfWeighting(document_query, query, normalize_tf=True, normalize_weight=True):\n",
        "    tf = tfWeighting(document_query, normalize_tf)\n",
        "    tf_document = tf.iloc[:-query.shape[0]]\n",
        "    doc_freq = idfWeighting(tf_document)\n",
        "    tf_idf = tf.mul(doc_freq['Inverse Document Frequency'], axis=1)\n",
        "    \n",
        "    if normalize_weight:\n",
        "      tf_idf_square = tf_idf ** 2\n",
        "      tf_idf_sum = tf_idf_square.sum(axis=0)\n",
        "      tf_idf_sqrt = np.sqrt(tf_idf_sum)\n",
        "      tf_idf = tf_idf / tf_idf_sqrt\n",
        "    \n",
        "    return tf_idf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8UuN1sNwYNs",
        "colab_type": "code",
        "outputId": "215e8f5c-b0c9-41d1-8dfe-4c8aa4406e9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        }
      },
      "source": [
        "tf_idf = tfIdfWeighting(document_query, query)\n",
        "tf_idf"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: RuntimeWarning: divide by zero encountered in log10\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>atas</th>\n",
              "      <th>biak</th>\n",
              "      <th>cari</th>\n",
              "      <th>kembang</th>\n",
              "      <th>kupu</th>\n",
              "      <th>mangga</th>\n",
              "      <th>pohon</th>\n",
              "      <th>telur</th>\n",
              "      <th>tempat</th>\n",
              "      <th>terbang</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.466720</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.419123</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.707107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.419123</td>\n",
              "      <td>0.57735</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.707107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.466720</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.545292</td>\n",
              "      <td>0.57735</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.466720</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.57735</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.358731</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.419123</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.466720</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.419123</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   atas  biak  cari  kembang  ...     pohon    telur  tempat   terbang\n",
              "0   1.0   0.0   0.0      0.0  ...  0.419123  0.00000     0.0  0.707107\n",
              "1   0.0   0.0   1.0      0.0  ...  0.419123  0.57735     0.0  0.707107\n",
              "2   0.0   0.0   0.0      0.0  ...  0.545292  0.57735     1.0  0.000000\n",
              "3   0.0   1.0   0.0      1.0  ...  0.000000  0.57735     0.0  0.000000\n",
              "4   0.0   0.0   0.0      0.0  ...  0.419123  0.00000     0.0  0.000000\n",
              "5   0.0   0.0   0.0      0.0  ...  0.419123  0.00000     0.0  0.000000\n",
              "\n",
              "[6 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 268
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6B1mz_rmIh5J",
        "colab_type": "text"
      },
      "source": [
        "## Cosine Similarity"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWUKhruiVkgG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def cosineSimilarity(weight_document, weight_query, normalize_weight=True):\n",
        "    cosim = []\n",
        "    if normalize_weight:\n",
        "      cosim = weight_document.dot(weight_query.T)\n",
        "    else:\n",
        "      weight_dot = weight_document.dot(weight_query.T)\n",
        "      weight_document_square = weight_document ** 2\n",
        "      weight_query_square = weight_query ** 2\n",
        "      weight_document_sum = weight_document_square.sum(axis=1)\n",
        "      weight_query_sum = weight_query_square.sum(axis=1)\n",
        "      \n",
        "      weight_sum = pd.DataFrame()\n",
        "      for i, weight in enumerate(weight_query_sum):\n",
        "        weight_sum['Query {}'.format(i)] = weight_document_sum * weight\n",
        "        \n",
        "      weight_sqrt = np.sqrt(weight_sum)\n",
        "      cosim = weight_dot / weight_sqrt.values\n",
        "    \n",
        "    column_name = [('Query {}'.format(i)) for i in range(weight_query.shape[0])]\n",
        "    cosim.columns = column_name\n",
        "    return cosim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U162vQYcdbQD",
        "colab_type": "code",
        "outputId": "5cbd3da8-5ede-4168-d855-4f3d951b82a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        }
      },
      "source": [
        "tf_idf_document, tf_idf_query = tf_idf.iloc[:-query.shape[0]], tf_idf.iloc[-query.shape[0]:]\n",
        "cosim = cosineSimilarity(tf_idf_document, tf_idf_query)\n",
        "cosim"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Query 0</th>\n",
              "      <th>Query 1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.343092</td>\n",
              "      <td>0.393492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.175664</td>\n",
              "      <td>0.175664</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.395972</td>\n",
              "      <td>0.446372</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.167427</td>\n",
              "      <td>0.217828</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Query 0   Query 1\n",
              "0  0.343092  0.393492\n",
              "1  0.175664  0.175664\n",
              "2  0.395972  0.446372\n",
              "3  0.167427  0.217828"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 270
        }
      ]
    }
  ]
}