Summarization_of_Legal_Texts.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1BYl_XmEIJWE"
      },
      "source": [
        "# Configuração inicial"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ngmpqQlXHI0j"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZY8WSTH-dk5m",
        "outputId": "91e40188-756e-4cf3-9641-3c4fb0c57f17"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "2022-10-22 00:00:31.832469: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "\u001b[K     |████████████████████████████████| 13.0 MB 59.6 MB/s \n",
            "\u001b[?25h\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('pt_core_news_sm')\n"
          ]
        }
      ],
      "source": [
        "!pip install codetiming --quiet\n",
        "!pip install spacy --quiet\n",
        "!pip install rouge-score --quiet\n",
        "!spacy download pt_core_news_sm --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G0k4TkkOxLo8",
        "outputId": "89090b9c-7332-41c8-da31-7f45a34ed4d0"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import matplotlib as plt\n",
        "import numpy as np\n",
        "np.set_printoptions(threshold=7)\n",
        "import networkx as nx\n",
        "from math import ceil, floor\n",
        "# from numpy.core.memmap import dtype\n",
        "\n",
        "# NLTK\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords as NLTK_stopwords\n",
        "from nltk.tokenize import sent_tokenize as NLTK_sent_tokenize, word_tokenize as NLTK_word_tokenize, RegexpTokenizer as NLTK_RegexpTokenizer\n",
        "from nltk.cluster.util import cosine_distance as NLTK_cosine_distance\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "# Spacy\n",
        "import spacy\n",
        "nlp = spacy.load('pt_core_news_sm')\n",
        "\n",
        "# Sklearn\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Widgets iterativos\n",
        "from google.colab import output\n",
        "output.enable_custom_widget_manager()\n",
        "from ipywidgets import interact\n",
        "from ipywidgets import IntSlider, FloatSlider, SelectionSlider, Dropdown\n",
        "from ipywidgets import Checkbox, SelectMultiple, Text\n",
        "import ipywidgets as w\n",
        "\n",
        "# Métricas de validação\n",
        "from rouge_score import rouge_scorer\n",
        "scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=False)\n",
        "\n",
        "# Permite type hint de funções e list no colab\n",
        "from typing import Callable, List, Dict"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eC8G0VLrIfV7"
      },
      "source": [
        "## Download e leitura do corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pl47PA2VRPEi",
        "outputId": "da01fced-8a7b-41bb-8e05-a94673ed7700"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'rulingbr'...\n",
            "remote: Enumerating objects: 31, done.\u001b[K\n",
            "remote: Counting objects: 100% (3/3), done.\u001b[K\n",
            "remote: Compressing objects: 100% (3/3), done.\u001b[K\n",
            "^C\n",
            "tar (child): rulingbr/rulingbr-v1.2.tar.xz: Cannot open: No such file or directory\n",
            "tar (child): Error is not recoverable: exiting now\n",
            "tar: Child returned status 2\n",
            "tar: Error is not recoverable: exiting now\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/diego-feijo/rulingbr/ # Download dos arquivos\n",
        "!tar xJvf rulingbr/rulingbr-v1.2.tar.xz # Extrai pasta compactada com o corpus rulingbr-v1.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "uq9OTxYEicsu",
        "outputId": "c2ce8acd-60c6-4bda-fe03-3fdd3617d49d"
      },
      "outputs": [
        {
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-ccd74221a741>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Load docs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdocs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'rulingbr-v1.2.jsonl'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     docs = list(map(\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mlambda\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'rulingbr-v1.2.jsonl'"
          ]
        }
      ],
      "source": [
        "# Load docs\n",
        "docs = []\n",
        "with open('rulingbr-v1.2.jsonl') as f:\n",
        "    docs = list(map(\n",
        "        lambda line: json.loads(line), \n",
        "        f\n",
        "    ))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ZPv3lEJLBa6"
      },
      "source": [
        "# Exploração do Corpus\n",
        "\n",
        "Lista de seções por ordem de relevância\n",
        " - Ementa: Gold Standart\n",
        " ---\n",
        " - Relatório: General description about the case.\n",
        " - Voto: Judge vote(s). It contains one or more votes from the judges. This document presents the critical aspects considered for the decision.\n",
        " - Acordão: This is the final decision. It is the compilation of the votes.\n",
        " - Area: Broad topic about the decision. [Lista de possiblidade aqui](https://github.com/diego-feijo/rulingbr#area)\n",
        " ---\n",
        " - Recursos: [Lista de possiblidade aqui](https://github.com/diego-feijo/rulingbr#a%C3%A7%C3%B5es-aut%C3%B4nomas)\n",
        " - Classe: It is the specific juridic instrument. [It can be one of the following](https://github.com/diego-feijo/rulingbr#classe). They may also be combined.\n",
        " ---\n",
        " - Extrato: Additional information. This section is missing in near 50% of the cases.\n",
        " - Relator: The name of the judge responsible for composing the report and the first vote of the decision."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84RM7oEqKUWZ"
      },
      "source": [
        "## Visualização inicial\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RFnWBA7fsdqK",
        "outputId": "8a6a5314-7559-495c-827b-667606d1261a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "dict_keys(['ementa', 'acordao', 'relatorio', 'voto', 'extrato', 'relator', 'classe', 'area'])\n",
            "\n",
            "ementa -> EMBARGOS DE DECLARAÇÃO - INOCORRÊNCIA DE CONTRADIÇÃO, OBSCURIDADE OU OMISSÃO - PRETENDIDO REEXAME DA CAUSA CARÁTER INFRINGENTE - INADMISSIBILIDADE – EMBARGOS DE DECLARAÇÃO REJEITADOS. - Não se revelam admissíveis os embargos de declaração, quando a parte recorrente - a pretexto de esclarecer uma inexistente situação de obscuridade, omissão ou contradição - vem a utilizá-los com o objetivo de infringir o julgado e de viabilizar, assim, um indevido reexame da causa. Precedentes. - A nulidade radical que afeta os diversos atos de interrogatório judicial, quando provocada pela conduta do magistrado que arbitrariamente nega, ao réu, o direito – por este titularizado – de formular reperguntas aos demais litisconsortes penais passivos, contamina, por efeito causal, todos os atos subseqüentes do processo, notadamente aqueles de índole probatória, como a inquirição de testemunhas arroladas pelo Ministério Público. Precedentes.\n",
            "acordao -> Vistos, relatados e discutidos estes autos, acordam os Ministros do Supremo Tribunal Federal, em Segunda Turma, sob a Presidência do Ministro Celso de Mello (RISTF, art. 37, II), na conformidade da ata de julgamentos e das notas taquigráficas, por unanimidade de votos, em rejeitar os embargos de declaração opostos pelo Ministério Público Federal, julgando prejudicado o exame da consulta formulada pelo Juízo da 6ª Vara Federal Criminal de São Paulo/SP (fls. 2202 e 2.203/2.207), nos termos do voto do Relator. HC 94.016 ED Ausentes, justificadamente, neste julgamento, a Ellen Gracie e o Senhor Ministro Joaquim Barbosa. Senhora Ministra\n",
            "relatorio -> O SENHOR MINISTRO CELSO DE MELLO - (Relator): Trata-se de embargos de declaração opostos pelo Ministério Público Federal a decisão proferida por esta colenda Segunda Turma, consubstanciada em acórdão assim ementado (fls. 2.196/2.198): “‘HABEAS CORPUS’ SÚMULA 691/STF INAPLICABILIDADE AO CASO - OCORRÊNCIA DE SITUAÇÃO EXCEPCIONAL QUE AFASTA A RESTRIÇÃO SUMULAR ESTRANGEIRO NÃO DOMICILIADO NO BRASIL - IRRELEVÂNCIA CONDIÇÃO JURÍDICA QUE NÃO O DESQUALIFICA COMO SUJEITO DE DIREITOS E TITULAR DE GARANTIAS CONSTITUCIONAIS E LEGAIS - PLENITUDE DE ACESSO, EM CONSEQÜÊNCIA, AOS INSTRUMENTOS PROCESSUAIS DE TUTELA DA LIBERDADE NECESSIDADE DE RESPEITO, PELO PODER PÚBLICO, ÀS PRERROGATIVAS JURÍDICAS QUE COMPÕEM O PRÓPRIO ESTATUTO CONSTITUCIONAL DO DIREITO DE DEFESA - A GARANTIA CONSTITUCIONAL DO ‘DUE PROCESS OF LAW’ COMO EXPRESSIVA LIMITAÇÃO À ATIVIDADE PERSECUTÓRIA DO ESTADO (INVESTIGAÇÃO PENAL E PROCESSO PENAL) - O CONTEÚDO MATERIAL DA CLÁUSULA DE GARANTIA DO ‘DUE PROCESS’ INTERROGATÓRIO JUDICIAL - NATUREZA JURÍDICA - MEIO DE DEFESA DO ACUSADO - POSSIBILIDADE DE QUALQUER DOS LITISCONSORTES PENAIS PASSIVOS FORMULAR REPERGUNTAS AOS DEMAIS CO-RÉUS, NOTADAMENTE SE AS DEFESAS DE TAIS ACUSADOS SE MOSTRAREM COLIDENTES PRERROGATIVA HC 94.016 ED JURÍDICA CUJA LEGITIMAÇÃO DECORRE DO POSTULADO CONSTITUCIONAL DA AMPLA DEFESA - PRECEDENTE DO SUPREMO TRIBUNAL FEDERAL (PLENO) - MAGISTÉRIO DA DOUTRINA CONSTRANGIMENTO ILEGAL CARACTERIZADO – ‘HABEAS CORPUS’ CONCEDIDO ‘EX OFFICIO’, COM EXTENSÃO DE SEUS EFEITOS AOS CO-RÉUS. DENEGAÇÃO DE MEDIDA LIMINAR - SÚMULA 691/STF SITUAÇÕES EXCEPCIONAIS QUE AFASTAM A RESTRIÇÃO SUMULAR. - A jurisprudência do Supremo Tribunal Federal tem admitido o afastamento, ‘hic et nunc’, da Súmula 691/STF, em hipóteses nas quais a decisão questionada divirja da jurisprudência predominante nesta Corte ou, então, veicule situações configuradoras de abuso de poder ou de manifesta ilegalidade. Precedentes. Hipótese ocorrente na espécie. O SÚDITO ESTRANGEIRO, MESMO AQUELE SEM DOMICÍLIO NO BRASIL, TEM DIREITO A TODAS AS PRERROGATIVAS BÁSICAS QUE LHE ASSEGUREM A PRESERVAÇÃO DO ‘STATUS LIBERTATIS’ E A OBSERVÂNCIA, PELO PODER PÚBLICO, DA CLÁUSULA CONSTITUCIONAL DO ‘DUE PROCESS’. - O súdito estrangeiro, mesmo o não domiciliado no Brasil, tem plena legitimidade para impetrar o remédio constitucional do ‘habeas corpus’, em ordem a tornar efetivo, nas hipóteses de persecução penal, o direito subjetivo, de que também é titular, à observância e ao integral respeito, por parte do Estado, das prerrogativas que compõem e dão significado à cláusula do devido processo legal. - A condição jurídica de não-nacional do Brasil e a circunstância de o réu estrangeiro não possuir domicílio em nosso país não legitimam a adoção, contra tal acusado, de qualquer tratamento arbitrário ou discriminatório. Precedentes. - Impõe-se, ao Judiciário, o dever de assegurar, mesmo ao réu estrangeiro sem domicílio no Brasil, os direitos básicos que resultam do postulado do devido processo legal, notadamente as prerrogativas inerentes à garantia da ampla defesa, à garantia do contraditório, à igualdade entre as partes perante o juiz natural e à garantia de imparcialidade do magistrado processante. HC 94.016 ED A ESSENCIALIDADE DO POSTULADO DO DEVIDO PROCESSO LEGAL, QUE SE QUALIFICA COMO REQUISITO LEGITIMADOR DA PRÓPRIA ‘PERSECUTIO CRIMINIS’. - O exame da cláusula referente ao ‘due process of law’ permite nela identificar alguns elementos essenciais à sua configuração como expressiva garantia de ordem constitucional, destacando-se, dentre eles, por sua inquestionável importância, as seguintes prerrogativas: (a) direito ao processo (garantia de acesso ao Poder Judiciário); (b) direito à citação e ao conhecimento prévio do teor da acusação; (c) direito a um julgamento público e célere, sem dilações indevidas; (d) direito ao contraditório e à plenitude de defesa (direito à autodefesa e à defesa técnica); (e) direito de não ser processado e julgado com base em leis ‘ex post facto’; (f) direito à igualdade entre as partes; (g) direito de não ser processado com fundamento em provas revestidas de ilicitude; (h) direito ao benefício da gratuidade; (i) direito à observância do princípio do juiz natural; (j) direito ao silêncio (privilégio contra a auto-incriminação); (l) direito à prova; e (m) direito de presença e de ‘participação ativa’ nos atos de interrogatório judicial dos demais litisconsortes penais passivos, quando existentes. - O direito do réu à observância, pelo Estado, da garantia pertinente ao ‘due process of law’, além de traduzir expressão concreta do direito de defesa, também encontra suporte legitimador em convenções internacionais que proclamam a essencialidade dessa franquia processual, que compõe o próprio estatuto constitucional do direito de defesa, enquanto complexo de princípios e de normas que amparam qualquer acusado em sede de persecução criminal, mesmo que se trate de réu estrangeiro, sem domicílio em território brasileiro, aqui processado por suposta prática de delitos a ele atribuídos. O INTERROGATÓRIO JUDICIAL COMO MEIO DE DEFESA DO RÉU. - Em sede de persecução penal, o interrogatório judicial notadamente após o advento da Lei nº 10.792/2003 - qualifica-se como ato de defesa do réu, que, além de não ser obrigado a responder a qualquer indagação feita pelo magistrado processante, HC 94.016 ED também não pode sofrer qualquer restrição em sua esfera jurídica em virtude do exercício, sempre legítimo, dessa especial prerrogativa. Doutrina. Precedentes. POSSIBILIDADE JURÍDICA DE UM DOS LITISCONSORTES PENAIS PASSIVOS, INVOCANDO A GARANTIA DO ‘DUE PROCESS OF LAW’, VER ASSEGURADO O SEU DIREITO DE FORMULAR REPERGUNTAS AOS CO-RÉUS, QUANDO DO RESPECTIVO INTERROGATÓRIO JUDICIAL. - Assiste, a cada um dos litisconsortes penais passivos, o direito fundado em cláusulas constitucionais (CF, art. 5º, incisos LIV e LV) - de formular reperguntas aos demais co-réus, que, no entanto, não estão obrigados a respondê-las, em face da prerrogativa contra a auto-incriminação, de que também são titulares. O desrespeito a essa franquia individual do réu, resultante da arbitrária recusa em lhe permitir a formulação de reperguntas, qualifica-se como causa geradora de nulidade processual absoluta, por implicar grave transgressão ao estatuto constitucional do direito de defesa. Doutrina. Precedente do STF.” A douta Procuradoria Geral da República, ao deduzir os presentes embargos de declaração, alega, em síntese, o que se segue (fls. 2.221/2.228): “(...) o acórdão incorreu, acidentalmente, em erro material na sua parte dispositiva. Isso acabou por gerar contradição e obscuridade no julgado (...). ................................................... (...) Muito embora toda a consistente fundamentação desenvolvida no voto condutor tenha-se debruçado exclusivamente sobre a questão da realização do interrogatório dos co-réus sem a participação dos defensores do paciente; muito embora em nenhum momento o acórdão desta Egrégia Corte haja feito qualquer consideração sobre quaisquer outros atos processuais praticados no curso da ação penal em epígrafe, a parte dispositiva, por equívoco, acabou determinando a HC 94.016 ED anulação dos atos praticados desde os interrogatórios judiciais dos demais co-réus. (...): ................................................... A interpretação literal do excerto em epígrafe pode levar à equivocada interpretação – não condizente com a fundamentação judicial – de que não foram anulados somente os interrogatórios dos co-réus, mas todos os atos processuais praticados posteriormente aos mesmos. Isso porque, nos termos da legislação vigente à época – recentemente alterada pela Lei nº 11.719/2008 – o interrogatório (que atualmente é o último ato da instrução, nos termos do art. 532 do CPP) era levado a efeito logo no início da instrução criminal, antes da oitiva das testemunhas. Desde modo, ao ler palavra ‘desde’ na frase ‘concedo, de ofício, a ordem de habeas corpus ao ora paciente, para anular, desde os interrogatórios judiciais dos demais co-réus [...] o Processo-Crime nº 2006.61.81.008647-8 [...]’, pode-se chegar à conclusão – certamente não pretendida por esta Excelsa Corte – que todos os demais atos processuais realizados posteriormente, tais como as oitivas de testemunhas, deveriam ser renovados. Tanto não foram essas as conseqüências desejadas por esta Suprema Corte que o dispositivo do acórdão, nas linhas seguintes, assume linguagem mais restritiva, determinando tão somente a realização de novos interrogatórios, sem fazer qualquer menção à renovação de outros atos processuais (vide nesse sentido, a parte sublinhada do excerto acima transcrito). E não poderia ser diferente, na medida em que, segundo o princípio da causalidade – ou da conservação do ato processual, na terminologia utilizada por Frederico Marques –, que rege a teoria das nulidades, só se contaminam da invalidade de um determinado ato aqueles outros que com ele guardem alguma relação de necessidade. (...): ................................................... Levando em conta tais reflexões, não haveria sentido algum em se determinar a anulação de depoimentos de testemunhas em que foram perfeitamente asseguradas as garantias do contraditório e da ampla defesa (nesse sentido, veja-se a certidão expedida pela Justiça Federal referente aos autos nº 2006.61.81.008647-8, à fl. 2207 – bem como as informações prestadas pelo juízo de origem à fl. 2203). Não há qualquer liame de HC 94.016 ED causalidade entre estes atos e aqueles atacados pelos impetrantes (interrogatórios). Entender a questão de outro modo seria negar qualquer eficácia aos princípios da efetividade e da instrumentalidade do processo. (...): .................................................... No caso em apreço, mais inusitada ainda seria a compreensão de que devem ser renovados todos os atos processuais cronologicamente subseqüentes ao interrogatório, se levarmos em consideração que, com a já mencionada reformulação do procedimento comum pela Lei nº 11.719/2008, o interrogatório passou a ser o último ato da instrução. Assim, tomando-se o processo em epígrafe, tendo em vista que as leis processuais têm aplicação imediata, seriam primeiramente repetidas todas as numerosas oitivas de testemunhas – que já haviam sido realizadas sob o crivo do contraditório e da ampla defesa, insista-se – para, somente ao final, serem renovados os atos processuais efetivamente nulos.” (grifei) A parte embargada, por sua vez, ao manifestar-se sobre os embargos de declaração (fls. 2.284/2.294), sustenta que “(...) não houve erro material, nem obscuridade ou contradição: o processo foi anulado desde os interrogatórios, inclusive. E a ressalva final a como devem ser feitos os novos interrogatórios é lógica, porque a base do ‘habeas’ era justamente contra a forma com que foram realizados pelo juízo de primeiro grau” (fls. 2.286 – grifei). Submeto, desse modo, ao exame desta colenda Segunda Turma, os presentes embargos de declaração. É o relatório.\n",
            "voto -> O SENHOR MINISTRO CELSO DE MELLO - (Relator): Entendo assistir plena razão à parte ora embargada, eis que não há, no acórdão ora impugnado, qualquer evidência de obscuridade, omissão ou contradição a sanar, circunstância essa que torna processualmente inviável o recurso em exame. Como se sabe, os embargos de declaração destinam-se, precipuamente, a desfazer obscuridades, a afastar contradições e a suprir omissões que eventualmente se registrem no acórdão proferido pelo Tribunal. acórdão de Essa modalidade embargado, viabilizar quando um recursal só permite o reexame utilizada com pronunciamento integrativo-retificador, o específico jurisdicional vocacionado a afastar as do objetivo de caráter situações de obscuridade, omissão ou contradição, e a complementar e esclarecer o conteúdo da decisão proferida. Desse modo, a decisão recorrida - que aprecia, como no caso, com plena determinada pretensão recursal dos embargos jurídico-processual exatidão jurídica de e em - não declaração, dessa toda permite sob modalidade pena de a sua o de inteireza, emprego grave recurso, da via disfunção eis que HC 94.016 ED inocorrentes, em tal situação, os pressupostos que justificariam a sua adequada utilização. Cumpre enfatizar, de outro lado, que não se revelam admissíveis os embargos de declaração, quando a parte embargante - a pretexto de esclarecer uma inexistente situação de obscuridade, omissão ou contradição – vem a utilizá-los com objetivo de infringir o julgado e de viabilizar, assim, um indevido reexame da causa (RTJ 191/694-695, Rel. Min. CELSO DE MELLO, v.g.). É por tal razão que a jurisprudência do Supremo Tribunal Federal, ao versar os aspectos ora mencionados, assim se tem pronunciado: “Embargos declaratórios. Inexistência de omissão, contradição, obscuridade ou dúvida, no acórdão embargado (art. 337 do RISTF). Embargos rejeitados. O que pretenderam os embargantes foi sustentar o desacerto do julgado e obter sua desconstituição. A isso não se prestam, porém, os embargos declaratórios.” (RTJ 134/1296, Rel. Min. SYDNEY SANCHES - grifei) “- A jurisprudência do Supremo Tribunal Federal tem ressaltado que os embargos de declaração não se revelam cabíveis, quando, utilizados com a finalidade de sustentar a incorreção do acórdão, objetivam, na realidade, a própria desconstituição do ato decisório proferido pelo Tribunal. Precedentes: RTJ 114/885 RTJ 116/1106 - RTJ 118/714 - RTJ 134/1296.” (AI 153.147-AgR-ED/RS, Rel. Min. CELSO DE MELLO) HC 94.016 ED “- O recurso de embargos de declaração não tem cabimento, quando, a pretexto de esclarecer uma inocorrente situação de obscuridade, contradição ou omissão do acórdão, vem a ser utilizado com o objetivo de infringir o julgado.” (RE 177.599-ED/DF, Rel. Min. CELSO DE MELLO) “Embargos declaratórios só se destinam a possibilitar a eliminação de obscuridade (...), contradição ou omissão do acórdão embargado (art. 337 do RISTF), não o reconhecimento de erro de julgamento. E como, no caso, é esse reconhecimento que neles se reclama, com a conseqüente reforma do acórdão, ficam eles rejeitados.” (RTJ 134/836, Rel. Min. SYDNEY SANCHES - grifei) O exame dos autos evidencia que os presentes embargos declaratórios consideradas revestem-se as razões de nítido expostas pela caráter própria infringente, parte embargante, circunstância essa que, por si só, basta para tornar incabível a espécie recursal ora em análise, consoante adverte o magistério jurisprudencial desta Corte, na linha dos precedentes anteriormente referidos. Impende assinalar, ainda, por necessário, e ao contrário do que sustenta a douta Procuradoria Geral da República, que a nulidade radical que afeta os diversos atos de interrogatório judicial, quando provocada pela conduta do magistrado que arbitrariamente nega, ao réu, o direito – por este titularizado – de HC 94.016 ED formular reperguntas contamina, processo, por aos efeito notadamente demais causal, litisconsortes todos aqueles de os índole penais atos passivos, subseqüentes probatória, do como a inquirição de testemunhas arroladas pelo Ministério Público. É que, no caso, há nítida relação de causa e efeito entre a conduta judicial e as conseqüências geradas no âmbito do processo penal de conhecimento, pois, com a arbitrária recusa do magistrado em permitir, ao réu, a legítima formulação de reperguntas aos demais litisconsortes penais passivos, o ora paciente, por intermédio de seus Advogados, deixou de obter dados informativos que lhe poderiam ser úteis quando da inquirição das testemunhas arroladas pelo Ministério Público. Não é por outra razão que a parte ora embargada repele, corretamente, a pretensão recursal do Ministério Público Federal, fazendo-o, em seus aspectos essenciais, nos seguintes termos (fls. 2.286/2.292): “(...) não houve erro material, nem obscuridade ou contradição: o processo foi anulado desde os interrogatórios, inclusive. E a ressalva final a como devem ser feitos os novos interrogatórios é lógica, porque a base do ‘habeas’ era justamente contra forma com que foram realizados pelo juízo de primeiro grau). HC 94.016 ED Abordando o mesmíssimo ‘princípio da causalidade’, invocado pelo embargante, expõem, com precisão e incrível ajuste ao caso ‘sub judice’, os Professores ANTONIO SCARANCE FERNANDES, ANTONIO MAGALHÃES GOMES FILHO e ADA PELLEGRINI GRINOVER que os ‘diversos atos que compõem o procedimento não têm existência isolada, independente, mas constituem uma cadeia lógica que objetiva a preparação da sentença final’. Daí que a ‘ausência ou invalidade de um determinado ato processual provoca sempre a indagação sobre a extensão da nulidade’. ‘Normalmente’, continuam, ‘sendo os vários atos processuais ordenados cronologicamente, a decretação da nulidade acarreta o recuo do procedimento ao momento em que se constatou o vício de forma, decorrendo daí a necessidade de se ordenar a renovação do processo a partir do ato originariamente nulo’. Todavia, advertem, nem sempre é assim. Sendo importante distinguir entre ‘atos da fase postulatória’ e ‘atos de instrução’. No caso dos primeiros, a nulidade ‘se propaga sempre para os demais atos’; o que não ocorre, necessariamente, com relação aos segundos. Daí, concluírem: ‘Nessa linha de raciocínio, a nulidade da denúncia, do despacho de recebimento, da citação, interrogatório e defesa prévia contamina necessariamente os atos posteriores’. Assim, é evidente que não era mesmo necessário que Vossa Excelência se alongasse no v. voto para explicar a extensão da nulidade absoluta declarada, já ela decorre da claridade (ofuscante) do disposto no artigo 573, §§ 1º e 2º, do CPP, que determinam que a ‘nulidade de um ato, uma vez declarada, causará a dos atos que dele diretamente dependam ou sejam consequência’, e que o ‘juiz que pronunciar a nulidade declarará os atos a que ela se estende’. Assim, nada mais ‘ex officio’ do que, declarando a nulidade, Vossa Excelência se visse obrigado a declarar os atos atingidos. ................................................... Ora, sendo o ato anulado um daqueles que, pelo rito anterior, integrava a fase postulatória, o próprio princípio da causalidade determina a nulidade dos atos da fase de instrução que lhe sucederam. E daí que a HC 94.016 ED incidência do novo rito, pelo juízo de primeiro grau, deva se dar, quando o feito retomar sua marcha a partir das citações, em conformidade com uma das seguintes soluções apresentadas, com maestria, por GUSTAVO HENRIQUE RIGHI IVAHY BADARÓ quando analisou o direito intertemporal e os novos ritos: ‘Caso já tenha havido o recebimento da denúncia e a citação do acusado, mas o interrogatório ainda não tiver sido realizado, o melhor a fazer é tornar sem efeito a designação da audiência de interrogatório, intimando-se o acusado do cancelamento da audiência, bem como de que deverá apresentar resposta escrita, no prazo de 10 dias, nos termos do artigo 396, ‘caput’, do CPP, seguindo o processo, posteriormente, as regras da nova lei.’” (grifei) Cabe registrar, finalmente, que o procedimento adotado na decisão ora embargada – que anulou o processo penal desde os interrogatórios judiciais dos demais co-réus, inclusive – tem sido observado pela jurisprudência desta Suprema Corte em casos em que também se reconheceu a nulidade do interrogatório judicial: “AÇÃO PENAL. Ato processual. Interrogatório. Realização mediante videoconferência. Inadmissibilidade. Forma singular não prevista no ordenamento jurídico. Ofensa a cláusulas do justo processo da lei (‘due process of law’). Limitação ao exercício da ampla defesa, compreendidas a autodefesa e a defesa técnica. Insulto às regras ordinárias do local de realização dos atos processuais penais e às garantias constitucionais da igualdade e da publicidade. Falta, ademais, de citação do réu preso, apenas instado a comparecer à sala da cadeia pública, no dia do interrogatório. Forma do ato determinada sem motivação alguma. Nulidade processual caracterizada. HC concedido para renovação do processo desde o HC 94.016 ED interrogatório, inclusive. Inteligência dos arts. 5º, LIV, LV, LVII, XXXVII e LIII, da CF, e 792, ‘caput’ e § 2º, 403, 2ª parte, 185, ‘caput’ e § 2º, 192, § único, 193, 188, todos do CPP. Enquanto modalidade de ato processual não prevista no ordenamento jurídico vigente, é absolutamente nulo o interrogatório penal realizado mediante videoconferência, sobretudo quando tal forma é determinada sem motivação alguma, nem citação do réu.” (HC 88.914/SP, Rel. Min. CEZAR PELUSO - grifei) “‘HABEAS CORPUS’. PROCESSO PENAL. LEI PAULISTA Nº 11.819/2005. INTERROGATÓRIO POR VIDEOCONFERÊNCIA. INCONSTITUCIONALIDADE FORMAL JÁ RECONHECIDA PELO PLENÁRIO DO SUPREMO TRIBUNAL FEDERAL. VIOLAÇÃO DO INCISO I DO ART. 22 DA CONSTITUIÇÃO FEDERAL. ORDEM CONDEDIDA. 1. No julgamento do HC 90.900, redator para o acórdão o ministro Menezes Direito, o Plenário do Supremo Tribunal Federal assentou, por expressiva maioria de votos, a inconstitucionalidade formal da Lei nº 11.819/2005, do Estado de São Paulo. Isto por entender que tal diploma legal ofende o inciso I do art. 22 da Constituição Federal, na medida em que disciplina matéria eminentemente processual. 2. Na concreta situação dos autos, em que pese a discordância da defesa, o paciente foi interrogado pelo sistema de videoconferência, nos termos do art. 1º da Lei nº 11.819/2005. Ordem concedida para anular, desde o interrogatório (inclusive), o processo-crime, expedindo-se alvará de soltura se por outro motivo paciente não tiver que permanecer preso.” (HC 91.859/SP, Rel. Min. CARLOS BRITTO - grifei) Sendo assim, pelas razões expostas, considerando o caráter infringente de que se reveste este recurso - que visa a um indevido inocorrência reexame dos da causa pressupostos -, e legais tendo de em vista, ainda, embargabilidade a (CPC, art. 535, CPP, art. 619, e RISTF, art. 337), rejeito os presentes HC 94.016 ED embargos de declaração, prejudicado o exame da consulta formulada pelo Juízo da 6ª Vara Federal Criminal de São Paulo/SP (fls. 2.202 e 2.203/2.207). É o meu voto. \n",
            "extrato -> SEGUNDA TURMA EXTRATO DE ATA EMB.DECL.NO HABEAS CORPUS 94.016-1 PROCED. : SÃO PAULO RELATOR : MIN. CELSO DE MELLO EMBTE.(S) : MINISTÉRIO PÚBLICO FEDERAL EMBDO.(A/S) : BORIS ABRAMOVICH BEREZOVSKY OU PLATON ELENIN ADV.(A/S) : ALBERTO ZACHARIAS TORON E OUTRO (A/S) COATOR(A/S)(ES) : RELATOR DO HC Nº 100.204 DO SUPERIOR TRIBUNAL DE JUSTIÇA Decisão: A Turma, por votação unânime, rejeitou os embargos de declaração opostos pelo Ministério Público Federal, julgando prejudicado o exame da consulta formulada pelo Juízo da 6ª Vara Federal Criminal de São Paulo/SP (fls. 2202/2203), nos termos do voto do Relator. Ausentes, justificadamente, neste julgamento, a Senhora Ministra Ellen Gracie e o Senhor Ministro Joaquim Barbosa. Presidiu, este julgamento, o Senhor Ministro Celso de Mello. 2ª Turma, 14.04.2009. Presidência do Senhor Ministro Celso de Mello. Presentes à sessão os Senhores Ministros Cezar Peluso, Joaquim Barbosa e Eros Grau. Ausente, justificadamente, a Senhora Ministra Ellen Gracie. Subprocurador-Geral da República, Dr. Wagner Gonçalves. Carlos Alberto Cantanhede Coordenador\n",
            "relator -> celso de mello\n",
            "classe -> embargos de declaração\n",
            "area -> direito processual civil\n",
            "=---------=\n",
            "celso de mello: EMBARGOS DE DECLARAÇÃO - INOCORRÊNCIA DE CONTRADIÇÃO\n"
          ]
        }
      ],
      "source": [
        "# Print do primeiro documento (ementa, acordão, relatório, voto, extrato, relator, classe, area)\n",
        "doc = docs[0]\n",
        "print(f\"{doc.keys()}\", end=\"\\n\\n\")\n",
        "for key in doc.keys():\n",
        "  print(f\"{key} -> {doc[key]}\")\n",
        "\n",
        "print(\"=---------=\")\n",
        "print(f\"{doc['relator']}: {doc['ementa'][:52]}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XyQ_Q0I59xXv"
      },
      "source": [
        "## Visualização iterativa + Correção de sentença + Abreviações"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7hmODaGu-SIh"
      },
      "source": [
        "#### Abreviações"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mpT2AT4D-VsB"
      },
      "outputs": [],
      "source": [
        "# https://www.stf.jus.br/arquivo/cms/publicacaoLegislacaoAnotada/anexo/siglas_cf.pdf\n",
        "ABREVIATION_DICT = {\n",
        "    \"fls.\": \"folhas\"          ,\n",
        "    \"fl.\" : \"folha\"           ,\n",
        "    \"art.\": \"artigo\"          ,\n",
        "    \"ART.\": \"ARTIGO\"          ,\n",
        "    \"Min.\": \"Ministro\"        ,\"min.\": \"Ministro\",\n",
        "    \"Rel.\": \"relator\"         ,\"rel.\": \"relator\",\n",
        "    \"Des.\": \"Desembargador\"   ,\"des.\": \"Desembargador\",\n",
        "    \"dec.\": \"decisão\"         ,\n",
        "    \"j.\"  : \"julgado em\"      ,\n",
        "}\n",
        "\n",
        "is_abreviation = lambda word: True if word.lower() in ABREVIATION_DICT.keys() else False\n",
        "\n",
        "def put_abreviations_in_lower_case(sentences):\n",
        "    for i in range(len(sentences)):\n",
        "        sentence = sentences[i]         \n",
        "        sentences[i] = \"\".join(list(map(\n",
        "            lambda word: word.lower() if is_abreviation(word) else word, \n",
        "            sentence\n",
        "        )))\n",
        "    return sentences"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IF_gD5bqe55F"
      },
      "source": [
        "#### Correção de sentença"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qYPUw3oie6tz"
      },
      "outputs": [],
      "source": [
        "def merge_wrong_splitted_sentence(sentences):\n",
        "    first_sentence_index = None\n",
        "    indexes_to_update = []\n",
        "    indexes_to_eliminate = []\n",
        "    for index, sentence in enumerate(sentences):\n",
        "\n",
        "        if first_sentence_index != index and first_sentence_index != None:\n",
        "            merged_sentence = sentences[first_sentence_index] + \" \" + sentence\n",
        "            indexes_to_update.append((merged_sentence, index))\n",
        "            indexes_to_eliminate.append(first_sentence_index)\n",
        "            first_sentence_index = None\n",
        "\n",
        "        if any(abreviation in sentence for abreviation in ABREVIATION_DICT.values()):\n",
        "            first_sentence_index = index\n",
        "        \n",
        "\n",
        "    for merged_sentence, index in indexes_to_update:\n",
        "        sentences[index] = merged_sentence\n",
        "        # print(\"=------=\")\n",
        "        # print(merged_sentence)\n",
        "        # print(\"=------=\")\n",
        "\n",
        "    for index in indexes_to_eliminate[::-1]: \n",
        "        sentences.pop(index)\n",
        "    # map(lambda i: sentences.pop(index), indexes_to_eliminate)\n",
        "\n",
        "\n",
        "    return sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pHkBahKLhPBh"
      },
      "outputs": [],
      "source": [
        "def correct_sentences(sentences):\n",
        "    sentences = put_abreviations_in_lower_case(sentences)\n",
        "\n",
        "    # Substitui as abreviações por palavras completas\n",
        "    for abreviation, word in ABREVIATION_DICT.items():\n",
        "        sentences = list(map(lambda s: s.replace(abreviation, word), sentences))\n",
        "\n",
        "    sentences = merge_wrong_splitted_sentence(sentences) # junta as sentenças separadas erroneamente\n",
        "    \n",
        "    return sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jtpxMJUM_YbQ"
      },
      "outputs": [],
      "source": [
        "def funcao(numero):\n",
        "    return numero * 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vZshW5b1_eLi",
        "outputId": "ba5d5c34-fa69-4cbe-841a-d3059f2e7b3a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "funcao(16)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ikParFD-DVFW"
      },
      "source": [
        "#### Visualização iterativa"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G2dMh95V230r"
      },
      "outputs": [],
      "source": [
        "def explore_text(\n",
        "                doc_index: int, \n",
        "                section_name: int,\n",
        "                put_emoji: bool,\n",
        "                RegEx: str\n",
        "                ):\n",
        "  \n",
        "  current_doc = docs[doc_index]\n",
        "  text = current_doc[section_name]\n",
        "\n",
        "  if( put_emoji ):\n",
        "    emoji = \" 👈\"\n",
        "  else:\n",
        "    emoji = \"\"\n",
        "\n",
        "  # Divide em sentenças\n",
        "  if (RegEx):\n",
        "    tokenizer = NLTK_RegexpTokenizer(RegEx)\n",
        "    sentences = tokenizer.tokenize(text) # Com regex\n",
        "  else:\n",
        "    sentences = NLTK_sent_tokenize(text) # Sem regex\n",
        "\n",
        "  sentences = correct_sentences(sentences)\n",
        "\n",
        "  for sentence in sentences:\n",
        "    print(f\"{sentence}{emoji}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284,
          "referenced_widgets": [
            "de94a90071774cba9937b2aaae90d39c",
            "a0e6f456f0b0412ca99ddf022458b7c2",
            "233bbbae11884d55a1c8c8a4b202f905",
            "56b3e6681f3d470aaa4149ef49803e7f",
            "f4da64fc4381466f8886c81475d7d4c0",
            "2386035ed3314fd0b97f9445e64afe34",
            "4195b4c6cb714905895f97360670eeb8",
            "052cd25a2b0f4b38a08460ff13201dcd",
            "b403e21ec19b4c549ee68c85f2f22fa0",
            "d060c0dc4bb54847b4d588785843e064",
            "8e50b564feb34cd3b446b755fbdce4dc",
            "7cb97a650751447f964b0dc80170ccea",
            "cd95dce0e3bd4b17952e59e55bad3df2",
            "89b7f6d45a08468ebb45187b20d34898",
            "220ee9d44d564bb3906a374b2cf9916e",
            "cc5561908bfa486cb523abfd725946d9"
          ]
        },
        "id": "fY-f8FvvLD7C",
        "outputId": "87c47271-7bab-4585-f3e3-ebb227a9632b"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "de94a90071774cba9937b2aaae90d39c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "interactive(children=(IntSlider(value=0, continuous_update=False, description='doc_index', max=10573), Dropdow…"
            ]
          },
          "metadata": {
            "application/vnd.jupyter.widget-view+json": {
              "colab": {
                "custom_widget_manager": {
                  "url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/d2e234f7cc04bf79/manager.min.js"
                }
              }
            }
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------\n"
          ]
        }
      ],
      "source": [
        "interact(\n",
        "    explore_text,\n",
        "\n",
        "    doc_index    = IntSlider(continuous_update=False, min=0, max=len(docs)-1, step=1, value=0),\n",
        "    section_name = Dropdown(options=[(\"Ementa\", \"ementa\"), (\"Relatório\", \"relatorio\"), (\"Area\", \"area\"), (\"Relator\", \"relator\"), (\"Voto\", \"voto\"), (\"Acordão\", \"acordao\"), (\"Classe\", \"classe\"), (\"Extrato\", \"extrato\") ], value=\"ementa\"),\n",
        "    put_emoji    = Checkbox(description=\"Marcar fim de sentença com 👈\", value=True),\n",
        "    RegEx        = w.Text(value=\"\", placeholder=\"Digite RegEx aqui\")\n",
        ")\n",
        "print(\"---------------\") # doc_index para testar regex: 1278, 1195"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284,
          "referenced_widgets": [
            "decc4aa3fe8945f7a7edc6a03f680658",
            "21ee69ce4feb49198afcc61122b2edea",
            "5792c68532b143e18855670b9d594551",
            "35a3c103a3b44a558da9b9118b8422a7",
            "ff9f786cd5244d728d1742a28a06b042",
            "26a0d516481f41fca10a9008821ae951",
            "a7089eb828b449edaa6a44e19e5003bf",
            "f7de7ba4bcce40dfaabb19f5aabd3b4e",
            "80405b0531ba4ee1815e528d41960da0",
            "cad015143c0c4cbc9dd54470c19a2ae1",
            "3dff8235753c4c0097455b9276f96c25",
            "2fc45387f79648b6b0ad44f27df15b14",
            "168af84d73c24052b7b3f04ca08e9d4c",
            "3eaffe6ab9a64532bf4c0997bf757ac8",
            "999941efbcb44c2daecdf5ca7694facf",
            "5cb04549f16b488abb948a3d6b8b2d13"
          ]
        },
        "id": "NOGjk9FhEVG_",
        "outputId": "9c23a0ca-6d86-4ba7-ee8a-8a281246832e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "decc4aa3fe8945f7a7edc6a03f680658",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "interactive(children=(IntSlider(value=0, continuous_update=False, description='doc_index', max=10573), Dropdow…"
            ]
          },
          "metadata": {
            "application/vnd.jupyter.widget-view+json": {
              "colab": {
                "custom_widget_manager": {
                  "url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/d2e234f7cc04bf79/manager.min.js"
                }
              }
            }
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------\n"
          ]
        }
      ],
      "source": [
        "interact(\n",
        "    explore_text,\n",
        "\n",
        "    doc_index    = IntSlider(continuous_update=False, min=0, max=len(docs)-1, step=1, value=0),\n",
        "    section_name = Dropdown(options=[(\"Ementa\", \"ementa\"), (\"Relatório\", \"relatorio\"), (\"Area\", \"area\"), (\"Relator\", \"relator\"), (\"Voto\", \"voto\"), (\"Acordão\", \"acordao\"), (\"Classe\", \"classe\"), (\"Extrato\", \"extrato\") ], value=\"ementa\"),\n",
        "    put_emoji    = Checkbox(description=\"Marcar fim de sentença com 👈\", value=True),\n",
        "    RegEx        = w.Text(value=\"\", placeholder=\"Digite RegEx aqui\")\n",
        ")\n",
        "print(\"---------------\") # doc_index para testar regex: 1278, 1195"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKTw55fhewiC"
      },
      "source": [
        "## Criação da lista de stop words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZHiDDSfgew53",
        "outputId": "f1ce6c79-824f-4c2f-d0d6-117cc43b3d42"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['houve', 'eu', 'novo', 'pontos', 'nuns', 'geral', 'vós', 'puderam', 'local', 'algo', 'daquela', 'atrás', 'seria', 'logo', 'talvez', 'tiverem', 'tenham', 'tua', 'tanta', 'dar', 'estávamos', 'mesmo', 'fazeis', 'entre', 'aos', 'debaixo', 'ser', 'ligado', 'te', 'do', 'cinco', 'será', 'dez', 'disso', 'certamente', 'foste', 'fazes', 'por', 'pois', 'mais', 'para', 'ali', 'algumas', 'delas', 'nível', 'pegar', 'estamos', 'longe', 'poderá', 'tive', 'nesta', 'conhecida', 'estivermos', 'tão', 'segunda', 'menos', 'querem', 'aquela', 'quieto', 'tipo', 'meses', 'forem', 'pelos', 'ir', 'final', 'aquelas', 'até', 'estavam', 'cento', 'enquanto', 'porquanto', 'terceira', 'número', 'tiveste', 'inclusive', 'vinte', 'bastante', 'tivemos', 'vosso', 'tivesse', 'formos', 'esse', 'breve', 'tente', 'haja', 'sem', 'nova', 'estivesse', 'os', 'tentei', 'irá', 'fomos', 'último', 'bem', 'sejamos', 'vezes', 'num', 'suas', 'obrigado', 'ter', 'cedo', 'dois', 'quinze', 'vem', 'estivessem', 'à', 'estiveste', 'houverem', 'nem', 'teve', 'se', 'vocês', 'tínhamos', 'nossas', 'vários', 'naquele', 'tivessem', 'meus', 'inicio', 'sétimo', 'pouco', 'mil', 'era', 'estava', 'o', 'são', 'nenhuma', 'quê', 'oitavo', 'houveria', 'questão', 'comprido', 'já', 'primeiro', 'lá', 'dezanove', 'seja', 'dizer', 'lado', 'põe', 'estiverem', 'tinham', 'pôde', 'porém', 'saber', 'em', 'estejamos', 'agora', 'ele', 'aqui', 'usar', 'teremos', 'tivera', 'poder', 'grandes', 'estará', 'favor', 'tivéssemos', 'vens', 'fazem', 'assim', 'sou', 'tiver', 'ao', 'onde', 'seremos', 'tendes', 'porque', 'tiveram', 'no', 'sobre', 'sexto', 'estivéramos', 'próximo', 'depois', 'houvera', 'minhas', 'então', 'vai', 'outras', 'essa', 'uma', 'usa', 'haver', 'embora', 'nos', 'as', 'de', 'dá', 'tivéramos', 'houvermos', 'todas', 'estiver', 'maior', 'aí', 'próprio', 'quatro', 'conhecido', 'sob', 'próxima', 'grupo', 'zero', 'p', 'mês', 'quer', 'ponto', 'estar', 'outros', 'umas', 'após', 'quinta', 'essas', 'corrente', 'tais', 'tens', 'vossos', 'sete', 'posição', 'minha', 'sua', 'houveram', 'máximo', 'elas', 'podia', 'houvesse', 'tempo', 'ela', 'houvéramos', 'das', 'momento', 'às', 'estivéssemos', 'estivera', 'nós', 'apoio', 'fossem', 'seis', 'meu', 'contudo', 'tenhamos', 'coisa', 'nada', 'tuas', 'sois', 'mal', 'muito', 'esteja', 'pouca', 'seu', 'com', 'hão', 'diante', 'povo', 'quinto', 'maioria', 'apoia', 'e', 'qual', 'ademais', 'esteve', 'tenha', 'um', 'novos', 'tivestes', 'estivemos', 'estive', 'houvessem', 'ainda', 'fostes', 'tenho', 'comprida', 'relação', 'este', 'a', 'é', 'naquela', 'uns', 'portanto', 'só', 'houverá', 'menor', 'primeira', 'dezassete', 'sistema', 'segundo', 'estejam', 'pelo', 'tinha', 'fim', 'quando', 'estivestes', 'fez', 'tentar', 'teria', 'houveríamos', 'custa', 'cá', 'valor', 'esses', 'vossa', 'estes', 'nossos', 'deverá', 'tarde', 'houverei', 'houvemos', 'contra', 'eram', 'diz', 'oitava', 'cima', 'seríamos', 'terão', 'teríamos', 'deles', 'foi', 'terei', 'treze', 'temos', 'fôssemos', 'oito', 'parte', 'vêm', 'fará', 'teriam', 'havemos', 'que', 'dezasseis', 'na', 'toda', 'des', 'me', 'isso', 'cada', 'sabe', 'estas', 'aquele', 'exemplo', 'apenas', 'seriam', 'quero', 'vão', 'perto', 'quarto', 'põem', 'ou', 'boa', 'éramos', 'quem', 'mas', 'catorze', 'pode', 'deve', 'isto', 'for', 'também', 'serei', 'muitos', 'desde', 'demais', 'você', 'nove', 'caminho', 'novas', 'neste', 'estão', 'vossas', 'estiveram', 'sei', 'todo', 'meio', 'dizem', 'numa', 'quais', 'grande', 'podem', 'ontem', 'houverão', 'ambas', 'fazemos', 'tudo', 'eventual', 'pela', 'fosse', 'baixo', 'bom', 'nas', 'fôramos', 'dele', 'lugar', 'possível', 'tém', 'hajamos', 'dela', 'falta', 'dos', 'vais', 'qualquer', 'da', 'vos', 'p/', 'sexta', 'nossa', 'estado', 'nessa', 'cujo', 'antes', 'vinda', 'estou', 'estás', 'tanto', 'certeza', 'dentro', 'faço', 'duas', 'ver', 'vindo', 'houveremos', 'não', 'obrigada', 'somos', 'tu', 'ambos', 'nunca', 'iniciar', 'lhe', 'alguns', 'porquê', 'esta', 'tentaram', 'houvéssemos', 'têm', 'conselho', 'teus', 'és', 'como', 'teu', 'eles', 'hajam', 'aquilo', 'desse', 'sejam', 'está', 'serão', 'além', 'desta', 'nosso', 'cuja', 'maiorias', 'posso', 'possivelmente', 'área', 'direita', 'terá', 'sétima', 'hei', 'vez', 'parece', 'apontar', 'acerca', 'fazer', 'há', 'quieta', 'outra', 'doze', 'adeus', 'veja', 'fui', 'houveriam', 'dessa', 'sim', 'seus', 'através', 'faz', 'onze', 'lhes', 'todos', 'dezoito', 'pelas', 'fazia', 'sempre', 'aqueles', 'tivermos', 'nesse', 'deste', 'quanto', 'tal', 'forma', 'números', 'somente', 'três', 'ora', 'partir', 'fora', 'dão', 'daquele', 'houver', 'terceiro', 'foram', 'tem', 'devem', 'quarta']\n"
          ]
        }
      ],
      "source": [
        "my_stop_list = [\"p\", \"p/\"]\n",
        "NLTK_stop_words = NLTK_stopwords.words(\"portuguese\")\n",
        "Spacy_stop_words = list(nlp.Defaults.stop_words)\n",
        "\n",
        "STOP_WORDS = list(set(\n",
        "    my_stop_list + \n",
        "    NLTK_stop_words + \n",
        "    Spacy_stop_words\n",
        "))\n",
        "\n",
        "print(STOP_WORDS)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oiIFuLIKRezH"
      },
      "source": [
        "# Extração de features beaseada em heurísticas para **pontuação de sentenças**\n",
        "\n",
        "Observações:\n",
        "A pontuação pode ser positiva ou negativa e mais de uma pontuação pode ser usada. As heurísticas são escolhidas de forma iterativa no final deste colab.\n",
        "\n",
        " - TF\n",
        " - IDF\n",
        " - TF_IDF\n",
        " - Quantidade de palavras iniciando com letra maiúscula\n",
        " - Existência de número\n",
        " - Quantidade de nomes próprios\n",
        " - Posição\n",
        " - Tamanho\n",
        " - Quantidade de conjunções\n",
        " <!-- - Coocorrência/Relaçãos Sintagmática (Frequência de n-grams) -->\n",
        " <!-- - Similaridade Léxica (Word Wise) -->\n",
        "  <!-- - TODO: Encontrar heurísticas para similaridade entre palavras -->\n",
        "  <!-- - https://pt.frwiki.wiki/wiki/Similarit%C3%A9_lexicale -->\n",
        "  <!-- - Exemplo: Início, Meio, Fim, Ementa, Acordão, Relatório, Voto, Extrato, Relator, Classe, Area -->\n",
        "  <!-- - TODO: Encontrar lendo os documentos do corpus uma boa posição -->\n",
        " <!-- - Centralidade (repetição de informação entre sentenças) -->\n",
        " <!-- - Comparação com título  -->"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MZ04qH0KHnLa"
      },
      "source": [
        "### Funções auxiliares"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h9De3lESN2oz"
      },
      "source": [
        "Criando dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZzLfFlyyN5pa"
      },
      "outputs": [],
      "source": [
        "df = pd.DataFrame(docs,columns=docs[0].keys())\n",
        "train, test = train_test_split(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Buiwu8uzN-KD"
      },
      "outputs": [],
      "source": [
        "def split_sentence(sentence: str):\n",
        "    words = nlp(sentence)\n",
        "    lemma = []\n",
        "    pos = []\n",
        "    for word in words:\n",
        "        lemma.append(word.lemma_)\n",
        "        pos.append(word.pos_)\n",
        "    return lemma, pos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4h3mZkLiuOJd"
      },
      "outputs": [],
      "source": [
        "def create_bag_of_word(words: List[str]):\n",
        "    bag = {}\n",
        "    unique_words = list(set(words))\n",
        "    for word in unique_words:\n",
        "        bag[word] = 0.0\n",
        "    return bag"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gCCxwQzS2wuc"
      },
      "outputs": [],
      "source": [
        "def bag_count(bag_zero, words):\n",
        "    bag = bag_zero.copy()\n",
        "    for word in words:\n",
        "        bag[word] += 1\n",
        "\n",
        "    return bag"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Po3XTNA748xQ"
      },
      "outputs": [],
      "source": [
        "def sentences_split(sentences: List[str]):\n",
        "    corpus = {\n",
        "        'sentences': [],\n",
        "        'pos': [],\n",
        "        'bags': [],\n",
        "        'features': {},\n",
        "        'b0': {}\n",
        "    }\n",
        "    for i in range(len(sentences)):\n",
        "        lemma, pos = split_sentence(sentences[i])\n",
        "        corpus[\"sentences\"].append(lemma)\n",
        "        corpus[\"pos\"].append(pos)\n",
        "    return corpus"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CgCxR1ww3J1f"
      },
      "source": [
        "### Extração de features baseadas em palavras"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gmAMyLMiR5bL"
      },
      "source": [
        "#### TF/IDF/TF-IDF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ccvStWyRSEX8"
      },
      "outputs": [],
      "source": [
        "def term_frequency(bag, size):\n",
        "    frequency = bag.copy()\n",
        "    for word in bag:\n",
        "        frequency[word] = bag[word] / size\n",
        "    return np.fromiter(frequency.values(), dtype=float) # list(frequency.values())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qrzn3CSxXBZY"
      },
      "outputs": [],
      "source": [
        "def inverse_document_frequency(corpus_bags, bag, bag_zero):\n",
        "    count = bag_zero.copy()\n",
        "    for corpus_bag in corpus_bags:\n",
        "        for word in bag:\n",
        "            if corpus_bag[word] > 0 and bag[word] > 0:\n",
        "                count[word] += 1\n",
        "    size = len(corpus_bags)\n",
        "    for word in count:\n",
        "        if not count[word] == 0:\n",
        "            count[word] = np.log10(size/count[word])\n",
        "\n",
        "    return np.fromiter(count.values(), dtype=float) # list(count.values())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HA3WNOOl3TEF"
      },
      "outputs": [],
      "source": [
        "def feature_TFIDF(corpus: List[str], query: List[str]):\n",
        "    for doc in [corpus, query]:\n",
        "        doc['features']['tf']    = [] # lista com frequência de cada palavra\n",
        "        doc['features']['idf']   = [] # lista com frequência inversa de cada documento\n",
        "        doc['features']['tfidf'] = []\n",
        "\n",
        "    all_sentences: List[str] = [] # Lista de palavras de todas as sentenças\n",
        "    for doc in [corpus, query]:\n",
        "        for sentence_ in doc[\"sentences\"]:\n",
        "            all_sentences += sentence_\n",
        "\n",
        "    bag_zero = create_bag_of_word(all_sentences)\n",
        "    corpus['b0'] = bag_zero\n",
        "    for doc in [corpus, query]:\n",
        "        for i in range(len(doc['sentences'])):\n",
        "            doc['bags'].append(bag_count(bag_zero, doc['sentences'][i]))\n",
        "    \n",
        "    for doc in [corpus, query]:\n",
        "        for i in range(len(doc['bags'])):\n",
        "            tf = term_frequency(doc['bags'][i], len(doc['sentences'][i]))\n",
        "            idf = inverse_document_frequency(corpus['bags'],doc['bags'][i], bag_zero)\n",
        "            doc['features']['tf'].append(tf)\n",
        "            doc['features']['idf'].append(idf)\n",
        "            doc['features']['tfidf'].append(tf * idf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xmL80ivCSEFD"
      },
      "source": [
        "####Letra maíscula e nome proprio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bb0l4PoERvpz"
      },
      "outputs": [],
      "source": [
        "def feature_UPPER_PROP(corpus: List[str], query: List[str]):\n",
        "    for doc in [corpus, query]:\n",
        "        doc['features']['upper'] = []\n",
        "        doc['features']['prop'] = []\n",
        "        for i in range(len(doc['pos'])):\n",
        "            bag = doc['bags'][i]\n",
        "            count_upper = 0\n",
        "            count_prop = 0\n",
        "            for ii in range(len(doc['pos'][i])):\n",
        "                word = doc['sentences'][i][ii]\n",
        "                if word[0].isupper(): \n",
        "                    count_upper += bag[word]\n",
        "                if doc['pos'][i][ii] == 'PROPN':\n",
        "                    count_prop += bag[word]\n",
        "\n",
        "            doc['features']['upper'].append(count_upper)\n",
        "            doc['features']['prop'].append(count_prop)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZqZliWwTjMMF"
      },
      "source": [
        "###Extração de features baseadas em Sentenças"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0wxXkysw5J8H"
      },
      "source": [
        "####Conter númerais\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fsz7A2ELjXZm"
      },
      "outputs": [],
      "source": [
        "def feature_CONTAIN_NUMBER(corpus: List[str], query: List[str]):\n",
        "    for doc in [corpus, query]:\n",
        "        doc['features']['number'] = []\n",
        "        \n",
        "        for pos_list in doc['pos']:\n",
        "            contains = 0\n",
        "            for pos in pos_list:\n",
        "                if pos == 'NUM':\n",
        "                    contains = 1\n",
        "                    break\n",
        "\n",
        "            doc['features']['number'].append(contains)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RItC5OqO5aZu"
      },
      "source": [
        "####Tamanho da sentença\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3ZPy5LkM5hk3"
      },
      "outputs": [],
      "source": [
        "def feature_LENGTH(corpus: List[str], query: List[str]):\n",
        "    for doc in [corpus, query]:\n",
        "        doc['features']['length'] = []\n",
        "    \n",
        "        for sentence in doc['sentences']:\n",
        "            doc['features']['length'].append(len(sentence))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PXP-p24WDLa0"
      },
      "source": [
        "####Posição da sentença\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Whq6-pfZUqhj"
      },
      "outputs": [],
      "source": [
        "def feature_POSITION(corpus: List[str], query: List[str], metric='INV'):\n",
        "    for doc in [corpus, query]:\n",
        "        l = len(doc['sentences'])\n",
        "        position = []\n",
        "        if metric == 'ORD': position = np.array(range(1,l+1), dtype=float)\n",
        "        elif metric == 'INV': position = np.array(range(l+1,1,-1), dtype=float)\n",
        "        elif metric == 'MID':\n",
        "            start = np.array(range(1,int(l/2)), dtype=float)\n",
        "            end = np.array(range(int(l/2)-1, 0,-1), dtype=float)\n",
        "            position = np.concatenate(start,end)\n",
        "        elif metric == 'EXT': \n",
        "            start = np.array(range(1,int(l/2)), dtype=float)\n",
        "            end = np.array(range(int(l/2)-1, 0,-1), dtype=float)\n",
        "            position = np.concatenate(end,start)\n",
        "            \n",
        "        doc['features']['position'] = position"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c0MdJT4W78yW"
      },
      "source": [
        "####Quantidade de conjunções\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m38JuP1G8IkH"
      },
      "outputs": [],
      "source": [
        "def feature_CUE_PHRASES(corpus: List[str], query: List[str]):\n",
        "    for doc in [corpus, query]:\n",
        "        doc['features']['cue'] = []\n",
        "    \n",
        "        for bag in doc['bags']:\n",
        "            count_cue = 0\n",
        "            for word in bag:\n",
        "                if doc['pos'][0] == 'CONJ':\n",
        "                    count_cue += bag[word]\n",
        "\n",
        "            doc['features']['cue'].append(count_cue)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aPfJnJ8kE531"
      },
      "outputs": [],
      "source": [
        "def stract_info(sentences: List[str], analysed_sentence: List[str]):\n",
        "    sentences = sentences_split(sentences)\n",
        "    analysed_sentence = sentences_split(analysed_sentence)\n",
        "    \n",
        "    # Feature baseados em palavras\n",
        "    feature_TFIDF(sentences, analysed_sentence)\n",
        "    feature_UPPER_PROP(sentences, analysed_sentence) \n",
        "\n",
        "    # Feature baseados em sentenças\n",
        "    feature_CUE_PHRASES(sentences, analysed_sentence)\n",
        "    feature_CONTAIN_NUMBER(sentences, analysed_sentence)\n",
        "    feature_LENGTH(sentences, analysed_sentence)\n",
        "    feature_POSITION(sentences, analysed_sentence)\n",
        "\n",
        "    return analysed_sentence['features']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jlSaqXsT3hiM",
        "outputId": "172486bf-f3ff-45fa-a620-77f6287c69a0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'tf': [array([0.14285714, 0.        , 0.14285714, ..., 0.14285714, 0.        ,\n",
              "         0.14285714])],\n",
              " 'idf': [array([0.60205999, 0.        , 0.60205999, ..., 0.        , 0.        ,\n",
              "         0.        ])],\n",
              " 'tfidf': [array([0.08600857, 0.        , 0.08600857, ..., 0.        , 0.        ,\n",
              "         0.        ])],\n",
              " 'upper': [2.0],\n",
              " 'prop': [1.0],\n",
              " 'cue': [0],\n",
              " 'number': [0],\n",
              " 'length': [7],\n",
              " 'position': array([2.])}"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stract_info(\n",
        "    [\n",
        "        'O Rodemarck restaurante italiano tem uma ÓTIMA pizza',\n",
        "        'Restaurante americano tem o melhor hamburguer',\n",
        "        'Restaurante coreano tem o melhor bibimbap',\n",
        "        'O MELHOR dos melhores restaurantes é americano'\n",
        "    ],\n",
        "    [\n",
        "        'O MELHOR dos Melhores restaurantes é americano',\n",
        "    ]\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gFq2-0YHYp4t"
      },
      "source": [
        "### Aplica heuristicas no score de cada frase"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WDXxRcIfMZ8S"
      },
      "outputs": [],
      "source": [
        "def apply_heuristic_score(raw_score: Dict,      # Pontuação de cada sentença com algoritmo de grafo \n",
        "                          sentences_info: Dict, # Informações extraidas das sentenças\n",
        "                          heuristics: List[str], # Lista de quais heurísticas aplicar\n",
        "                          print_scores = False,\n",
        "                          ):\n",
        "  # positives = ['tf+', 'idf+', 'tfidf+', 'upper+', 'prop+', 'cue+', 'number+', 'length+', 'position+']\n",
        "  # negatives = ['tf-', 'idf-', 'tfidf-', 'upper-', 'prop-', 'cue-', 'number-', 'length-', 'position-']\n",
        "\n",
        "  scores = list(raw_score.values())\n",
        "\n",
        "  # Soma os tf/idf/tfidf das palavras para gerar o valor da frase\n",
        "  sentences_info['tf'] = list(map(sum, sentences_info['tf']))\n",
        "  sentences_info['idf'] = list(map(sum, sentences_info['idf']))\n",
        "  sentences_info['tfidf'] = list(map(sum, sentences_info['tfidf']))\n",
        "\n",
        "  if (print_scores): print(f\"Pontuação antes -> {scores}\")\n",
        "\n",
        "  for i in range(len(raw_score)):\n",
        "    for heuristic in heuristics:\n",
        "      positive_negative = heuristic[-1] # Pega a parte + ou -\n",
        "      positive_negative = +1 if positive_negative == \"+\" else -1\n",
        "\n",
        "      name = heuristic[:-1] # Pega o nome da heuristica\n",
        "\n",
        "      # print(sentences_info[name][i])\n",
        "      scores[i] += sentences_info[name][i] * positive_negative # Atualiza o score\n",
        "\n",
        "  if (print_scores): print(f\"Pontuação depois -> {scores}\")\n",
        "\n",
        "  return scores\n",
        "\n",
        "# score[i] += sentences_info['tf'][i] * positive_negative\n",
        "# score[i] += sentences_info['idf'][i] * positive_negative\n",
        "# score[i] += sentences_info['tfidf'][i] * positive_negative\n",
        "# score[i] += sentences_info['upper'][i] * positive_negative\n",
        "# score[i] += sentences_info['prop'][i] * positive_negative\n",
        "# score[i] += sentences_info['cue'][i] * positive_negative\n",
        "# score[i] += sentences_info['number'][i] * positive_negative\n",
        "# score[i] += sentences_info['length'][i] * positive_negative\n",
        "# score[i] += sentences_info['position'][i] * positive_negative\n",
        "\n",
        "# if 'tf+' in heuristics:\n",
        "\n",
        "# if 'tf-' in heuristics:\n",
        "#   score[i] -= sentences_info['tf'][i]\n",
        "\n",
        "# if '' in heuristics:\n",
        "#   score[i] -= sentences_info['tf'][i]\n",
        "# if 'tfidf+' in heuristics:\n",
        "# for key in sentences_info:\n",
        "#     print(len(sentences_info[key]))\n",
        "#   print(len(df[\"processed_sentences\"]))\n",
        "# # teste\n",
        "# all_words = map(lambda word: word.lower(), NLTK_word_tokenize(' '.join(sentences)))\n",
        "# all_words = list(set(all_words))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-VFnkycIZ_Zf"
      },
      "source": [
        "# Geração da matrix de similaridade entre cada par de sentenças usando BOW / TF / TF-IDF\n",
        "\n",
        "BOW &emsp; 👉 Apenas frequênzia <br>\n",
        "TF  &emsp; 👉 Frequência dividida pelo tamanho da frase <br>\n",
        "TF-IDF     👉 TF * IDF\n",
        "\n",
        "https://pt.wikipedia.org/wiki/Tf%E2%80%93idf\n",
        "\n",
        "https://drive.google.com/file/d/1fCAJo2s6fu7edE0jiP3VUW9Bxo3aSGge/view\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n2ro5_ZhZ7YW"
      },
      "outputs": [],
      "source": [
        "def build_similarity_matrix(sentences: List[str], similarity_function: Callable):\n",
        "\n",
        "  len_sentences = len(sentences)\n",
        "\n",
        "  # Create an empty similarity matrix\n",
        "  similarity_matrix = np.zeros( shape=(len_sentences, len_sentences) )\n",
        "\n",
        "  for idx1 in range(len_sentences):\n",
        "    for idx2 in range(len_sentences):\n",
        "      if idx1 == idx2: \n",
        "        continue # ignore if both are same sentences\n",
        "\n",
        "      similarity_matrix[idx1][idx2] = similarity_function(sentences[idx1], sentences[idx2])\n",
        "\n",
        "  return similarity_matrix\n",
        "\n",
        "################################################################################\n",
        "\n",
        "def sentence_similarity_by_BOW(sentence_1, sentence_2):\n",
        "\n",
        "    splited_sentence_1 = NLTK_word_tokenize(sentence_1)\n",
        "    splited_sentence_2 = NLTK_word_tokenize(sentence_2)\n",
        " \n",
        "    all_words = splited_sentence_1 + splited_sentence_2\n",
        "\n",
        "    bag_of_word_1 = create_bag_of_word(all_words)\n",
        "    bag_of_word_2 = create_bag_of_word(all_words)\n",
        " \n",
        "    for word in splited_sentence_1:\n",
        "        bag_of_word_1[word] += 1\n",
        "        \n",
        "    for word in splited_sentence_2:\n",
        "        bag_of_word_2[word] += 1\n",
        "\n",
        "    vector_1 = list(bag_of_word_1.values())\n",
        "    vector_2 = list(bag_of_word_2.values())\n",
        "\n",
        "    return 1 - NLTK_cosine_distance(vector_1, vector_2)\n",
        "\n",
        "################################################################################\n",
        "\n",
        "def sentence_similarity_by_TF(sentence_1, sentence_2):\n",
        "\n",
        "    splited_sentence_1 = NLTK_word_tokenize(sentence_1)\n",
        "    splited_sentence_2 = NLTK_word_tokenize(sentence_2)\n",
        " \n",
        "    all_words = splited_sentence_1 + splited_sentence_2\n",
        "\n",
        "    bag_of_word_1 = create_bag_of_word(all_words)\n",
        "    bag_of_word_2 = create_bag_of_word(all_words)\n",
        " \n",
        "    for word in splited_sentence_1:\n",
        "        bag_of_word_1[word] += 1\n",
        "        \n",
        "    for word in splited_sentence_2:\n",
        "        bag_of_word_2[word] += 1\n",
        "\n",
        "    for word in splited_sentence_1:\n",
        "        bag_of_word_1[word] = bag_of_word_1[word] / len(splited_sentence_1)\n",
        "        \n",
        "    for word in splited_sentence_2:\n",
        "        bag_of_word_2[word] = bag_of_word_2[word] / len(splited_sentence_2)\n",
        "    \n",
        "    vector_1 = list(bag_of_word_1.values())\n",
        "    vector_2 = list(bag_of_word_2.values())\n",
        "\n",
        "    return 1 - NLTK_cosine_distance(vector_1, vector_2)\n",
        "\n",
        "################################################################################\n",
        "\n",
        "def sentence_similarity_by_TFIDF(sentence_1, sentence_2):\n",
        "\n",
        "    splited_sentence_1 = NLTK_word_tokenize(sentence_1)\n",
        "    splited_sentence_2 = NLTK_word_tokenize(sentence_2)\n",
        " \n",
        "    all_words = splited_sentence_1 + splited_sentence_2\n",
        "\n",
        "    bag_of_word_1_idf = create_bag_of_word(all_words)\n",
        "    bag_of_word_2_idf = create_bag_of_word(all_words)\n",
        "\n",
        "    bag_of_word_1_tf = create_bag_of_word(all_words)\n",
        "    bag_of_word_2_tf = create_bag_of_word(all_words)\n",
        " \n",
        "    for word in splited_sentence_1:\n",
        "        bag_of_word_1_idf[word] += 1\n",
        "        bag_of_word_2_idf[word] += 1\n",
        "\n",
        "        bag_of_word_1_tf[word] += 1\n",
        "        \n",
        "    for word in splited_sentence_2:\n",
        "        bag_of_word_1_idf[word] += 1\n",
        "        bag_of_word_2_idf[word] += 1\n",
        "\n",
        "        bag_of_word_2_tf[word] += 1\n",
        "\n",
        "    for word in splited_sentence_1:\n",
        "        bag_of_word_1_tf[word] = bag_of_word_1_tf[word] / len(splited_sentence_1)\n",
        "        \n",
        "    for word in splited_sentence_2:\n",
        "        bag_of_word_2_tf[word] = bag_of_word_2_tf[word] / len(splited_sentence_2)\n",
        "\n",
        "    M = 3 # 2 sentenças +1\n",
        "    for word in all_words:\n",
        "        if bag_of_word_1_idf[word] > 0: \n",
        "            bag_of_word_1_idf[word] = np.log10( M / bag_of_word_1_idf[word] )\n",
        "\n",
        "        if bag_of_word_2_idf[word] > 0:\n",
        "            bag_of_word_2_idf[word] = np.log10( M / bag_of_word_2_idf[word] )\n",
        "    \n",
        "    idf_1 = np.fromiter(bag_of_word_1_idf.values(), dtype=float)\n",
        "    idf_2 = np.fromiter(bag_of_word_2_idf.values(), dtype=float)\n",
        "\n",
        "    tf_1 = np.fromiter(bag_of_word_1_tf.values(), dtype=float)\n",
        "    tf_2 = np.fromiter(bag_of_word_2_tf.values(), dtype=float)\n",
        "\n",
        "    vector_1 = tf_1 * idf_1\n",
        "    vector_2 = tf_2 * idf_2\n",
        "\n",
        "    return 1 - NLTK_cosine_distance(vector_1, vector_2)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V5lzzNR23Utp"
      },
      "source": [
        "# Pontuação de sentenças com grafo de similaridade\n",
        "\n",
        "Documentação: https://networkx.org/documentation/stable/tutorial.html\n",
        "\n",
        "Rank em Grafo:\n",
        " - Bushy path of the node\n",
        " - Similaridade agregada\n",
        " - [Page Pank](https://youtu.be/MG0fIXfrT9A?t=114)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Enb7zcE83Y6z"
      },
      "outputs": [],
      "source": [
        "def bush_path(similarity_graph: nx.Graph, threshold = 0.5) -> Dict[str, float]:\n",
        "  G = similarity_graph\n",
        "\n",
        "  scores = {\n",
        "      # Sentença/Node: Score\n",
        "  }\n",
        "\n",
        "  for node in G.nodes():\n",
        "    scores[node] = 0 # Inicializa a pontuação de todas as sentenças\n",
        "\n",
        "    for neighbor_node in G.neighbors(node):\n",
        "      similarity = G.get_edge_data(node, neighbor_node)['weight']\n",
        "      \n",
        "      if similarity > threshold:\n",
        "        scores[node] += 1\n",
        "  \n",
        "  return scores\n",
        "\n",
        "  # rows, cols = similarity_matrix.shape\n",
        "      \n",
        "  # apply_threshold = lambda value: 1 if value > threshold else 0\n",
        "  # vectorized_apply_threshold = np.vectorize(apply_threshold)\n",
        "\n",
        "  # thresholded_matrix = vectorized_apply_threshold(similarity_matrix)\n",
        "\n",
        "  # print(thresholded_matrix)\n",
        "\n",
        "  # Iteração sobre o triângulo de cima da matrix simétrica\n",
        "  # https://people.revoledu.com/kardi/tutorial/VB/tips/Symmetric-Matrix.html\n",
        "  # for i in range(1, rows -1):\n",
        "  #   for j in range(i +1, cols):\n",
        "  #     if similarity_matrix[i, j] > threshold:\n",
        "  #       pass\n",
        "\n",
        "################################################################################\n",
        "\n",
        "def similaridade_agregada(similarity_graph: nx.Graph) -> Dict[str, float]:\n",
        "  G = similarity_graph\n",
        "\n",
        "  scores = {\n",
        "      # Sentença/Node: Score\n",
        "  }\n",
        "\n",
        "  for node in G.nodes():\n",
        "    scores[node] = 0 # Inicializa a pontuação de todas as sentenças\n",
        "\n",
        "    for neighbor_node in G.neighbors(node):\n",
        "      similarity = G.get_edge_data(node, neighbor_node)['weight']\n",
        "      scores[node] += similarity\n",
        "  \n",
        "  return scores\n",
        "\n",
        "################################################################################\n",
        "\n",
        "def text_rank(similarity_graph: nx.Graph) -> Dict[str, float]:\n",
        "  return nx.pagerank(similarity_graph)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wn97lYS1aEmc"
      },
      "source": [
        "# Sumarização com Algoritmos de Resumo em Grafo + Heurísticas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 743,
          "referenced_widgets": [
            "f9e672c2fad948dea720dff1e4067200",
            "7624fc3082a543df9aa2c7648519b942",
            "716f10a6d2344c9ca5281225a39139e8",
            "66601b493d104fae9a49086aa67ded2d",
            "513ca112a32544fe8b4b6dd74a6026d0",
            "87c5903b232143daa74f7d015fbb1a5e",
            "4964ff7abe15444c9a7b62bc9e66a166",
            "376971c5a9d841caa01486a459227973",
            "5042779160bd4b9eb59b0420bda65a57",
            "86fd88c6db7a4a518e42b03b4b30f83c",
            "2da2a866a48d4e77891746ac5e841dd3",
            "e973190dbe614dd7b3166a637dd6c128",
            "868cf916b8c449eebfeef9f273393255",
            "954e2b406ce14c118a88fdf68805e504",
            "e703a6a38a8a4e52b2b667fc8a072149",
            "d9378a10007f459d97bfca81db498ac6",
            "10c37ea9fa3d4566a67f8682c0066657",
            "34b5727c18194e8093818d3239222202",
            "cbd0d32997f34c968e4d776c4c0c3447",
            "6f2c5ac6218c4597936cbe0f7a721421",
            "ffa4f595d39d4ae7abc8e70c5ca11b3f",
            "0739744961d947d6a081815fbe018817",
            "275dcab0715443dba3d6bd3b3c68fbc8",
            "621a5c8ed16943be951b5e52ea615afa",
            "4bfd5aa55dcd4b41a7188389d5b8d38b",
            "09d6b87a55a64ce99b34226cdd85667c",
            "f67e1c6cb5ab4dc4b9732cef42e36db0",
            "baca034e01864ac68ebef601b47e2301",
            "97a5c1da276543d1ae4a9ed2f8a52c1d",
            "62e00a80351d4ba9a4a440023f0c21f1",
            "9a6738557bb743338ba5c88d118c43c6",
            "2afb0d88ec404b3a9acbac06b4f9bc49",
            "94f3c8d171aa4673a359e9e8428b070e",
            "420ad5b967784da6a2376493d7e5e869",
            "45751c7ea6f743718d004af57c9c18a0",
            "37b10ae1a439425a9d9f84ba9fea557e",
            "61badc70be6e4e7f8c74658e6a4eb841",
            "67cd2b6d951d42b09fc4685f806f0548",
            "16e569df09194ae09efd13ef89c52df5",
            "e5207743eaa948deb199e7cf6476c3e6",
            "dae729d25f014a268b1ac044a62e2b95",
            "6ae53a1bdd6b4dbb807e73a47c0207f9",
            "aabdad6cfe2a48139cb9f1772acada72",
            "34fa7e9fb2e9475bb13912645f4aef5a",
            "f9de4d6f438b4ccf88c9ca2a6a542f43",
            "7271cd6401b646859e6ffb179a746e74",
            "9168fb9351634741a7b6b2a19d13f72f",
            "555547d9a166409e84a85af41b24d0c2",
            "dc0575a4ff064a1e8a97ed6903108427"
          ]
        },
        "id": "JjauV7gI97kC",
        "outputId": "edecd817-903f-468d-bc70-2f84c0aff8a4"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f9e672c2fad948dea720dff1e4067200",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "interactive(children=(IntSlider(value=0, continuous_update=False, description='doc_index', max=10573), Checkbo…"
            ]
          },
          "metadata": {
            "application/vnd.jupyter.widget-view+json": {
              "colab": {
                "custom_widget_manager": {
                  "url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/d2e234f7cc04bf79/manager.min.js"
                }
              }
            }
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "#--------------------#\n"
          ]
        }
      ],
      "source": [
        "def generate_summary(doc_index: int, \n",
        "                     return_text: bool,\n",
        "                     filter_stop_word: bool,\n",
        "                     lemmatize: bool,\n",
        "                     join: bool,\n",
        "                     show_matrix: bool,\n",
        "                     print_scores: bool,\n",
        "                     heuristics: List[str], \n",
        "                     section_names: List[str], \n",
        "                     RegEx: str, \n",
        "                     algorithm: str,\n",
        "                     threshold: float,\n",
        "                     similarity: str,\n",
        "                     top_n: int,\n",
        "                     validation: str\n",
        "                     ):\n",
        "  \n",
        "  doc = docs[doc_index] # Documento que vai ser resumido\n",
        "\n",
        "  ### Step 1 - Junta o texto das seções do documento que vai ser resumido\n",
        "  text = list(map(lambda section_name: doc[section_name], section_names))\n",
        "  text = \". \".join(text)\n",
        "\n",
        "  ### Step 2 - Divide o texto em sentenças\n",
        "  if (RegEx):\n",
        "    tokenizer = NLTK_RegexpTokenizer(RegEx)\n",
        "    sentences = tokenizer.tokenize(text) # Com regex\n",
        "  else:\n",
        "    sentences = NLTK_sent_tokenize(text) # Sem regex\n",
        "    sentences: List[str] = correct_sentences(sentences)\n",
        "  \n",
        "\n",
        "  # Declara dataframe de auxílio\n",
        "  df = pd.DataFrame()\n",
        "  df[\"original_sentences\"]  = sentences\n",
        "  df[\"processed_sentences\"] = sentences.copy() # Começa como uma cópia do original\n",
        "  df[\"sentence_scores\"]     = [0.0] * len(sentences) \n",
        "\n",
        "\n",
        "  ### Step 3 - Aplica Pre-processamento - Remove frase pequena - Filtra stop word - Lematiza o texto\n",
        "  df = df[df[\"original_sentences\"].map(NLTK_word_tokenize).map(len) >= 5]\n",
        "  df.reset_index(drop = False, inplace = True)\n",
        "\n",
        "  if (filter_stop_word):\n",
        "    df[\"processed_sentences\"] = df[\"processed_sentences\"].apply(\n",
        "        lambda sentence: \" \".join(list(filter(lambda word: word.lower() not in STOP_WORDS, NLTK_word_tokenize(sentence))))\n",
        "    )\n",
        "\n",
        "  if (lemmatize):\n",
        "    df[\"processed_sentences\"] = df[\"processed_sentences\"].apply(\n",
        "        lambda sentence: \" \".join(list(map(lambda word: word.lemma_, nlp(sentence))))\n",
        "    )\n",
        "  \n",
        "  ### Step 4 - Gerar matrix de similaridade entre sentenças\n",
        "  if similarity == \"BOW\":\n",
        "    similarity_function = sentence_similarity_by_BOW\n",
        "  elif similarity == \"TF\":\n",
        "    similarity_function = sentence_similarity_by_TF\n",
        "  elif similarity == \"TF-IDF\":\n",
        "    similarity_function = sentence_similarity_by_TFIDF\n",
        "\n",
        "  sentence_similarity_martix = build_similarity_matrix(df[\"processed_sentences\"], similarity_function)\n",
        "  if (show_matrix): \n",
        "    print(\"=\" + \"-\"*20 + \"=\")\n",
        "    print(sentence_similarity_martix)\n",
        "    print(\"=\" + \"-\"*20 + \"=\")\n",
        "\n",
        "  ### Step 5 - Pontua sentença usando a matrix de similaridade no algoritmo escolhido\n",
        "  if algorithm == \"Bush Path Node\":\n",
        "    algorithm_function = bush_path\n",
        "  elif algorithm == \"Similaridade Agregada\":\n",
        "    algorithm_function = similaridade_agregada\n",
        "  elif algorithm == \"Text Rank\":\n",
        "    algorithm_function = text_rank\n",
        "  \n",
        "  sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)\n",
        "  args = {\n",
        "      \"Bush Path Node\":        [sentence_similarity_graph, threshold,],\n",
        "      \"Similaridade Agregada\": [sentence_similarity_graph,],\n",
        "      \"Text Rank\":             [sentence_similarity_graph,]\n",
        "  }\n",
        "  args = args[algorithm]\n",
        "  raw_scores = algorithm_function(*args)\n",
        "\n",
        "  ### Stop 6 - Extrair informações sobre cada sentença e pontuar a heurística\n",
        "  sentences_info = stract_info([], df[\"processed_sentences\"])\n",
        "\n",
        "  ### Stop 7 - Pontuar sentenças com heurística\n",
        "  final_score = apply_heuristic_score(raw_scores, sentences_info, heuristics, print_scores)\n",
        "  df[\"sentence_scores\"] = final_score\n",
        "  # df[\"sentence_scores\"] = list(raw_scores.values())\n",
        "\n",
        "  ### Step 8 - Ordena as sentenças por pontuação\n",
        "  df.sort_values(by=[\"sentence_scores\"], ascending=False, inplace=True)\n",
        "\n",
        "  ### Step 9 - Escolhe as melhores sentences\n",
        "  summarize_text = df[\"original_sentences\"][:top_n]\n",
        "  out_text = \" ⚪ \".join(summarize_text) if join else summarize_text\n",
        "\n",
        "  ### Step 10 - Avalia a pontuação do resumo\n",
        "  print(\"=\" + \"-\"*20 + \"=\")\n",
        "  if validation == \"ROUGE\":\n",
        "    score = scorer.score(\" \".join(summarize_text), doc['ementa'])\n",
        "    p1, r1, f1 = score['rouge1'].precision, score['rouge1'].recall, score['rouge1'].fmeasure\n",
        "    p2, r2, f2 = score['rouge2'].precision, score['rouge2'].recall, score['rouge2'].fmeasure\n",
        "    pL, rL, fL = score['rougeL'].precision, score['rougeL'].recall, score['rougeL'].fmeasure\n",
        "    print(f\"SCORE ROUGE1 -> precision = {p1:.2f}, recall = {p1:.2f}, fmeasure = {p1:.2f}\")\n",
        "    print(f\"SCORE ROUGE2 -> precision = {p2:.2f}, recall = {p2:.2f}, fmeasure = {p2:.2f}\")\n",
        "    print(f\"SCORE ROUGEL -> precision = {pL:.2f}, recall = {pL:.2f}, fmeasure = {pL:.2f}\")\n",
        "  # elif validation == \"Pyramid\":\n",
        "  #   pass\n",
        "  # elif validation == \"Sentence Mapping\":\n",
        "  #   pass\n",
        "  print(\"=\" + \"-\"*20 + \"=\")\n",
        "\n",
        "\n",
        "  return out_text if return_text else score\n",
        "\n",
        "################################################################################\n",
        "heuristics_options = [\n",
        "    (\"TF+\", \"tf+\"), (\"IDF+\", \"idf+\"), (\"TF-IDF+\", \"tfidf+\"), (\"Upper+\", \"upper+\"), (\"Prop+\", \"prop+\"), (\"Cue Phrases+\", \"cue+\"), (\"Have Number+\", \"number+\"), (\"Length+\", \"length+\"), (\"Position+\", \"position+\"),\n",
        "    (\"TF-\", \"tf-\"), (\"IDF-\", \"idf-\"), (\"TF-IDF-\", \"tfidf-\"), (\"Upper-\", \"upper-\"), (\"Prop-\", \"prop-\"), (\"Cue Phrases-\", \"cue-\"), (\"Have Number-\", \"number-\"), (\"Length-\", \"length-\"), (\"Position-\", \"position-\"),\n",
        "]\n",
        "interact(\n",
        "    generate_summary,\n",
        "\n",
        "    filter_stop_word = Checkbox(description=\"Filter Stop Word\", value=True),\n",
        "    return_text      = Checkbox(description=\"Return Text\", value=True),\n",
        "    lemmatize        = Checkbox(description=\"Preprocess with lemma\", value=True),\n",
        "    join             = Checkbox(description=\"Return joined sentences\", value=True),\n",
        "    show_matrix      = Checkbox(description=\"Show similarity matrix\", value=False),\n",
        "    print_scores     = Checkbox(description=\"Show heuristic score change\", value=False),\n",
        "    doc_index        = IntSlider(continuous_update=False, min=0, max=len(docs)-1, step=1, value=0),\n",
        "    heuristics       = SelectMultiple(options=heuristics_options, description='Heuristicas', value=[(\"upper+\")]),\n",
        "    section_names    = SelectMultiple(options=[(\"Ementa\", \"ementa\"), (\"Relatório\", \"relatorio\"), (\"Area\", \"area\"), (\"Relator\", \"relator\"), (\"Voto\", \"voto\"), (\"Acordão\", \"acordao\"), (\"Classe\", \"classe\"), (\"Extrato\", \"extrato\") ], description='Ctrl + Click', value=[(\"relatorio\")]),\n",
        "    RegEx            = Text(value=\"\", description=\"RegEx\", placeholder=\"Digite RegEx aqui\"),\n",
        "    algorithm        = Dropdown(options=[\"Bush Path Node\", \"Similaridade Agregada\", \"Text Rank\"], description=\"Algoritmo\", value=\"Bush Path Node\"),\n",
        "    threshold        = FloatSlider(continuous_update=False, description=\"Limiar Bush\", min = 0.0, max = 1.0, step = 0.05, value = 0.5), # if algorithm == \"Bush Path Node\" else 0.5\n",
        "    similarity       = Dropdown(options=[\"BOW\", \"TF\", \"TF-IDF\"], description=\"Similaridade\", value=\"BOW\"),\n",
        "    top_n            = IntSlider(continuous_update=False, min=1, max=15, step=1, value=3),\n",
        "    validation       = Dropdown(options=[\"ROUGE\"], description=\"Validação\", value=\"ROUGE\"), # , \"Pyramid\", \"Sentence Mapping\"\n",
        ")\n",
        "print(\"#\" + \"-\"*20 + \"#\") # doc_index: 1278"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_EcM0Uc5O8ZN"
      },
      "source": [
        "## Resume e cálcula o score de 200 documentos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p7SB8AoyNFNW",
        "outputId": "c9b79093-8811-4de3-e5ee-f6b454ecb3b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.86, recall = 0.86, fmeasure = 0.86\n",
            "SCORE ROUGE2 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGEL -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.85, recall = 0.85, fmeasure = 0.85\n",
            "SCORE ROUGE2 -> precision = 0.76, recall = 0.76, fmeasure = 0.76\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGEL -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGEL -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.86, recall = 0.86, fmeasure = 0.86\n",
            "SCORE ROUGE2 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGE2 -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGE2 -> precision = 0.04, recall = 0.04, fmeasure = 0.04\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGEL -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGE2 -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "SCORE ROUGEL -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.05, recall = 0.05, fmeasure = 0.05\n",
            "SCORE ROUGE2 -> precision = 0.02, recall = 0.02, fmeasure = 0.02\n",
            "SCORE ROUGEL -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGEL -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.61, recall = 0.61, fmeasure = 0.61\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGEL -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGEL -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.61, recall = 0.61, fmeasure = 0.61\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.91, recall = 0.91, fmeasure = 0.91\n",
            "SCORE ROUGE2 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGEL -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:46: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.82, recall = 0.82, fmeasure = 0.82\n",
            "SCORE ROUGE2 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGEL -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGEL -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.86, recall = 0.86, fmeasure = 0.86\n",
            "SCORE ROUGE2 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "SCORE ROUGEL -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.85, recall = 0.85, fmeasure = 0.85\n",
            "SCORE ROUGE2 -> precision = 0.76, recall = 0.76, fmeasure = 0.76\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGEL -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.76, recall = 0.76, fmeasure = 0.76\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGEL -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.86, recall = 0.86, fmeasure = 0.86\n",
            "SCORE ROUGE2 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGE2 -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGE2 -> precision = 0.04, recall = 0.04, fmeasure = 0.04\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.82, recall = 0.82, fmeasure = 0.82\n",
            "SCORE ROUGE2 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGE2 -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGEL -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "SCORE ROUGEL -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.05, recall = 0.05, fmeasure = 0.05\n",
            "SCORE ROUGE2 -> precision = 0.02, recall = 0.02, fmeasure = 0.02\n",
            "SCORE ROUGEL -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGEL -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.61, recall = 0.61, fmeasure = 0.61\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.76, recall = 0.76, fmeasure = 0.76\n",
            "SCORE ROUGE2 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGEL -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGEL -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGEL -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGEL -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.91, recall = 0.91, fmeasure = 0.91\n",
            "SCORE ROUGE2 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGEL -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.82, recall = 0.82, fmeasure = 0.82\n",
            "SCORE ROUGE2 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGEL -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGEL -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.82, recall = 0.82, fmeasure = 0.82\n",
            "SCORE ROUGE2 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGEL -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGEL -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGEL -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.85, recall = 0.85, fmeasure = 0.85\n",
            "SCORE ROUGE2 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGEL -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGE2 -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGE2 -> precision = 0.04, recall = 0.04, fmeasure = 0.04\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGEL -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGE2 -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.86, recall = 0.86, fmeasure = 0.86\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.84, recall = 0.84, fmeasure = 0.84\n",
            "SCORE ROUGE2 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGEL -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.68, recall = 0.68, fmeasure = 0.68\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "SCORE ROUGEL -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.66, recall = 0.66, fmeasure = 0.66\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.54, recall = 0.54, fmeasure = 0.54\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.05, recall = 0.05, fmeasure = 0.05\n",
            "SCORE ROUGE2 -> precision = 0.02, recall = 0.02, fmeasure = 0.02\n",
            "SCORE ROUGEL -> precision = 0.03, recall = 0.03, fmeasure = 0.03\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.76, recall = 0.76, fmeasure = 0.76\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGEL -> precision = 0.49, recall = 0.49, fmeasure = 0.49\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.69, recall = 0.69, fmeasure = 0.69\n",
            "SCORE ROUGE2 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "SCORE ROUGEL -> precision = 0.52, recall = 0.52, fmeasure = 0.52\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "SCORE ROUGE2 -> precision = 0.27, recall = 0.27, fmeasure = 0.27\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.45, recall = 0.45, fmeasure = 0.45\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGEL -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.22, recall = 0.22, fmeasure = 0.22\n",
            "SCORE ROUGEL -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGE2 -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.40, recall = 0.40, fmeasure = 0.40\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.80, recall = 0.80, fmeasure = 0.80\n",
            "SCORE ROUGE2 -> precision = 0.71, recall = 0.71, fmeasure = 0.71\n",
            "SCORE ROUGEL -> precision = 0.56, recall = 0.56, fmeasure = 0.56\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.47, recall = 0.47, fmeasure = 0.47\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGEL -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGEL -> precision = 0.50, recall = 0.50, fmeasure = 0.50\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.32, recall = 0.32, fmeasure = 0.32\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.36, recall = 0.36, fmeasure = 0.36\n",
            "SCORE ROUGE2 -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGEL -> precision = 0.75, recall = 0.75, fmeasure = 0.75\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.44, recall = 0.44, fmeasure = 0.44\n",
            "SCORE ROUGEL -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.01, recall = 0.01, fmeasure = 0.01\n",
            "SCORE ROUGEL -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.38, recall = 0.38, fmeasure = 0.38\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.70, recall = 0.70, fmeasure = 0.70\n",
            "SCORE ROUGE2 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGEL -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.78, recall = 0.78, fmeasure = 0.78\n",
            "SCORE ROUGE2 -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "SCORE ROUGEL -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.63, recall = 0.63, fmeasure = 0.63\n",
            "SCORE ROUGE2 -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "SCORE ROUGEL -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGE2 -> precision = 0.12, recall = 0.12, fmeasure = 0.12\n",
            "SCORE ROUGEL -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.89, recall = 0.89, fmeasure = 0.89\n",
            "SCORE ROUGE2 -> precision = 0.64, recall = 0.64, fmeasure = 0.64\n",
            "SCORE ROUGEL -> precision = 0.59, recall = 0.59, fmeasure = 0.59\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.58, recall = 0.58, fmeasure = 0.58\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGE2 -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "SCORE ROUGEL -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.73, recall = 0.73, fmeasure = 0.73\n",
            "SCORE ROUGE2 -> precision = 0.33, recall = 0.33, fmeasure = 0.33\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.29, recall = 0.29, fmeasure = 0.29\n",
            "SCORE ROUGEL -> precision = 0.39, recall = 0.39, fmeasure = 0.39\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.72, recall = 0.72, fmeasure = 0.72\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "SCORE ROUGE2 -> precision = 0.13, recall = 0.13, fmeasure = 0.13\n",
            "SCORE ROUGEL -> precision = 0.19, recall = 0.19, fmeasure = 0.19\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.51, recall = 0.51, fmeasure = 0.51\n",
            "SCORE ROUGE2 -> precision = 0.17, recall = 0.17, fmeasure = 0.17\n",
            "SCORE ROUGEL -> precision = 0.23, recall = 0.23, fmeasure = 0.23\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.67, recall = 0.67, fmeasure = 0.67\n",
            "SCORE ROUGE2 -> precision = 0.31, recall = 0.31, fmeasure = 0.31\n",
            "SCORE ROUGEL -> precision = 0.43, recall = 0.43, fmeasure = 0.43\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGE2 -> precision = 0.06, recall = 0.06, fmeasure = 0.06\n",
            "SCORE ROUGEL -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.81, recall = 0.81, fmeasure = 0.81\n",
            "SCORE ROUGE2 -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "SCORE ROUGEL -> precision = 0.48, recall = 0.48, fmeasure = 0.48\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.60, recall = 0.60, fmeasure = 0.60\n",
            "SCORE ROUGE2 -> precision = 0.21, recall = 0.21, fmeasure = 0.21\n",
            "SCORE ROUGEL -> precision = 0.26, recall = 0.26, fmeasure = 0.26\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.79, recall = 0.79, fmeasure = 0.79\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.57, recall = 0.57, fmeasure = 0.57\n",
            "SCORE ROUGE2 -> precision = 0.30, recall = 0.30, fmeasure = 0.30\n",
            "SCORE ROUGEL -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.41, recall = 0.41, fmeasure = 0.41\n",
            "SCORE ROUGE2 -> precision = 0.16, recall = 0.16, fmeasure = 0.16\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGEL -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.42, recall = 0.42, fmeasure = 0.42\n",
            "SCORE ROUGE2 -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "SCORE ROUGE2 -> precision = 0.11, recall = 0.11, fmeasure = 0.11\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.14, recall = 0.14, fmeasure = 0.14\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.20, recall = 0.20, fmeasure = 0.20\n",
            "SCORE ROUGE2 -> precision = 0.09, recall = 0.09, fmeasure = 0.09\n",
            "SCORE ROUGEL -> precision = 0.10, recall = 0.10, fmeasure = 0.10\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.08, recall = 0.08, fmeasure = 0.08\n",
            "SCORE ROUGEL -> precision = 0.18, recall = 0.18, fmeasure = 0.18\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.77, recall = 0.77, fmeasure = 0.77\n",
            "SCORE ROUGE2 -> precision = 0.65, recall = 0.65, fmeasure = 0.65\n",
            "SCORE ROUGEL -> precision = 0.55, recall = 0.55, fmeasure = 0.55\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.62, recall = 0.62, fmeasure = 0.62\n",
            "SCORE ROUGE2 -> precision = 0.24, recall = 0.24, fmeasure = 0.24\n",
            "SCORE ROUGEL -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "SCORE ROUGE2 -> precision = 0.07, recall = 0.07, fmeasure = 0.07\n",
            "SCORE ROUGEL -> precision = 0.28, recall = 0.28, fmeasure = 0.28\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.74, recall = 0.74, fmeasure = 0.74\n",
            "SCORE ROUGE2 -> precision = 0.35, recall = 0.35, fmeasure = 0.35\n",
            "SCORE ROUGEL -> precision = 0.46, recall = 0.46, fmeasure = 0.46\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.53, recall = 0.53, fmeasure = 0.53\n",
            "SCORE ROUGE2 -> precision = 0.25, recall = 0.25, fmeasure = 0.25\n",
            "SCORE ROUGEL -> precision = 0.34, recall = 0.34, fmeasure = 0.34\n",
            "=--------------------=\n",
            "=--------------------=\n",
            "SCORE ROUGE1 -> precision = 0.37, recall = 0.37, fmeasure = 0.37\n",
            "SCORE ROUGE2 -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "SCORE ROUGEL -> precision = 0.15, recall = 0.15, fmeasure = 0.15\n",
            "=--------------------=\n"
          ]
        }
      ],
      "source": [
        "args_1 = [False, True, True, True, False, False, ['tfidf+', 'upper+', 'prop+', 'cue+', 'length+', 'position-'], ['relatorio'], '', 'Text Rank', 0.5, 'TF-IDF', 15, 'ROUGE']\n",
        "teste_1_scores = list(map(lambda doc_index: generate_summary(doc_index, *args_1), range(200)))\n",
        "\n",
        "args_2 = [False, True, True, True, False, False, ['tfidf+', 'upper+', 'length+', 'position-'], ['relatorio'], '', 'Text Rank', 0.5, 'TF-IDF', 15, 'ROUGE']\n",
        "teste_2_scores = list(map(lambda doc_index: generate_summary(doc_index, *args_2), range(200)))\n",
        "\n",
        "args_3 = [False, True, True, True, False, False, ['tfidf+', 'upper+', 'prop+', 'cue+'], ['relatorio'], '', 'Bush Path Node', 1.00, 'BOW', 15, 'ROUGE']\n",
        "teste_3_scores = list(map(lambda doc_index: generate_summary(doc_index, *args_3), range(200)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        },
        "id": "jWI2JJegPcaJ",
        "outputId": "90bed6dd-95ee-4326-8429-8904271d7569"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-0ff0281b-01d0-46ee-9345-e7507fb04cf5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ROUGE-1 Precision</th>\n",
              "      <th>ROUGE-1 Recall</th>\n",
              "      <th>ROUGE-1 fmeasure</th>\n",
              "      <th>ROUGE-2 Precision</th>\n",
              "      <th>ROUGE-2 Recall</th>\n",
              "      <th>ROUGE-2 fmeasure</th>\n",
              "      <th>ROUGE-L Precision</th>\n",
              "      <th>ROUGE-L Recall</th>\n",
              "      <th>ROUGE-L Fmeasure</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.532339</td>\n",
              "      <td>0.318858</td>\n",
              "      <td>0.329544</td>\n",
              "      <td>0.250206</td>\n",
              "      <td>0.142621</td>\n",
              "      <td>0.150100</td>\n",
              "      <td>0.306936</td>\n",
              "      <td>0.171036</td>\n",
              "      <td>0.177246</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.199412</td>\n",
              "      <td>0.191353</td>\n",
              "      <td>0.146558</td>\n",
              "      <td>0.155868</td>\n",
              "      <td>0.120452</td>\n",
              "      <td>0.111516</td>\n",
              "      <td>0.147991</td>\n",
              "      <td>0.102693</td>\n",
              "      <td>0.070499</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.045279</td>\n",
              "      <td>0.022487</td>\n",
              "      <td>0.042984</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.034682</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>0.032086</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.381774</td>\n",
              "      <td>0.153717</td>\n",
              "      <td>0.214845</td>\n",
              "      <td>0.136725</td>\n",
              "      <td>0.058357</td>\n",
              "      <td>0.083307</td>\n",
              "      <td>0.187013</td>\n",
              "      <td>0.091293</td>\n",
              "      <td>0.132177</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.548094</td>\n",
              "      <td>0.295104</td>\n",
              "      <td>0.331804</td>\n",
              "      <td>0.219307</td>\n",
              "      <td>0.109915</td>\n",
              "      <td>0.129293</td>\n",
              "      <td>0.283398</td>\n",
              "      <td>0.154151</td>\n",
              "      <td>0.173694</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.708338</td>\n",
              "      <td>0.480780</td>\n",
              "      <td>0.431180</td>\n",
              "      <td>0.335466</td>\n",
              "      <td>0.189126</td>\n",
              "      <td>0.185822</td>\n",
              "      <td>0.408235</td>\n",
              "      <td>0.229795</td>\n",
              "      <td>0.212207</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.913043</td>\n",
              "      <td>0.829146</td>\n",
              "      <td>0.798742</td>\n",
              "      <td>0.762144</td>\n",
              "      <td>0.730479</td>\n",
              "      <td>0.716535</td>\n",
              "      <td>0.754386</td>\n",
              "      <td>0.578125</td>\n",
              "      <td>0.508511</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0ff0281b-01d0-46ee-9345-e7507fb04cf5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0ff0281b-01d0-46ee-9345-e7507fb04cf5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0ff0281b-01d0-46ee-9345-e7507fb04cf5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       ROUGE-1 Precision  ROUGE-1 Recall  ROUGE-1 fmeasure  ROUGE-2 Precision  \\\n",
              "count         200.000000      200.000000        200.000000         200.000000   \n",
              "mean            0.532339        0.318858          0.329544           0.250206   \n",
              "std             0.199412        0.191353          0.146558           0.155868   \n",
              "min             0.045279        0.022487          0.042984           0.000000   \n",
              "25%             0.381774        0.153717          0.214845           0.136725   \n",
              "50%             0.548094        0.295104          0.331804           0.219307   \n",
              "75%             0.708338        0.480780          0.431180           0.335466   \n",
              "max             0.913043        0.829146          0.798742           0.762144   \n",
              "\n",
              "       ROUGE-2 Recall  ROUGE-2 fmeasure  ROUGE-L Precision  ROUGE-L Recall  \\\n",
              "count      200.000000        200.000000         200.000000      200.000000   \n",
              "mean         0.142621          0.150100           0.306936        0.171036   \n",
              "std          0.120452          0.111516           0.147991        0.102693   \n",
              "min          0.000000          0.000000           0.034682        0.016667   \n",
              "25%          0.058357          0.083307           0.187013        0.091293   \n",
              "50%          0.109915          0.129293           0.283398        0.154151   \n",
              "75%          0.189126          0.185822           0.408235        0.229795   \n",
              "max          0.730479          0.716535           0.754386        0.578125   \n",
              "\n",
              "       ROUGE-L Fmeasure  \n",
              "count        200.000000  \n",
              "mean           0.177246  \n",
              "std            0.070499  \n",
              "min            0.032086  \n",
              "25%            0.132177  \n",
              "50%            0.173694  \n",
              "75%            0.212207  \n",
              "max            0.508511  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ef3b84d6-6434-45cc-9070-d060ef2e8647\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ROUGE-1 Precision</th>\n",
              "      <th>ROUGE-1 Recall</th>\n",
              "      <th>ROUGE-1 fmeasure</th>\n",
              "      <th>ROUGE-2 Precision</th>\n",
              "      <th>ROUGE-2 Recall</th>\n",
              "      <th>ROUGE-2 fmeasure</th>\n",
              "      <th>ROUGE-L Precision</th>\n",
              "      <th>ROUGE-L Recall</th>\n",
              "      <th>ROUGE-L Fmeasure</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.531601</td>\n",
              "      <td>0.318874</td>\n",
              "      <td>0.329618</td>\n",
              "      <td>0.250299</td>\n",
              "      <td>0.142454</td>\n",
              "      <td>0.150114</td>\n",
              "      <td>0.304949</td>\n",
              "      <td>0.170118</td>\n",
              "      <td>0.176218</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.198954</td>\n",
              "      <td>0.190694</td>\n",
              "      <td>0.145987</td>\n",
              "      <td>0.155920</td>\n",
              "      <td>0.119611</td>\n",
              "      <td>0.110947</td>\n",
              "      <td>0.146370</td>\n",
              "      <td>0.101633</td>\n",
              "      <td>0.068443</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.045279</td>\n",
              "      <td>0.023352</td>\n",
              "      <td>0.044561</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.034682</td>\n",
              "      <td>0.016544</td>\n",
              "      <td>0.031858</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.381774</td>\n",
              "      <td>0.153717</td>\n",
              "      <td>0.222307</td>\n",
              "      <td>0.137255</td>\n",
              "      <td>0.059302</td>\n",
              "      <td>0.083195</td>\n",
              "      <td>0.184733</td>\n",
              "      <td>0.091275</td>\n",
              "      <td>0.133605</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.546886</td>\n",
              "      <td>0.295104</td>\n",
              "      <td>0.329504</td>\n",
              "      <td>0.218573</td>\n",
              "      <td>0.110636</td>\n",
              "      <td>0.129293</td>\n",
              "      <td>0.287166</td>\n",
              "      <td>0.151645</td>\n",
              "      <td>0.174112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.701848</td>\n",
              "      <td>0.477527</td>\n",
              "      <td>0.431180</td>\n",
              "      <td>0.335089</td>\n",
              "      <td>0.192491</td>\n",
              "      <td>0.185822</td>\n",
              "      <td>0.403646</td>\n",
              "      <td>0.227327</td>\n",
              "      <td>0.209632</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.913043</td>\n",
              "      <td>0.829146</td>\n",
              "      <td>0.798742</td>\n",
              "      <td>0.763819</td>\n",
              "      <td>0.730479</td>\n",
              "      <td>0.718110</td>\n",
              "      <td>0.754386</td>\n",
              "      <td>0.578125</td>\n",
              "      <td>0.508511</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ef3b84d6-6434-45cc-9070-d060ef2e8647')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ef3b84d6-6434-45cc-9070-d060ef2e8647 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ef3b84d6-6434-45cc-9070-d060ef2e8647');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       ROUGE-1 Precision  ROUGE-1 Recall  ROUGE-1 fmeasure  ROUGE-2 Precision  \\\n",
              "count         200.000000      200.000000        200.000000         200.000000   \n",
              "mean            0.531601        0.318874          0.329618           0.250299   \n",
              "std             0.198954        0.190694          0.145987           0.155920   \n",
              "min             0.045279        0.023352          0.044561           0.000000   \n",
              "25%             0.381774        0.153717          0.222307           0.137255   \n",
              "50%             0.546886        0.295104          0.329504           0.218573   \n",
              "75%             0.701848        0.477527          0.431180           0.335089   \n",
              "max             0.913043        0.829146          0.798742           0.763819   \n",
              "\n",
              "       ROUGE-2 Recall  ROUGE-2 fmeasure  ROUGE-L Precision  ROUGE-L Recall  \\\n",
              "count      200.000000        200.000000         200.000000      200.000000   \n",
              "mean         0.142454          0.150114           0.304949        0.170118   \n",
              "std          0.119611          0.110947           0.146370        0.101633   \n",
              "min          0.000000          0.000000           0.034682        0.016544   \n",
              "25%          0.059302          0.083195           0.184733        0.091275   \n",
              "50%          0.110636          0.129293           0.287166        0.151645   \n",
              "75%          0.192491          0.185822           0.403646        0.227327   \n",
              "max          0.730479          0.718110           0.754386        0.578125   \n",
              "\n",
              "       ROUGE-L Fmeasure  \n",
              "count        200.000000  \n",
              "mean           0.176218  \n",
              "std            0.068443  \n",
              "min            0.031858  \n",
              "25%            0.133605  \n",
              "50%            0.174112  \n",
              "75%            0.209632  \n",
              "max            0.508511  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f0452362-b127-4130-8388-aa7253cd8c7c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ROUGE-1 Precision</th>\n",
              "      <th>ROUGE-1 Recall</th>\n",
              "      <th>ROUGE-1 fmeasure</th>\n",
              "      <th>ROUGE-2 Precision</th>\n",
              "      <th>ROUGE-2 Recall</th>\n",
              "      <th>ROUGE-2 fmeasure</th>\n",
              "      <th>ROUGE-L Precision</th>\n",
              "      <th>ROUGE-L Recall</th>\n",
              "      <th>ROUGE-L Fmeasure</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "      <td>200.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.518173</td>\n",
              "      <td>0.326805</td>\n",
              "      <td>0.332185</td>\n",
              "      <td>0.238858</td>\n",
              "      <td>0.144729</td>\n",
              "      <td>0.148949</td>\n",
              "      <td>0.300444</td>\n",
              "      <td>0.176558</td>\n",
              "      <td>0.180393</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.192502</td>\n",
              "      <td>0.193153</td>\n",
              "      <td>0.141879</td>\n",
              "      <td>0.146191</td>\n",
              "      <td>0.121280</td>\n",
              "      <td>0.107066</td>\n",
              "      <td>0.145379</td>\n",
              "      <td>0.106412</td>\n",
              "      <td>0.074181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.045279</td>\n",
              "      <td>0.029228</td>\n",
              "      <td>0.056112</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.034682</td>\n",
              "      <td>0.020367</td>\n",
              "      <td>0.038797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.372996</td>\n",
              "      <td>0.163346</td>\n",
              "      <td>0.231501</td>\n",
              "      <td>0.134774</td>\n",
              "      <td>0.060926</td>\n",
              "      <td>0.083497</td>\n",
              "      <td>0.185258</td>\n",
              "      <td>0.099944</td>\n",
              "      <td>0.135668</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.533279</td>\n",
              "      <td>0.307157</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.209139</td>\n",
              "      <td>0.114815</td>\n",
              "      <td>0.131477</td>\n",
              "      <td>0.281622</td>\n",
              "      <td>0.157053</td>\n",
              "      <td>0.178025</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.677680</td>\n",
              "      <td>0.481385</td>\n",
              "      <td>0.433341</td>\n",
              "      <td>0.318411</td>\n",
              "      <td>0.189126</td>\n",
              "      <td>0.181793</td>\n",
              "      <td>0.400679</td>\n",
              "      <td>0.239167</td>\n",
              "      <td>0.211794</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.891304</td>\n",
              "      <td>0.821549</td>\n",
              "      <td>0.818792</td>\n",
              "      <td>0.725293</td>\n",
              "      <td>0.730185</td>\n",
              "      <td>0.727731</td>\n",
              "      <td>0.754386</td>\n",
              "      <td>0.572391</td>\n",
              "      <td>0.570470</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f0452362-b127-4130-8388-aa7253cd8c7c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f0452362-b127-4130-8388-aa7253cd8c7c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f0452362-b127-4130-8388-aa7253cd8c7c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       ROUGE-1 Precision  ROUGE-1 Recall  ROUGE-1 fmeasure  ROUGE-2 Precision  \\\n",
              "count         200.000000      200.000000        200.000000         200.000000   \n",
              "mean            0.518173        0.326805          0.332185           0.238858   \n",
              "std             0.192502        0.193153          0.141879           0.146191   \n",
              "min             0.045279        0.029228          0.056112           0.000000   \n",
              "25%             0.372996        0.163346          0.231501           0.134774   \n",
              "50%             0.533279        0.307157          0.333333           0.209139   \n",
              "75%             0.677680        0.481385          0.433341           0.318411   \n",
              "max             0.891304        0.821549          0.818792           0.725293   \n",
              "\n",
              "       ROUGE-2 Recall  ROUGE-2 fmeasure  ROUGE-L Precision  ROUGE-L Recall  \\\n",
              "count      200.000000        200.000000         200.000000      200.000000   \n",
              "mean         0.144729          0.148949           0.300444        0.176558   \n",
              "std          0.121280          0.107066           0.145379        0.106412   \n",
              "min          0.000000          0.000000           0.034682        0.020367   \n",
              "25%          0.060926          0.083497           0.185258        0.099944   \n",
              "50%          0.114815          0.131477           0.281622        0.157053   \n",
              "75%          0.189126          0.181793           0.400679        0.239167   \n",
              "max          0.730185          0.727731           0.754386        0.572391   \n",
              "\n",
              "       ROUGE-L Fmeasure  \n",
              "count        200.000000  \n",
              "mean           0.180393  \n",
              "std            0.074181  \n",
              "min            0.038797  \n",
              "25%            0.135668  \n",
              "50%            0.178025  \n",
              "75%            0.211794  \n",
              "max            0.570470  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Estatísicas dos testes\n",
        "\n",
        "for test_score in [teste_1_scores, teste_2_scores, teste_3_scores]:\n",
        "  df = pd.DataFrame()\n",
        "  p1s, r1s, f1s = [], [], []\n",
        "  p2s, r2s, f2s = [], [], []\n",
        "  pLs, rLs, fLs = [], [], []\n",
        "\n",
        "  for score in test_score:\n",
        "    p1, r1, f1 = score['rouge1'].precision, score['rouge1'].recall, score['rouge1'].fmeasure\n",
        "    p2, r2, f2 = score['rouge2'].precision, score['rouge2'].recall, score['rouge2'].fmeasure\n",
        "    pL, rL, fL = score['rougeL'].precision, score['rougeL'].recall, score['rougeL'].fmeasure\n",
        "\n",
        "    p1s.append(p1)\n",
        "    r1s.append(r1)\n",
        "    f1s.append(f1)\n",
        "\n",
        "    p2s.append(p2)\n",
        "    r2s.append(r2)\n",
        "    f2s.append(f2)\n",
        "    \n",
        "    pLs.append(pL)\n",
        "    rLs.append(rL)\n",
        "    fLs.append(fL)\n",
        "\n",
        "  df['ROUGE-1 Precision'] = p1s\n",
        "  df['ROUGE-1 Recall']    = r1s\n",
        "  df['ROUGE-1 fmeasure']  = f1s\n",
        "\n",
        "  df['ROUGE-2 Precision'] = p2s\n",
        "  df['ROUGE-2 Recall']    = r2s\n",
        "  df['ROUGE-2 fmeasure']  = f2s\n",
        "\n",
        "  df['ROUGE-L Precision'] = pLs\n",
        "  df['ROUGE-L Recall']    = rLs\n",
        "  df['ROUGE-L Fmeasure']  = fLs\n",
        "\n",
        "  display(df.describe())"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "052cd25a2b0f4b38a08460ff13201dcd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0739744961d947d6a081815fbe018817": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "09d6b87a55a64ce99b34226cdd85667c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10c37ea9fa3d4566a67f8682c0066657": {
          "model_module": "@jupyter-widgets/output",
          "model_module_version": "1.0.0",
          "model_name": "OutputModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_dc0575a4ff064a1e8a97ed6903108427",
            "msg_id": "",
            "outputs": [
              {
                "name": "stdout",
                "output_type": "stream",
                "text": [
                  "=--------------------=\n",
                  "SCORE ROUGE1 -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
                  "SCORE ROUGE2 -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
                  "SCORE ROUGEL -> precision = 0.00, recall = 0.00, fmeasure = 0.00\n",
                  "=--------------------=\n"
                ]
              },
              {
                "data": {
                  "application/vnd.google.colaboratory.intrinsic+json": {
                    "type": "string"
                  },
                  "text/plain": "'41, folhas 01-13). ⚪ 41, folha 07). ⚪ 37, folha 01).'"
                },
                "metadata": {},
                "output_type": "display_data"
              }
            ]
          }
        },
        "168af84d73c24052b7b3f04ca08e9d4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16e569df09194ae09efd13ef89c52df5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21ee69ce4feb49198afcc61122b2edea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": false,
            "description": "doc_index",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_f7de7ba4bcce40dfaabb19f5aabd3b4e",
            "max": 10573,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_80405b0531ba4ee1815e528d41960da0",
            "value": 0
          }
        },
        "220ee9d44d564bb3906a374b2cf9916e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "233bbbae11884d55a1c8c8a4b202f905": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DropdownModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DropdownModel",
            "_options_labels": [
              "Ementa",
              "Relatório",
              "Area",
              "Relator",
              "Voto",
              "Acordão",
              "Classe",
              "Extrato"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "DropdownView",
            "description": "section_name",
            "description_tooltip": null,
            "disabled": false,
            "index": 0,
            "layout": "IPY_MODEL_d060c0dc4bb54847b4d588785843e064",
            "style": "IPY_MODEL_8e50b564feb34cd3b446b755fbdce4dc"
          }
        },
        "2386035ed3314fd0b97f9445e64afe34": {
          "model_module": "@jupyter-widgets/output",
          "model_module_version": "1.0.0",
          "model_name": "OutputModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_cc5561908bfa486cb523abfd725946d9",
            "msg_id": "",
            "outputs": [
              {
                "name": "stdout",
                "output_type": "stream",
                "text": [
                  "EMBARGOS DE DECLARAÇÃO - INOCORRÊNCIA DE CONTRADIÇÃO, OBSCURIDADE OU OMISSÃO - PRETENDIDO REEXAME DA CAUSA CARÁTER INFRINGENTE - INADMISSIBILIDADE – EMBARGOS DE DECLARAÇÃO REJEITADOS. 👈\n",
                  "- Não se revelam admissíveis os embargos de declaração, quando a parte recorrente - a pretexto de esclarecer uma inexistente situação de obscuridade, omissão ou contradição - vem a utilizá-los com o objetivo de infringir o julgado e de viabilizar, assim, um indevido reexame da causa. 👈\n",
                  "Precedentes. 👈\n",
                  "- A nulidade radical que afeta os diversos atos de interrogatório judicial, quando provocada pela conduta do magistrado que arbitrariamente nega, ao réu, o direito – por este titularizado – de formular reperguntas aos demais litisconsortes penais passivos, contamina, por efeito causal, todos os atos subseqüentes do processo, notadamente aqueles de índole probatória, como a inquirição de testemunhas arroladas pelo Ministério Público. 👈\n",
                  "Precedentes. 👈\n"
                ]
              }
            ]
          }
        },
        "26a0d516481f41fca10a9008821ae951": {
          "model_module": "@jupyter-widgets/output",
          "model_module_version": "1.0.0",
          "model_name": "OutputModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_5cb04549f16b488abb948a3d6b8b2d13",
            "msg_id": "",
            "outputs": [
              {
                "name": "stdout",
                "output_type": "stream",
                "text": [
                  "EMBARGOS DE DECLARAÇÃO - INOCORRÊNCIA DE CONTRADIÇÃO, OBSCURIDADE OU OMISSÃO - PRETENDIDO REEXAME DA CAUSA CARÁTER INFRINGENTE - INADMISSIBILIDADE – EMBARGOS DE DECLARAÇÃO REJEITADOS. 👈\n",
                  "- Não se revelam admissíveis os embargos de declaração, quando a parte recorrente - a pretexto de esclarecer uma inexistente situação de obscuridade, omissão ou contradição - vem a utilizá-los com o objetivo de infringir o julgado e de viabilizar, assim, um indevido reexame da causa. 👈\n",
                  "Precedentes. 👈\n",
                  "- A nulidade radical que afeta os diversos atos de interrogatório judicial, quando provocada pela conduta do magistrado que arbitrariamente nega, ao réu, o direito – por este titularizado – de formular reperguntas aos demais litisconsortes penais passivos, contamina, por efeito causal, todos os atos subseqüentes do processo, notadamente aqueles de índole probatória, como a inquirição de testemunhas arroladas pelo Ministério Público. 👈\n",
                  "Precedentes. 👈\n"
                ]
              }
            ]
          }
        },
        "275dcab0715443dba3d6bd3b3c68fbc8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2afb0d88ec404b3a9acbac06b4f9bc49": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2da2a866a48d4e77891746ac5e841dd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "TextModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "TextModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "TextView",
            "continuous_update": true,
            "description": "RegEx",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_61badc70be6e4e7f8c74658e6a4eb841",
            "placeholder": "Digite RegEx aqui",
            "style": "IPY_MODEL_67cd2b6d951d42b09fc4685f806f0548",
            "value": ""
          }
        },
        "2fc45387f79648b6b0ad44f27df15b14": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34b5727c18194e8093818d3239222202": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34fa7e9fb2e9475bb13912645f4aef5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35a3c103a3b44a558da9b9118b8422a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Marcar fim de sentença com 👈",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_2fc45387f79648b6b0ad44f27df15b14",
            "style": "IPY_MODEL_168af84d73c24052b7b3f04ca08e9d4c",
            "value": true
          }
        },
        "376971c5a9d841caa01486a459227973": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Show heuristic score change",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_9a6738557bb743338ba5c88d118c43c6",
            "style": "IPY_MODEL_2afb0d88ec404b3a9acbac06b4f9bc49",
            "value": false
          }
        },
        "37b10ae1a439425a9d9f84ba9fea557e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3dff8235753c4c0097455b9276f96c25": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3eaffe6ab9a64532bf4c0997bf757ac8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4195b4c6cb714905895f97360670eeb8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "420ad5b967784da6a2376493d7e5e869": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45751c7ea6f743718d004af57c9c18a0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4964ff7abe15444c9a7b62bc9e66a166": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Show similarity matrix",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_97a5c1da276543d1ae4a9ed2f8a52c1d",
            "style": "IPY_MODEL_62e00a80351d4ba9a4a440023f0c21f1",
            "value": false
          }
        },
        "4bfd5aa55dcd4b41a7188389d5b8d38b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5042779160bd4b9eb59b0420bda65a57": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SelectMultipleModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SelectMultipleModel",
            "_options_labels": [
              "TF+",
              "IDF+",
              "TF-IDF+",
              "Upper+",
              "Prop+",
              "Cue Phrases+",
              "Have Number+",
              "Length+",
              "Position+",
              "TF-",
              "IDF-",
              "TF-IDF-",
              "Upper-",
              "Prop-",
              "Cue Phrases-",
              "Have Number-",
              "Length-",
              "Position-"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "SelectMultipleView",
            "description": "Heuristicas",
            "description_tooltip": null,
            "disabled": false,
            "index": [
              0,
              1
            ],
            "layout": "IPY_MODEL_94f3c8d171aa4673a359e9e8428b070e",
            "rows": 5,
            "style": "IPY_MODEL_420ad5b967784da6a2376493d7e5e869"
          }
        },
        "513ca112a32544fe8b4b6dd74a6026d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Preprocess with lemma",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_4bfd5aa55dcd4b41a7188389d5b8d38b",
            "style": "IPY_MODEL_09d6b87a55a64ce99b34226cdd85667c",
            "value": true
          }
        },
        "555547d9a166409e84a85af41b24d0c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "56b3e6681f3d470aaa4149ef49803e7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Marcar fim de sentença com 👈",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_7cb97a650751447f964b0dc80170ccea",
            "style": "IPY_MODEL_cd95dce0e3bd4b17952e59e55bad3df2",
            "value": true
          }
        },
        "5792c68532b143e18855670b9d594551": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DropdownModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DropdownModel",
            "_options_labels": [
              "Ementa",
              "Relatório",
              "Area",
              "Relator",
              "Voto",
              "Acordão",
              "Classe",
              "Extrato"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "DropdownView",
            "description": "section_name",
            "description_tooltip": null,
            "disabled": false,
            "index": 0,
            "layout": "IPY_MODEL_cad015143c0c4cbc9dd54470c19a2ae1",
            "style": "IPY_MODEL_3dff8235753c4c0097455b9276f96c25"
          }
        },
        "5cb04549f16b488abb948a3d6b8b2d13": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61badc70be6e4e7f8c74658e6a4eb841": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "621a5c8ed16943be951b5e52ea615afa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "62e00a80351d4ba9a4a440023f0c21f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "66601b493d104fae9a49086aa67ded2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Filter Stop Word",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_275dcab0715443dba3d6bd3b3c68fbc8",
            "style": "IPY_MODEL_621a5c8ed16943be951b5e52ea615afa",
            "value": true
          }
        },
        "67cd2b6d951d42b09fc4685f806f0548": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ae53a1bdd6b4dbb807e73a47c0207f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "6f2c5ac6218c4597936cbe0f7a721421": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "716f10a6d2344c9ca5281225a39139e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Return Text",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_ffa4f595d39d4ae7abc8e70c5ca11b3f",
            "style": "IPY_MODEL_0739744961d947d6a081815fbe018817",
            "value": true
          }
        },
        "7271cd6401b646859e6ffb179a746e74": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "7624fc3082a543df9aa2c7648519b942": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": false,
            "description": "doc_index",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_cbd0d32997f34c968e4d776c4c0c3447",
            "max": 10573,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_6f2c5ac6218c4597936cbe0f7a721421",
            "value": 8
          }
        },
        "7cb97a650751447f964b0dc80170ccea": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80405b0531ba4ee1815e528d41960da0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "868cf916b8c449eebfeef9f273393255": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "FloatSliderView",
            "continuous_update": false,
            "description": "Limiar Bush",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_dae729d25f014a268b1ac044a62e2b95",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": ".2f",
            "step": 0.05,
            "style": "IPY_MODEL_6ae53a1bdd6b4dbb807e73a47c0207f9",
            "value": 0.5
          }
        },
        "86fd88c6db7a4a518e42b03b4b30f83c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SelectMultipleModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SelectMultipleModel",
            "_options_labels": [
              "Ementa",
              "Relatório",
              "Area",
              "Relator",
              "Voto",
              "Acordão",
              "Classe",
              "Extrato"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "SelectMultipleView",
            "description": "Ctrl + Click",
            "description_tooltip": null,
            "disabled": false,
            "index": [
              1
            ],
            "layout": "IPY_MODEL_45751c7ea6f743718d004af57c9c18a0",
            "rows": 5,
            "style": "IPY_MODEL_37b10ae1a439425a9d9f84ba9fea557e"
          }
        },
        "87c5903b232143daa74f7d015fbb1a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Return joined sentences",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_f67e1c6cb5ab4dc4b9732cef42e36db0",
            "style": "IPY_MODEL_baca034e01864ac68ebef601b47e2301",
            "value": true
          }
        },
        "89b7f6d45a08468ebb45187b20d34898": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e50b564feb34cd3b446b755fbdce4dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9168fb9351634741a7b6b2a19d13f72f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94f3c8d171aa4673a359e9e8428b070e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "954e2b406ce14c118a88fdf68805e504": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DropdownModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DropdownModel",
            "_options_labels": [
              "BOW",
              "TF",
              "TF-IDF"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "DropdownView",
            "description": "Similaridade",
            "description_tooltip": null,
            "disabled": false,
            "index": 0,
            "layout": "IPY_MODEL_aabdad6cfe2a48139cb9f1772acada72",
            "style": "IPY_MODEL_34fa7e9fb2e9475bb13912645f4aef5a"
          }
        },
        "97a5c1da276543d1ae4a9ed2f8a52c1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "999941efbcb44c2daecdf5ca7694facf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9a6738557bb743338ba5c88d118c43c6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0e6f456f0b0412ca99ddf022458b7c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": false,
            "description": "doc_index",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_052cd25a2b0f4b38a08460ff13201dcd",
            "max": 10573,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_b403e21ec19b4c549ee68c85f2f22fa0",
            "value": 0
          }
        },
        "a7089eb828b449edaa6a44e19e5003bf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aabdad6cfe2a48139cb9f1772acada72": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b403e21ec19b4c549ee68c85f2f22fa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "baca034e01864ac68ebef601b47e2301": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cad015143c0c4cbc9dd54470c19a2ae1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbd0d32997f34c968e4d776c4c0c3447": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc5561908bfa486cb523abfd725946d9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd95dce0e3bd4b17952e59e55bad3df2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d060c0dc4bb54847b4d588785843e064": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9378a10007f459d97bfca81db498ac6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DropdownModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DropdownModel",
            "_options_labels": [
              "ROUGE"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "DropdownView",
            "description": "Validação",
            "description_tooltip": null,
            "disabled": false,
            "index": 0,
            "layout": "IPY_MODEL_9168fb9351634741a7b6b2a19d13f72f",
            "style": "IPY_MODEL_555547d9a166409e84a85af41b24d0c2"
          }
        },
        "dae729d25f014a268b1ac044a62e2b95": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc0575a4ff064a1e8a97ed6903108427": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de94a90071774cba9937b2aaae90d39c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [
              "widget-interact"
            ],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a0e6f456f0b0412ca99ddf022458b7c2",
              "IPY_MODEL_233bbbae11884d55a1c8c8a4b202f905",
              "IPY_MODEL_56b3e6681f3d470aaa4149ef49803e7f",
              "IPY_MODEL_f4da64fc4381466f8886c81475d7d4c0",
              "IPY_MODEL_2386035ed3314fd0b97f9445e64afe34"
            ],
            "layout": "IPY_MODEL_4195b4c6cb714905895f97360670eeb8"
          }
        },
        "decc4aa3fe8945f7a7edc6a03f680658": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [
              "widget-interact"
            ],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_21ee69ce4feb49198afcc61122b2edea",
              "IPY_MODEL_5792c68532b143e18855670b9d594551",
              "IPY_MODEL_35a3c103a3b44a558da9b9118b8422a7",
              "IPY_MODEL_ff9f786cd5244d728d1742a28a06b042",
              "IPY_MODEL_26a0d516481f41fca10a9008821ae951"
            ],
            "layout": "IPY_MODEL_a7089eb828b449edaa6a44e19e5003bf"
          }
        },
        "e5207743eaa948deb199e7cf6476c3e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e703a6a38a8a4e52b2b667fc8a072149": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": false,
            "description": "top_n",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_f9de4d6f438b4ccf88c9ca2a6a542f43",
            "max": 15,
            "min": 1,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_7271cd6401b646859e6ffb179a746e74",
            "value": 3
          }
        },
        "e973190dbe614dd7b3166a637dd6c128": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DropdownModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DropdownModel",
            "_options_labels": [
              "Bush Path Node",
              "Similaridade Agregada",
              "Text Rank"
            ],
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "DropdownView",
            "description": "Algoritmo",
            "description_tooltip": null,
            "disabled": false,
            "index": 0,
            "layout": "IPY_MODEL_16e569df09194ae09efd13ef89c52df5",
            "style": "IPY_MODEL_e5207743eaa948deb199e7cf6476c3e6"
          }
        },
        "f4da64fc4381466f8886c81475d7d4c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "TextModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "TextModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "TextView",
            "continuous_update": true,
            "description": "RegEx",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_89b7f6d45a08468ebb45187b20d34898",
            "placeholder": "Digite RegEx aqui",
            "style": "IPY_MODEL_220ee9d44d564bb3906a374b2cf9916e",
            "value": ""
          }
        },
        "f67e1c6cb5ab4dc4b9732cef42e36db0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7de7ba4bcce40dfaabb19f5aabd3b4e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f9de4d6f438b4ccf88c9ca2a6a542f43": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f9e672c2fad948dea720dff1e4067200": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [
              "widget-interact"
            ],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7624fc3082a543df9aa2c7648519b942",
              "IPY_MODEL_716f10a6d2344c9ca5281225a39139e8",
              "IPY_MODEL_66601b493d104fae9a49086aa67ded2d",
              "IPY_MODEL_513ca112a32544fe8b4b6dd74a6026d0",
              "IPY_MODEL_87c5903b232143daa74f7d015fbb1a5e",
              "IPY_MODEL_4964ff7abe15444c9a7b62bc9e66a166",
              "IPY_MODEL_376971c5a9d841caa01486a459227973",
              "IPY_MODEL_5042779160bd4b9eb59b0420bda65a57",
              "IPY_MODEL_86fd88c6db7a4a518e42b03b4b30f83c",
              "IPY_MODEL_2da2a866a48d4e77891746ac5e841dd3",
              "IPY_MODEL_e973190dbe614dd7b3166a637dd6c128",
              "IPY_MODEL_868cf916b8c449eebfeef9f273393255",
              "IPY_MODEL_954e2b406ce14c118a88fdf68805e504",
              "IPY_MODEL_e703a6a38a8a4e52b2b667fc8a072149",
              "IPY_MODEL_d9378a10007f459d97bfca81db498ac6",
              "IPY_MODEL_10c37ea9fa3d4566a67f8682c0066657"
            ],
            "layout": "IPY_MODEL_34b5727c18194e8093818d3239222202"
          }
        },
        "ff9f786cd5244d728d1742a28a06b042": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "TextModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "TextModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "TextView",
            "continuous_update": true,
            "description": "RegEx",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_3eaffe6ab9a64532bf4c0997bf757ac8",
            "placeholder": "Digite RegEx aqui",
            "style": "IPY_MODEL_999941efbcb44c2daecdf5ca7694facf",
            "value": ""
          }
        },
        "ffa4f595d39d4ae7abc8e70c5ca11b3f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}