QutationLevelAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8a064be7-608b-43c2-b2ca-4b9a7477614a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data handling and visualization\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy.stats as ss\n",
    "\n",
    "# PySpark\n",
    "import numpy as np\n",
    "import pyspark\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql import SparkSession, Window\n",
    "from pyspark.sql.types import StringType, BooleanType, StructType, StructField, FloatType, ArrayType, MapType\n",
    "\n",
    "# String handling\n",
    "import html\n",
    "import tld\n",
    "\n",
    "# Helpers\n",
    "import sys\n",
    "sys.path.append('/home/culjak/speaker-disambiguation-quotebank/')\n",
    "\n",
    "# Calculating scores\n",
    "import profanity_check\n",
    "from sparknlp.pretrained import PretrainedPipeline\n",
    "from empath import Empath\n",
    "import plotly as pl\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "# Stats\n",
    "import scipy.stats as ss\n",
    "from statsmodels.stats.proportion import proportion_confint\n",
    "\n",
    "import chart_studio.plotly as py\n",
    "import chart_studio\n",
    "from itertools import cycle\n",
    "from plotly.offline import iplot\n",
    "chart_studio.tools.set_credentials_file(username='mculyak', api_key='IQgDLvC6thrs3aANLk35')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b49e08f3-5b30-42de-9df2-8d285d5cdb8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21/12/16 00:08:25 WARN Utils: Your hostname, iccluster111 resolves to a loopback address: 127.0.1.1; using 10.90.36.41 instead (on interface eno1)\n",
      "21/12/16 00:08:25 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
      "21/12/16 00:08:27 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "21/12/16 00:08:30 WARN Utils: Service 'SparkUI' could not bind on port 4866. Attempting port 4867.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://10.90.36.41:4867\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.1.2</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[24]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>pyspark-shell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f08218c5250>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Starting the spark session\n",
    "conf = pyspark.SparkConf().setMaster(\"local[24]\").setAll([\n",
    "    ('spark.driver.memory','20G'),\n",
    "    ('spark.driver.maxResultSize', '20G'),\n",
    "    ('spark.ui.port', 4866),\n",
    "    ('spark.sql.execution.arrow.pyspark.enabled', True)\n",
    "])\n",
    "\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n",
    "sc = spark.sparkContext\n",
    "sc.setLogLevel('WARN') # ERROR, WARN, INFO, DEBUG, ...\n",
    "spark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4ca7081-03e9-4dc9-a4db-d9d980620f3c",
   "metadata": {},
   "source": [
    "# Quotation-level analysis\n",
    "In this notebook we plot some simple visualizations concerning profanity distribution through time and some general profanity and censorship statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0cbbdb55-7277-4f7b-9641-2cacaf954b5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- numOccurrences: long (nullable = true)\n",
      " |-- phase: string (nullable = true)\n",
      " |-- probas: array (nullable = true)\n",
      " |    |-- element: array (containsNull = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- qids: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- quotation: string (nullable = true)\n",
      " |-- quoteID: string (nullable = true)\n",
      " |-- speaker: string (nullable = true)\n",
      " |-- domains: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- date: string (nullable = true)\n",
      " |-- time: string (nullable = true)\n",
      " |-- year: integer (nullable = true)\n",
      " |-- month: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "DATA_DIR = '../data/'\n",
    "\n",
    "quotes = spark.read.parquet(DATA_DIR + 'quotse.parquet')\n",
    "# TODO add code for extracting domains\n",
    "quotes.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f5cb74b6-6a31-41ae-9cb5-f6496d61ef28",
   "metadata": {},
   "outputs": [],
   "source": [
    "profanity = spark.read.parquet(DATA_DIR + 'profanity_expanded.parquet')\n",
    "empath = spark.read.parquet(DATA_DIR + 'empath_ultimate.parquet')\n",
    "censored_quotes_without_censorship = spark.read.parquet(DATA_DIR 'censorship_removed.parquet')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d072f0b-311b-4d33-b96e-ad2c405d6fc0",
   "metadata": {},
   "source": [
    "# Profanity and censorship analysis\n",
    "## General stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "b2ca2056-6be3-4ead-9d9b-95b92c1cdf36",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>profanity</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1146168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>113818503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   profanity      count\n",
       "0          1    1146168\n",
       "1          0  113818503"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "profanity_counts = profanity.groupby('profanity').count().toPandas()\n",
    "profanity_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "c0301265-ac25-40aa-8de8-28be949c8dbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>censored</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>114599181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>365490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   censored      count\n",
       "0       0.0  114599181\n",
       "1       1.0     365490"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "censored_counts = profanity.groupby('censored').count().toPandas()\n",
    "censored_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "2ed5bd57-c25c-4fce-bf5a-0cc6bd33d999",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/1.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f08b2c96fd0>"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layout = go.Layout(\n",
    "    title=\"Profanity in Quotebank\",\n",
    "    bargap=0,\n",
    ")\n",
    "\n",
    "figure = go.Figure(data=[go.Pie(labels=['Profane', 'Not profane'],\n",
    "                             values=profanity_counts['count'], hole=.7)], layout=layout)\n",
    "\n",
    "py.iplot(figure, filename='profanity_donut')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "id": "8f98c107-b03b-4973-b52f-154743f56608",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/3.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f0812dc5fd0>"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layout = go.Layout(\n",
    "    title=\"Censorship in Quotebank\",\n",
    "    bargap=0,\n",
    ")\n",
    "\n",
    "figure = go.Figure(data=[go.Pie(labels=['Censored', 'Not censored'],\n",
    "                             values=censored_counts['count'], hole=.7)], layout=layout)\n",
    "\n",
    "py.iplot(figure, filename='censorship_donut')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "82ca81f7-bb8b-4e4d-85c3-48176ae4c6b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>censored</th>\n",
       "      <th>profanity</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>780678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>365490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   censored  profanity   count\n",
       "0       0.0          1  780678\n",
       "1       1.0          1  365490"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "censored_profanity_counts = profanity.where(F.col('profanity') == 1).groupby('censored', 'profanity').count().toPandas()\n",
    "censored_profanity_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "2e8baf66-3661-4088-9a17-726c95f04ed0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/11.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f08108c9490>"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layout = go.Layout(\n",
    "    title=\"Censored profanities\",\n",
    "    bargap=0,\n",
    ")\n",
    "\n",
    "figure = go.Figure(data=[go.Pie(labels=['Censored profanities', 'Not censored profanities'],\n",
    "                             values=censored_profanity_counts['count'], hole=.7)], layout=layout)\n",
    "\n",
    "py.iplot(figure, filename='cen_prof_donut')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7abc7f79-8a48-4bf6-a822-3be3e5c58355",
   "metadata": {},
   "source": [
    "## Temporal analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6771d4d2-a469-4937-9bc8-773107a7dfe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "profanity_date = profanity.join(quotes, on='quoteID').select('profanity', 'censored', 'date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "2f1235b0-4be8-42d9-aa27-7da3adae2d01",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "agg_per_day = profanity_date.groupby('date')\\\n",
    "    .agg(F.sum('profanity').alias('profanity'),\n",
    "         F.sum('censored').alias('censored'),\n",
    "         F.count('date').alias('n_quotes'))\\\n",
    "    .toPandas()\\\n",
    "    .sort_values('date')\n",
    "\n",
    "agg_per_day['date'] = pd.to_datetime(agg_per_day['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "db868811-e455-47cc-a11c-aeee821e6e54",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_per_week = agg_per_day.resample('W-Mon', on='date').sum().reset_index().sort_values(by='date')\n",
    "agg_per_month = agg_per_day.resample('M', on='date').sum().reset_index().sort_values(by='date')\n",
    "agg_per_year = agg_per_day.resample('Y', on='date').sum().reset_index().sort_values(by='date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "d5a2b403-befe-4b15-a793-cb082f09b804",
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_area_plot(fig, df, columns, fill, names, palette=px.colors.diverging.Portland, palette_indices=(1, 4, 2), log=True, visible=False):\n",
    "    for c, f, n, i in zip(columns, fill, names, palette_indices):\n",
    "        fig.add_trace(go.Scatter(x=df['date'],\n",
    "                         y=df[c],\n",
    "                         fill=f,\n",
    "                         name=n,\n",
    "                         line_color=palette[i],\n",
    "                         visible=visible))\n",
    "    if log:\n",
    "        fig.update_yaxes(type='log')\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "id": "cbf43051-f3c2-40eb-8979-8a930f279c16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/5.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f08801c99d0>"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = ['Censored quotes', 'Profane quotes', 'All quotes']\n",
    "fills = ['tozeroy', 'tonexty', 'tonexty']\n",
    "columns = ['censored', 'profanity', 'n_quotes']\n",
    "fig = go.Figure()\n",
    "\n",
    "fig = time_area_plot(fig, agg_per_day, columns, fills, names, visible=True)\n",
    "fig = time_area_plot(fig, agg_per_week, columns, fills, names)\n",
    "fig = time_area_plot(fig, agg_per_month, columns, fills, names)\n",
    "fig = time_area_plot(fig, agg_per_year, columns, fills, names)\n",
    "\n",
    "fig.update_layout(\n",
    "    updatemenus=[\n",
    "        dict(\n",
    "            type=\"buttons\",\n",
    "            direction=\"right\",\n",
    "            x=0.205,\n",
    "            y=1.21,\n",
    "            showactive=True,\n",
    "            buttons=list(\n",
    "                [\n",
    "                    dict(\n",
    "                        label=\"Daily\",\n",
    "                        method=\"update\",\n",
    "                        args=[\n",
    "                            {\"visible\": [True, True, True, False, False, False, False, False, False, False, False, False]},\n",
    "                        ],\n",
    "                    ),\n",
    "                    dict(\n",
    "                        label=\"Weekly\",\n",
    "                        method=\"update\",\n",
    "                        args=[\n",
    "                            {\"visible\": [False, False, False, True, True, True, False, False, False, False, False, False]},\n",
    "                        ],\n",
    "                    ),\n",
    "                    dict(\n",
    "                        label=\"Monthly\",\n",
    "                        method=\"update\",\n",
    "                        args=[\n",
    "                            {\"visible\": [False, False, False, False, False, False, True, True, True, False, False, False]},\n",
    "                        ],\n",
    "                    ),\n",
    "                    dict(\n",
    "                        label=\"Yearly\",\n",
    "                        method=\"update\",\n",
    "                        args=[\n",
    "                            {\"visible\": [False, False, False, False, False, False, False, False, False, True, True, True,]},\n",
    "                        ],\n",
    "                    )\n",
    "                ]\n",
    "            ),\n",
    "        )\n",
    "    ]\n",
    ")\n",
    "\n",
    "fig.update_layout(title='Profanity through time',\n",
    "                  xaxis_title='Year',\n",
    "                  yaxis_title='Quote frequency',\n",
    "                  font=dict(family='Helvetica'))\n",
    "py.iplot(fig, filename='profanity_through_time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "c5d38c78-c728-42d1-946f-51424e79016d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordered_weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday',\n",
    "                    'Friday', 'Saturday', 'Sunday']# because pandas sorts weekdays alphabetically\n",
    "weekday_agg = agg_per_day.groupby(agg_per_day['date'].dt.day_name())\\\n",
    "                    .sum()\\\n",
    "                    .reindex(ordered_weekdays)\\\n",
    "                    .reset_index()\n",
    "\n",
    "weekday_agg['profanity_perc'] = weekday_agg['profanity'] / weekday_agg['n_quotes']\n",
    "weekday_agg['censorship_perc'] = weekday_agg['censored'] / weekday_agg['n_quotes']\n",
    "# Confidence intervals\n",
    "profanity_CI_lower_week, profanity_CI_upper_week = proportion_confint(weekday_agg['profanity'],\n",
    "                                                                      weekday_agg['n_quotes'], alpha=0.01)\n",
    "censorship_CI_lower_week, censorship_CI_upper_week = proportion_confint(weekday_agg['censored'],\n",
    "                                                                      weekday_agg['n_quotes'], alpha=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "c5fb6538-b900-4a30-b473-455d81f67d00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/9.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f0810b1bcd0>"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "palette = px.colors.diverging.Portland\n",
    "fig = go.Figure()\n",
    "fig.add_trace(go.Bar(\n",
    "    name='Profanity',\n",
    "    marker_color=palette[4],\n",
    "    x=weekday_agg.date, y=weekday_agg.profanity_perc,\n",
    "    error_y=dict(type='data', array=profanity_CI_lower_week - profanity_CI_upper_week)\n",
    "))\n",
    "\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "    name='Censorship',\n",
    "    marker_color=palette[1],\n",
    "    x=weekday_agg.date, y=weekday_agg.censorship_perc,\n",
    "    error_y=dict(type='data', array=censorship_CI_lower_week - censorship_CI_upper_week, width=5)\n",
    "))\n",
    "\n",
    "fig.update_layout(\n",
    "    title='Average profanity by day of the week',\n",
    "    xaxis_title='Day of the week',\n",
    "    yaxis_title='Average profanity',\n",
    "    barmode='overlay'\n",
    ")\n",
    "\n",
    "py.iplot(fig, filename='profanity_dotw')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "b1f3c582-4aba-48c2-a28b-4ed015a6fdda",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordered_months = ['January', 'February', 'March', 'April', 'May', 'June',\n",
    "                  'July', 'August', 'September', 'October', 'November', 'December'] # because pandas sorts weekdays alphabetically\n",
    "month_agg = agg_per_day.groupby(agg_per_day['date'].dt.month_name())\\\n",
    "                    .sum()\\\n",
    "                    .reindex(ordered_months)\\\n",
    "                    .reset_index()\n",
    "\n",
    "\n",
    "month_agg['profanity_perc'] = month_agg['profanity'] / month_agg['n_quotes']\n",
    "month_agg['censorship_perc'] = month_agg['censored'] / month_agg['n_quotes']\n",
    "\n",
    "profanity_CI_lower_month, profanity_CI_upper_month = proportion_confint(month_agg['profanity'],\n",
    "                                                                        month_agg['n_quotes'], alpha=0.01)\n",
    "\n",
    "censorship_CI_lower_month, censorship_CI_upper_month = proportion_confint(month_agg['censored'],\n",
    "                                                                      month_agg['n_quotes'], alpha=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "c3308171-af17-41ac-b400-41e17fafc0a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"525px\"\n",
       "            src=\"https://plotly.com/~mculyak/13.embed\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f0810924e50>"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fig = go.Figure()\n",
    "fig.add_trace(go.Bar(\n",
    "    name='Profanity',\n",
    "    marker_color=palette[4],\n",
    "    x=month_agg.date, y=month_agg.profanity_perc,\n",
    "    error_y=dict(type='data', array=profanity_CI_upper_month - profanity_CI_lower_month)\n",
    "))\n",
    "\n",
    "\n",
    "fig.add_trace(go.Bar(\n",
    "    name='Censorship',\n",
    "    marker_color=palette[1],\n",
    "    x=month_agg.date, y=month_agg.censorship_perc,\n",
    "    error_y=dict(type='data', array=censorship_CI_upper_month - censorship_CI_lower_month)\n",
    "))\n",
    "fig.update_layout(\n",
    "    title='Average profanity by month',\n",
    "    xaxis_title='Month',\n",
    "    yaxis_title='Average profanity',\n",
    "    barmode='overlay'\n",
    ")\n",
    "\n",
    "py.iplot(fig, filename='profanity_month')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "speaker-disambiguation",
   "language": "python",
   "name": "speaker-disambiguation"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}