Data-Exlporation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAAAAW128F429D538.json  TRAAAFD128F92F423A.json  TRAAARJ128F9320760.json\r\n",
      "TRAAABD128F429CF47.json  TRAAAMO128F1481E7F.json  TRAAAVG12903CFA543.json\r\n",
      "TRAAADZ128F9348C2E.json  TRAAAMQ128F1460CD3.json  TRAAAVO128F93133D4.json\r\n",
      "TRAAAEF128F4273421.json  TRAAAPK128E0786D96.json\r\n"
     ]
    }
   ],
   "source": [
    "ls data/song_data/A/A/A/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list((PATH / 'song_data').rglob('*.json')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 71 songs metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list((PATH / 'log_data').rglob('*.json')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 30 files where each one has logs data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(PATH.rglob('*.json')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Songs Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_id</th>\n",
       "      <th>artist_latitude</th>\n",
       "      <th>artist_location</th>\n",
       "      <th>artist_longitude</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>duration</th>\n",
       "      <th>num_songs</th>\n",
       "      <th>song_id</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AR7G5I41187FB4CE6C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>London, England</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Adam Ant</td>\n",
       "      <td>233.40363</td>\n",
       "      <td>1</td>\n",
       "      <td>SONHOTT12A8C13493C</td>\n",
       "      <td>Something Girls</td>\n",
       "      <td>1982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_id  artist_latitude  artist_location  artist_longitude  \\\n",
       "0  AR7G5I41187FB4CE6C              NaN  London, England               NaN   \n",
       "\n",
       "  artist_name   duration  num_songs             song_id            title  year  \n",
       "0    Adam Ant  233.40363          1  SONHOTT12A8C13493C  Something Girls  1982  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_files = list((PATH / 'song_data').rglob('*.json'))\n",
    "song_df = pd.read_json(songs_files[0], lines=True)\n",
    "song_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, f in enumerate(songs_files):\n",
    "    if i == 0:\n",
    "        songs_df = pd.read_json(f, lines=True)\n",
    "    else:\n",
    "        songs_df = songs_df.append(pd.read_json(f, lines=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_id</th>\n",
       "      <th>artist_latitude</th>\n",
       "      <th>artist_location</th>\n",
       "      <th>artist_longitude</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>duration</th>\n",
       "      <th>num_songs</th>\n",
       "      <th>song_id</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AR7G5I41187FB4CE6C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>London, England</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Adam Ant</td>\n",
       "      <td>233.40363</td>\n",
       "      <td>1</td>\n",
       "      <td>SONHOTT12A8C13493C</td>\n",
       "      <td>Something Girls</td>\n",
       "      <td>1982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AR8ZCNI1187B9A069B</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>Planet P Project</td>\n",
       "      <td>269.81832</td>\n",
       "      <td>1</td>\n",
       "      <td>SOIAZJW12AB01853F1</td>\n",
       "      <td>Pink World</td>\n",
       "      <td>1984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ARXR32B1187FB57099</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>Gob</td>\n",
       "      <td>209.60608</td>\n",
       "      <td>1</td>\n",
       "      <td>SOFSOCN12A8C143F5D</td>\n",
       "      <td>Face the Ashes</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR10USD1187B99F3F1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Burlington, Ontario, Canada</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tweeterfriendly Music</td>\n",
       "      <td>189.57016</td>\n",
       "      <td>1</td>\n",
       "      <td>SOHKNRJ12A6701D1F8</td>\n",
       "      <td>Drop of Rain</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ARGSJW91187B9B1D6B</td>\n",
       "      <td>35.21962</td>\n",
       "      <td>North Carolina</td>\n",
       "      <td>-80.01955</td>\n",
       "      <td>JennyAnyKind</td>\n",
       "      <td>218.77506</td>\n",
       "      <td>1</td>\n",
       "      <td>SOQHXMF12AB0182363</td>\n",
       "      <td>Young Boy Blues</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_id  artist_latitude              artist_location  \\\n",
       "0  AR7G5I41187FB4CE6C              NaN              London, England   \n",
       "1  AR8ZCNI1187B9A069B              NaN                                \n",
       "2  ARXR32B1187FB57099              NaN                                \n",
       "3  AR10USD1187B99F3F1              NaN  Burlington, Ontario, Canada   \n",
       "4  ARGSJW91187B9B1D6B         35.21962               North Carolina   \n",
       "\n",
       "   artist_longitude            artist_name   duration  num_songs  \\\n",
       "0               NaN               Adam Ant  233.40363          1   \n",
       "1               NaN       Planet P Project  269.81832          1   \n",
       "2               NaN                    Gob  209.60608          1   \n",
       "3               NaN  Tweeterfriendly Music  189.57016          1   \n",
       "4         -80.01955           JennyAnyKind  218.77506          1   \n",
       "\n",
       "              song_id            title  year  \n",
       "0  SONHOTT12A8C13493C  Something Girls  1982  \n",
       "1  SOIAZJW12AB01853F1       Pink World  1984  \n",
       "2  SOFSOCN12A8C143F5D   Face the Ashes  2007  \n",
       "3  SOHKNRJ12A6701D1F8     Drop of Rain     0  \n",
       "4  SOQHXMF12AB0182363  Young Boy Blues     0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(71, 10)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logs Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>auth</th>\n",
       "      <th>firstName</th>\n",
       "      <th>gender</th>\n",
       "      <th>itemInSession</th>\n",
       "      <th>lastName</th>\n",
       "      <th>length</th>\n",
       "      <th>level</th>\n",
       "      <th>location</th>\n",
       "      <th>method</th>\n",
       "      <th>page</th>\n",
       "      <th>registration</th>\n",
       "      <th>sessionId</th>\n",
       "      <th>song</th>\n",
       "      <th>status</th>\n",
       "      <th>ts</th>\n",
       "      <th>userAgent</th>\n",
       "      <th>userId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Frumpies</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>0</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>134.47791</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>Fuck Kitty</td>\n",
       "      <td>200</td>\n",
       "      <td>1541903636796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kenny G with Peabo Bryson</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>264.75057</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>By The Time This Night Is Over</td>\n",
       "      <td>200</td>\n",
       "      <td>1541903770796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Biffy Clyro</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>2</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>189.83138</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>God &amp; Satan</td>\n",
       "      <td>200</td>\n",
       "      <td>1541904034796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>None</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Lily</td>\n",
       "      <td>F</td>\n",
       "      <td>0</td>\n",
       "      <td>Burns</td>\n",
       "      <td>NaN</td>\n",
       "      <td>free</td>\n",
       "      <td>New York-Newark-Jersey City, NY-NJ-PA</td>\n",
       "      <td>GET</td>\n",
       "      <td>Home</td>\n",
       "      <td>1.540621e+12</td>\n",
       "      <td>456</td>\n",
       "      <td>None</td>\n",
       "      <td>200</td>\n",
       "      <td>1541910841796</td>\n",
       "      <td>\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HIM</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Lily</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>Burns</td>\n",
       "      <td>212.06159</td>\n",
       "      <td>free</td>\n",
       "      <td>New York-Newark-Jersey City, NY-NJ-PA</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.540621e+12</td>\n",
       "      <td>456</td>\n",
       "      <td>Beautiful</td>\n",
       "      <td>200</td>\n",
       "      <td>1541910973796</td>\n",
       "      <td>\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      artist       auth firstName gender  itemInSession  \\\n",
       "0                   Frumpies  Logged In  Anabelle      F              0   \n",
       "1  Kenny G with Peabo Bryson  Logged In  Anabelle      F              1   \n",
       "2                Biffy Clyro  Logged In  Anabelle      F              2   \n",
       "3                       None  Logged In      Lily      F              0   \n",
       "4                        HIM  Logged In      Lily      F              1   \n",
       "\n",
       "  lastName     length level                                     location  \\\n",
       "0  Simpson  134.47791  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "1  Simpson  264.75057  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "2  Simpson  189.83138  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "3    Burns        NaN  free        New York-Newark-Jersey City, NY-NJ-PA   \n",
       "4    Burns  212.06159  free        New York-Newark-Jersey City, NY-NJ-PA   \n",
       "\n",
       "  method      page  registration  sessionId                            song  \\\n",
       "0    PUT  NextSong  1.541044e+12        455                      Fuck Kitty   \n",
       "1    PUT  NextSong  1.541044e+12        455  By The Time This Night Is Over   \n",
       "2    PUT  NextSong  1.541044e+12        455                     God & Satan   \n",
       "3    GET      Home  1.540621e+12        456                            None   \n",
       "4    PUT  NextSong  1.540621e+12        456                       Beautiful   \n",
       "\n",
       "   status             ts                                          userAgent  \\\n",
       "0     200  1541903636796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "1     200  1541903770796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "2     200  1541904034796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "3     200  1541910841796  \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...   \n",
       "4     200  1541910973796  \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...   \n",
       "\n",
       "  userId  \n",
       "0     69  \n",
       "1     69  \n",
       "2     69  \n",
       "3     32  \n",
       "4     32  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_files = list((PATH / 'log_data').rglob('*.json'))\n",
    "log_df = pd.read_json(logs_files[0], lines=True)\n",
    "log_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95, 18), (1, 10))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_df.shape, song_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs_files = list((PATH / 'log_data').rglob('*.json'))\n",
    "for i, f in enumerate(logs_files):\n",
    "    if i == 0:\n",
    "        logs_df = pd.read_json(f, lines=True)\n",
    "    else:\n",
    "        logs_df = logs_df.append(pd.read_json(f, lines=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>auth</th>\n",
       "      <th>firstName</th>\n",
       "      <th>gender</th>\n",
       "      <th>itemInSession</th>\n",
       "      <th>lastName</th>\n",
       "      <th>length</th>\n",
       "      <th>level</th>\n",
       "      <th>location</th>\n",
       "      <th>method</th>\n",
       "      <th>page</th>\n",
       "      <th>registration</th>\n",
       "      <th>sessionId</th>\n",
       "      <th>song</th>\n",
       "      <th>status</th>\n",
       "      <th>ts</th>\n",
       "      <th>userAgent</th>\n",
       "      <th>userId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Frumpies</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>0</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>134.47791</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>Fuck Kitty</td>\n",
       "      <td>200</td>\n",
       "      <td>1541903636796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kenny G with Peabo Bryson</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>264.75057</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>By The Time This Night Is Over</td>\n",
       "      <td>200</td>\n",
       "      <td>1541903770796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Biffy Clyro</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Anabelle</td>\n",
       "      <td>F</td>\n",
       "      <td>2</td>\n",
       "      <td>Simpson</td>\n",
       "      <td>189.83138</td>\n",
       "      <td>free</td>\n",
       "      <td>Philadelphia-Camden-Wilmington, PA-NJ-DE-MD</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.541044e+12</td>\n",
       "      <td>455</td>\n",
       "      <td>God &amp; Satan</td>\n",
       "      <td>200</td>\n",
       "      <td>1541904034796</td>\n",
       "      <td>\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>None</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Lily</td>\n",
       "      <td>F</td>\n",
       "      <td>0</td>\n",
       "      <td>Burns</td>\n",
       "      <td>NaN</td>\n",
       "      <td>free</td>\n",
       "      <td>New York-Newark-Jersey City, NY-NJ-PA</td>\n",
       "      <td>GET</td>\n",
       "      <td>Home</td>\n",
       "      <td>1.540621e+12</td>\n",
       "      <td>456</td>\n",
       "      <td>None</td>\n",
       "      <td>200</td>\n",
       "      <td>1541910841796</td>\n",
       "      <td>\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HIM</td>\n",
       "      <td>Logged In</td>\n",
       "      <td>Lily</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>Burns</td>\n",
       "      <td>212.06159</td>\n",
       "      <td>free</td>\n",
       "      <td>New York-Newark-Jersey City, NY-NJ-PA</td>\n",
       "      <td>PUT</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>1.540621e+12</td>\n",
       "      <td>456</td>\n",
       "      <td>Beautiful</td>\n",
       "      <td>200</td>\n",
       "      <td>1541910973796</td>\n",
       "      <td>\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      artist       auth firstName gender  itemInSession  \\\n",
       "0                   Frumpies  Logged In  Anabelle      F              0   \n",
       "1  Kenny G with Peabo Bryson  Logged In  Anabelle      F              1   \n",
       "2                Biffy Clyro  Logged In  Anabelle      F              2   \n",
       "3                       None  Logged In      Lily      F              0   \n",
       "4                        HIM  Logged In      Lily      F              1   \n",
       "\n",
       "  lastName     length level                                     location  \\\n",
       "0  Simpson  134.47791  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "1  Simpson  264.75057  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "2  Simpson  189.83138  free  Philadelphia-Camden-Wilmington, PA-NJ-DE-MD   \n",
       "3    Burns        NaN  free        New York-Newark-Jersey City, NY-NJ-PA   \n",
       "4    Burns  212.06159  free        New York-Newark-Jersey City, NY-NJ-PA   \n",
       "\n",
       "  method      page  registration  sessionId                            song  \\\n",
       "0    PUT  NextSong  1.541044e+12        455                      Fuck Kitty   \n",
       "1    PUT  NextSong  1.541044e+12        455  By The Time This Night Is Over   \n",
       "2    PUT  NextSong  1.541044e+12        455                     God & Satan   \n",
       "3    GET      Home  1.540621e+12        456                            None   \n",
       "4    PUT  NextSong  1.540621e+12        456                       Beautiful   \n",
       "\n",
       "   status             ts                                          userAgent  \\\n",
       "0     200  1541903636796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "1     200  1541903770796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "2     200  1541904034796  \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4...   \n",
       "3     200  1541910841796  \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...   \n",
       "4     200  1541910973796  \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebK...   \n",
       "\n",
       "  userId  \n",
       "0     69  \n",
       "1     69  \n",
       "2     69  \n",
       "3     32  \n",
       "4     32  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8056, 18)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['artist', 'auth', 'firstName', 'gender', 'itemInSession', 'lastName',\n",
       "       'length', 'level', 'location', 'method', 'page', 'registration',\n",
       "       'sessionId', 'song', 'status', 'ts', 'userAgent', 'userId'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6820"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(logs_df['page'] == 'NextSong').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NextSong            6820\n",
       "Home                 806\n",
       "Login                 92\n",
       "Logout                90\n",
       "Downgrade             60\n",
       "Settings              56\n",
       "Help                  47\n",
       "About                 36\n",
       "Upgrade               21\n",
       "Save Settings         10\n",
       "Error                  9\n",
       "Submit Upgrade         8\n",
       "Submit Downgrade       1\n",
       "Name: page, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df['page'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/Users/imad/Documents/data_engineering/project-1/data/log_data/2018/11/2018-11-11-events.json')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_files[0].resolve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}