TEST-all-scrape-worlds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib.request\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup,NavigableString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#  THIS IS THE SCRIPT FOR SCRAPE THE DATABASE FROM IBJJF FROM WORLD BJJ COMPETITION\n",
    "\n",
    "data = []\n",
    "categories = []\n",
    "a=[]\n",
    "listz=[]\n",
    "\n",
    "\n",
    "for i in range(1996,2013):      # Number of pages plus one \n",
    "    if i<= 2011:\n",
    "        url='https://ibjjf.com/events/results/{}-world-jiu-jitsu-ibjjf-championship'.format(i)\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text,\"html.parser\")\n",
    "        result = soup.find('div', class_ = 'col-sm-12 col-md-6 athletes-result')\n",
    "\n",
    "\n",
    "\n",
    "        for new in result.find_all('div', {'class':'col-sm-12 athletes'},'tr'):\n",
    "            for athletes in new.find_all('tr'):\n",
    "                data.append(athletes.find('td',class_ = 'place').text)\n",
    "                data.append(athletes.find('div',class_ = 'athlete-name').text)\n",
    "                data.append(athletes.find('div',class_ = 'academy-name').text)\n",
    "                data.append(soup.find('h3', {'class':'title-section text-center'}).text)\n",
    "\n",
    "        for new in result.find_all('div',{'class':'category mt-4 mb-3'}):\n",
    "            categories.append(new.get_text(strip=True))\n",
    "    else:\n",
    "        worldcc = [114, 177, 272, 415, 535, 730, 926, 1209]\n",
    "        for j in worldcc:\n",
    "            url='https://www.ibjjfdb.com/ChampionshipResults/{}/PublicResults'.format(j)\n",
    "            response = requests.get(url)\n",
    "            soup = BeautifulSoup(response.text,\"html.parser\")\n",
    "            result = soup.find('div', class_ = 'col-xs-12 col-md-6 col-athlete')\n",
    "\n",
    "            for new in result.find_all('div', {'class':'list'}):\n",
    "                for athletes in new.find_all('div', {'class':'athlete-item'}):\n",
    "                    data.append(athletes.find('div',{\"class\": 'position-athlete'}).get_text(strip=True).replace(\"\\r\"\"\\n\"'                         ', \"\"))\n",
    "                    data.append(athletes.find('p').get_text(strip=True).replace(\"\\r\"\"\\n\"'                         ', \"\"))\n",
    "                    data.append(athletes.find('span').get_text(strip=True).replace(\"\\r\"\"\\n\"'                         ', \"\"))\n",
    "                    data.append(soup.find_all('h2', {'class':'title'})[1].text.replace(\"\\r\\n\"\" \", \"\"))\n",
    "\n",
    "            for new in result.find_all('h4',{'class':'subtitle'}):\n",
    "                categories.append(new.get_text(strip=True).replace(\"\\r\"\"\\n\"'                         ', \"\"))\n",
    "                \n",
    "\n",
    "inner_size = 4\n",
    "finaldata = [ data[i:i+inner_size] for i in range(0, len(data), inner_size) ]\n",
    "dataf = pd.DataFrame(finaldata)\n",
    "dataf.columns =['Place','Athlete','Team','Tournament']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# copying the colum PLACE of the dataf and excluding the first line of this one\n",
    "newplace= dataf[['Place']].copy()\n",
    "newplace2= newplace.iloc[1:]\n",
    "# transform in a list\n",
    "newplace3= dataf['Place'].tolist()\n",
    "#transform these string items and in integer\n",
    "results1 = map(int, newplace3)\n",
    "\n",
    "#transform places of athletes in 1 or -1 and deleting the first row\n",
    "a = [-1 if i <= 1 else 1 for i in results1]\n",
    "del a[0]\n",
    "\n",
    "index = 0\n",
    "while len(listz)<len(a):\n",
    "    for y in a:\n",
    "        if y>=0:\n",
    "            listz.append(categories[index])\n",
    "        else:\n",
    "            index = (index + 1) % len(categories)\n",
    "            listz.append(categories[index])\n",
    "\n",
    "\n",
    "#matching both dataframes\n",
    "listz.insert(0, categories[0])\n",
    "\n",
    "datafz = pd.DataFrame(listz)\n",
    "datafz.columns =['Category']\n",
    "\n",
    "d=pd.concat([dataf, datafz], axis=1)\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=[]\n",
    "listz=[]\n",
    "\n",
    "inner_size = 4\n",
    "finaldata = [ data[i:i+inner_size] for i in range(0, len(data), inner_size) ]\n",
    "dataf = pd.DataFrame(finaldata)\n",
    "dataf.columns =['Place','Athlete','Team','Tournament']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# copying the colum PLACE of the dataf and excluding the first line of this one\n",
    "newplace= dataf[['Place']].copy()\n",
    "newplace2= newplace.iloc[1:]\n",
    "# transform in a list\n",
    "newplace3= dataf['Place'].tolist()\n",
    "#transform these string items and in integer\n",
    "results1 = map(int, newplace3)\n",
    "\n",
    "#transform places of athletes in 1 or -1 and deleting the first row\n",
    "a = [-1 if i <= 1 else 1 for i in results1]\n",
    "del a[0]\n",
    "\n",
    "index = 0\n",
    "while len(listz)<len(a):\n",
    "    for y in a:\n",
    "        if y>=0:\n",
    "            listz.append(categories[index])\n",
    "        else:\n",
    "            index = (index + 1) % len(categories)\n",
    "            listz.append(categories[index])\n",
    "\n",
    "\n",
    "#matching both dataframes\n",
    "listz.insert(0, categories[0])\n",
    "\n",
    "datafz = pd.DataFrame(listz)\n",
    "datafz.columns =['Category']\n",
    "\n",
    "d=pd.concat([dataf, datafz], axis=1)\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}