10_stc_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look-up table from FAA DRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ADFRAWD', 'ADNPRM', 'AC', 'AB', 'POLICY', 'CAM', 'CAR', 'CANIC',\n",
       "       'ADFREAD', 'ELOS', 'CFRFRSFAR', 'NORSEE', 'NPRM', 'PMA', 'SAIB',\n",
       "       'SCFINAL', 'SCPROPOSED', 'SFAR', 'STC', 'TSOI', 'TSO', 'FAR',\n",
       "       'TCDSMODEL', 'UNAPPROVED_PARTS_NOTIFICATIONS', 'INFO', 'SAFO',\n",
       "       'ORDER_8900.1', 'CLARIFY_POLICY', 'AFS-1_MEMORANDUMS', 'AT_JTA',\n",
       "       'AIRCRAFT_STANDARDIZED_CURRICULUM', 'AIRCRAFT_MASTER_SCHEDULE',\n",
       "       'OTHER_AWO', 'ALERTS', 'OTHER_AWARDS_INFORMATION_GUIDES',\n",
       "       '8900.1_EDITORIAL_CORRECTIONS', 'OTHER_EFB_RESEARCH_REPORTS',\n",
       "       'OTHER_EFB_CHECKLISTS', 'OTHER_FAA_90_DAY_SAFETY_REVIEW',\n",
       "       'OTHER_CPD_6.03', 'AFS_FFS_UPDATEPUB', 'AFS_FFS_UPDATES', 'FOEB',\n",
       "       'FSB_REPORTS', 'AFS_POLICY_DEV_MEMOS',\n",
       "       'OTHER_FLIGHT_STANDARDS_ORM_WORKSHEETS', 'AFS_FOCUS_TEAMS',\n",
       "       'GA_JTA', 'BULLETINS', 'OTHER_PS_HANDBOOKS',\n",
       "       'OTHER_INFORMATION_GUIDES', 'PILOT_QUALIFICATION_CURRICULUM',\n",
       "       'OTHER_INTERNATIONAL_PUBLICATIONS', 'OTHER_JOB_AIDS',\n",
       "       'OTHER_LASER_INVESTIGATION_REFERENCES', 'MMEL_POLICY_LETTERS',\n",
       "       'MMEL', 'NOTICES', 'OSR', 'OPSS_GUIDANCE', 'OSWG', 'ORDER_8300.10',\n",
       "       'ORDER_8400.10', 'ORDER_8700.1', 'ORDER_8740.1', 'ORDERS',\n",
       "       'PART_129_OPSPEC_JA', 'OTHER_PS_FEDERAL_AVIATION_ACTS',\n",
       "       'OTHER_PS_LEGAL_INTERPRETATIONS', 'OTHER_PS_POLICY_MEMORANDA',\n",
       "       'OTHER_PS_PREAMBLES', 'OTHER_QMS_AND_BP', 'OTHER_RCCB',\n",
       "       'OTHER_RIRTP', 'SAS_AXH_DCT', 'SAS_DCT', 'STCRELIEFAPPLETTER',\n",
       "       '8900.1_SUMMARY_OF_CHANGES', 'OTHER_SPRS', 'OTHER_VPM',\n",
       "       'LEGAL_INTERPRETATIONS'], dtype=object)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "metadata_mapping = pd.read_excel(os.path.join(os.getcwd(),'FAA DRS','DRS Document Types Metadata Mapping.xlsx')) # from https://drs.faa.gov/help/helpdetails and https://drs.faa.gov/help/helpdetails\n",
    "doc_types = pd.unique(metadata_mapping.loc[:, \"Document Type Name in API request\"])\n",
    "doc_types"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We just want to collect STCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_types = ['STC'] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just a test to collect STC list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "df_drs = None\n",
    "doc_type = 'STC'\n",
    "\n",
    "with open(os.path.join(os.getcwd(),'drs_api.txt'), \"r\") as f:\n",
    "    x_api_key = f.read()\n",
    "\n",
    "offset = 1250\n",
    "\n",
    "url = \"https://drs.faa.gov/api/drs/data-pull/\"+doc_type+\"?offset=\"+str(offset) # ?drs:offset ?drs:offset\n",
    "headers = {'x-api-key':x_api_key}\n",
    "response = requests.get(url, headers=headers)\n",
    "print(url)\n",
    "\n",
    "df_drs = pd.DataFrame.from_dict(response.json()['documents'])\n",
    "\n",
    "df_drs.to_excel(os.path.join(os.getcwd(),'database','stc_collection_test_offset='+str(offset)+'.xlsx'),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'loc'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\10_stc_collection.ipynb Cell 7\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/victor/Documents/DeepLearning/FAA%20NLP%20Project/10_stc_collection.ipynb#W6sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m df_drs\u001b[39m.\u001b[39;49mloc[:, :]\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'loc'"
     ]
    }
   ],
   "source": [
    "df_drs.loc[:, :]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collect full STC list from FAA DRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import requests\n",
    "import os\n",
    "import datetime\n",
    "\n",
    "df_drs = None\n",
    "logging.basicConfig(filename='10_stc_collection.log', encoding='utf-8',format='%(asctime)s %(message)s', level=logging.INFO)\n",
    "\n",
    "stc_path = os.path.join(os.getcwd(),'database','stc.xlsx')\n",
    "\n",
    "with open(os.path.join(os.getcwd(),'drs_api.txt'), \"r\") as f:\n",
    "    x_api_key = f.read()\n",
    "\n",
    "for doc_type in doc_types:\n",
    "    hasMoreItems = True\n",
    "    offset = 0\n",
    "    logging.info(\"FileType: \"+doc_type)\n",
    "\n",
    "    while hasMoreItems:\n",
    "        try:\n",
    "            url = \"https://drs.faa.gov/api/drs/data-pull/\"+doc_type+\"?offset=\"+str(offset)\n",
    "            headers = {'x-api-key':x_api_key}\n",
    "            response = requests.get(url, headers=headers)\n",
    "\n",
    "            if df_drs is None:\n",
    "                df_drs = pd.DataFrame.from_dict(response.json()['documents'])\n",
    "            else:\n",
    "                df_drs = pd.concat([df_drs, pd.DataFrame.from_dict(response.json()['documents'])])\n",
    "            \n",
    "            hasMoreItems = response.json()['summary']['hasMoreItems'] and (offset<response.json()['summary']['totalItems'])\n",
    "            logging.info(\"Passed with Offset: \"+str(offset)+\"/\"+str(response.json()['summary']['totalItems'])+\"; hasMoreItems=\"+str(response.json()['summary']['hasMoreItems'])+\"; url: \"+url)\n",
    "            offset = offset + response.json()['summary']['count']\n",
    "        except:\n",
    "            logging.info(\"Failed with Offset: \"+str(offset)+\"; url: \"+url)\n",
    "\n",
    "        df_drs.to_excel(stc_path,index=False)\n",
    "\n",
    "try:\n",
    "    #os.rename(os.path.join(os.getcwd(),'stc.xlsx'), os.path.join(os.getcwd(),'data','stc','stc_'+datetime.datetime.now().strftime(\"%Y%m%d\"-\"%H%M%S\")+'.xlsx'))\n",
    "    os.rename(stc_path, \n",
    "              os.path.join(os.getcwd(),'database','stc_'+datetime.datetime.now().strftime(\"%Y%m%d - %H%M%S\")+'.xlsx'))\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting PDFs from FAA DRS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### A quick try to collect for one given STC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\victor\\AppData\\Local\\Temp\\ipykernel_17668\\2185806852.py:15: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome(chrome_options=chrome_options)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\database\\data\\stc\\pdf\\SB03032CH.pdf\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import re\n",
    "import glob\n",
    "import os\n",
    "\n",
    "dl_dir = os.path.join(os.getcwd(),'database','data','stc','pdf')\n",
    "\n",
    "chrome_options = webdriver.ChromeOptions()\n",
    "prefs = {'download.default_directory' : dl_dir}\n",
    "chrome_options.add_experimental_option('prefs', prefs)\n",
    "\n",
    "driver = webdriver.Chrome(chrome_options=chrome_options)\n",
    "\n",
    "#driver.get(\"https://drs.faa.gov/browse/excelExternalWindow/BFEFEB9191F15EB286258876006E8C75.0001?modalOpened=true\")\n",
    "#driver.get(\"https://drs.faa.gov/browse/excelExternalWindow/C467DBFA533DC75A85256CC2000E3B05.0001?modalOpened=true\")\n",
    "#collect_drs_pdf(\"https://drs.faa.gov/browse/excelExternalWindow/C467DBFA533DC75A85256CC2000E3B05.0001?modalOpened=true\", driver)\n",
    "stc_name_from_site = collect_drs_pdf(\"https://drs.faa.gov/browse/excelExternalWindow/4AED288A0E84EDD3862579640077B32A.0001?modalOpened=true\", driver)\n",
    "\n",
    "list_of_files = glob.glob(os.path.join(dl_dir,\"*.pdf\"))\n",
    "latest_file = max(list_of_files, key=os.path.getctime)\n",
    "print(latest_file)\n",
    "\n",
    "if os.path.exists(os.path.join(dl_dir, stc_name_from_site)) and (stc_name_from_site in latest_file):\n",
    "    os.rename(latest_file, os.path.join(dl_dir, \"SB03032CH\"+\"__\"+\"Current\"+\"__\"+\"4AED288A0E84EDD3862579640077B32A\"+\".pdf\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Main function for DRS PDF collector "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import time\n",
    "import datetime\n",
    "import os.path\n",
    "\n",
    "def collect_drs_pdf(url, driver):\n",
    "    driver.get(url)\n",
    "    stc_name_from_site = \"NO STC DOWNLOADED\"\n",
    "    dl_dir = os.path.join(os.getcwd(),'database','data','stc','pdf')\n",
    "\n",
    "    try:\n",
    "        WebDriverWait(driver, 30).until(\n",
    "            EC.presence_of_element_located((By.CSS_SELECTOR, \"[title*='Supplemental Type Certificates (STC)']\")) \n",
    "        )  \n",
    "\n",
    "        if len(driver.find_elements(By.ID, \"printButton\"))>0 and len(driver.find_elements(By.ID, \"STC Number\"))>0:\n",
    "            stc_name_from_site = \"no_pdf_stc_\"+datetime.datetime.now().strftime(\"%H%M%S%f\")+\".pdf\"\n",
    "            with open(os.path.join(dl_dir,stc_name_from_site), 'w') as f:\n",
    "                f.write(url)\n",
    "        else:\n",
    "            WebDriverWait(driver, 30).until(\n",
    "                EC.element_to_be_clickable((By.CSS_SELECTOR, \"#download\"))\n",
    "            )\n",
    "            time_counter = 0\n",
    "            while driver.find_element(By.CSS_SELECTOR, \"#loadingBar\").get_attribute(\"class\")!=\"hidden\" and time_counter<60:\n",
    "                time.sleep(0.5)\n",
    "                time_counter = time_counter + 1\n",
    "            driver.find_element(By.CSS_SELECTOR, \"#download\").click()\n",
    "\n",
    "            stc_name_from_site = driver.find_element(By.CLASS_NAME, \"doc-view\").find_element(By.CLASS_NAME, \"ng-star-inserted\").text\n",
    "            if stc_name_from_site[0:2]==\"- \":\n",
    "                stc_name_from_site = stc_name_from_site[2:]\n",
    "\n",
    "            time_counter = 0\n",
    "            while not os.path.exists(os.path.join(dl_dir, stc_name_from_site)) and time_counter<60:\n",
    "                time.sleep(0.5)\n",
    "                time_counter = time_counter + 1\n",
    "            time.sleep(0.5)\n",
    "    except TimeoutException as ex:\n",
    "        pass\n",
    "    except:\n",
    "        pass        \n",
    "\n",
    "    return stc_name_from_site"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Collects all DRS PDF no optimization using the previously collected full STC list from FAA DRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77965, 26)\n",
      "(77661, 26)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "df_stc = pd.read_excel(os.path.join(os.getcwd(),'database','stc.xlsx'))\n",
    "print(df_stc.shape)\n",
    "df_stc = df_stc.drop_duplicates()\n",
    "print(df_stc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\victor\\AppData\\Local\\Temp\\ipykernel_13400\\519162152.py:17: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome(chrome_options=chrome_options)\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "import os\n",
    "import logging\n",
    "import re\n",
    "import glob\n",
    "import time\n",
    "regcomp = re.compile(\" \\(\")\n",
    "\n",
    "dl_dir = os.path.join(os.getcwd(),'database','data','stc','pdf')\n",
    "\n",
    "chrome_options = webdriver.ChromeOptions()\n",
    "prefs = {'download.default_directory' : dl_dir}\n",
    "chrome_options.add_experimental_option('prefs', prefs)\n",
    "logging.basicConfig(filename='drs_stc_collection.log', encoding='utf-8',format='%(asctime)s %(message)s', level=logging.INFO)\n",
    "logging.info(\"DRS PDF collector no optimization\")\n",
    "\n",
    "driver = webdriver.Chrome(chrome_options=chrome_options)\n",
    "for index, row in df_stc.sample(frac=1).iterrows():\n",
    "    matched_stc = regcomp.search(row['drs:chronicleId'])\n",
    "    if matched_stc is not None:\n",
    "        stc_num = row['drs:chronicleId'][0:matched_stc.span()[0]]\n",
    "    else:\n",
    "        stc_num = row['drs:chronicleId']\n",
    "\n",
    "    if not os.path.exists(os.path.join(dl_dir,row['drs:chronicleId']+\"__\"+row['drs:status']+\"__\"+row['documentGuid']+\".pdf\")):\n",
    "        try:            \n",
    "            stc_name_from_site = collect_drs_pdf(row['documentURL']+\"?modalOpened=true\", driver)\n",
    "\n",
    "            list_of_files = glob.glob(os.path.join(dl_dir,\"*.pdf\"))\n",
    "            latest_file = sorted(list_of_files, key=os.path.getctime, reverse=True)[:10]\n",
    "\n",
    "            time.sleep(0.2)\n",
    "            if (stc_name_from_site in [os.path.basename(val) for val in latest_file]):\n",
    "                os.rename(os.path.join(dl_dir, stc_name_from_site), os.path.join(dl_dir, row['drs:chronicleId']+\"__\"+row['drs:status']+\"__\"+row['documentGuid']+\".pdf\"))\n",
    "                \n",
    "            logging.info(\"drs:chronicleId: \"+row['drs:chroni  cleId']+\" -> Pdf dl Pass\")\n",
    "        except:\n",
    "            logging.info(\"drs:chronicleId: \"+row['drs:chronicleId']+\" -> Pdf dl Fail\")\n",
    "\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\"no_pdf_stc_211603973870.pdf\" in ['no_pdf_stc_211603973870.pdf', 'no_pdf_stc_211601549633.pdf', 'ST02959AT.pdf', 'no_pdf_stc_211555126680.pdf', 'no_pdf_stc_211550821480.pdf', 'no_pdf_stc_211548432536.pdf', 'no_pdf_stc_211546012346.pdf', 'SA02571AT-D.pdf']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove files incorrectly downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "\n",
    "dl_dir = os.path.join(os.getcwd(),'database','data','stc')\n",
    "\n",
    "for file in glob.glob(os.path.join(os.path.join(dl_dir,'pdf'),\"*.pdf\")):\n",
    "    if \"__\" not in os.path.basename(file):\n",
    "        try:\n",
    "            os.rename(file, os.path.join(os.path.join(os.path.join(dl_dir,'pdf did not work')), os.path.basename(file)))\n",
    "        except:\n",
    "            os.remove(file)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Collects all DRS PDF multi thread optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from multiprocessing.pool import ThreadPool as Pool\n",
    "import os\n",
    "\n",
    "path = os.path.join(os.getcwd(),'database','data','stc','pdf')\n",
    "stcs = [\"SR02762SE.pdf\", \"SR02650SE.pdf\", \"SR04501CH.pdf\", \"ST04442NY.pdf\", \"SB04185CH.pdf\", \"SR04557NY.pdf\", \"SR01813CH.pdf\", \"SH5977NM.pdf\"]\n",
    "urls = [val for val in stcs]\n",
    "\n",
    "df = pd.DataFrame(stcs, columns=['stcs'])\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    with Pool(5) as p:\n",
    "        print(p.map(collect_drs_pdf, [1, 2, 3]))\n",
    "\n",
    "def sitesize(url):\n",
    "    with requests.get(url) as u:\n",
    "        return url, len(u.content)\n",
    "\n",
    "pool = Pool(10)\n",
    "for result in pool.imap_unordered(sitesize, sites):\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\victor\\AppData\\Local\\Temp\\ipykernel_8144\\4141224413.py:20: DeprecationWarning: use options instead of chrome_options\n",
      "  webdriver = webdriver.Chrome(chrome_options=chrome_options)\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import pandas as pd\n",
    "from functools import partial\n",
    "from itertools import repeat\n",
    "#from multiprocessing import Pool\n",
    "from multiprocess import Pool\n",
    "import os\n",
    "\n",
    "urls = [\"https://drs.faa.gov/browse/excelExternalWindow/d2b9293f-76c6-499d-8328-dffcda388dc5?modalOpened=true\", \n",
    "        \"https://drs.faa.gov/browse/excelExternalWindow/0D55897B0F4DADC885256CC100821507.0001?modalOpened=true\", \n",
    "        \"https://drs.faa.gov/browse/excelExternalWindow/5850CE209CB40D16862579E2006009B7.0001?modalOpened=true\", \n",
    "        \"https://drs.faa.gov/browse/excelExternalWindow/606dd138-75b4-4d49-96c6-e525c00a6730?modalOpened=true\", \n",
    "        \"https://drs.faa.gov/browse/excelExternalWindow/7b1708f9-c7c9-4020-8779-c64eb3e80e0b?modalOpened=true\"]\n",
    "\n",
    "chrome_options = webdriver.ChromeOptions()\n",
    "prefs = {'download.default_directory' : os.path.join(os.getcwd(),'database','data','stc','pdf')}\n",
    "\n",
    "chrome_options.add_experimental_option('prefs', prefs)\n",
    "webdriver = webdriver.Chrome(chrome_options=chrome_options)\n",
    "\n",
    "#collect_drs_pdf(\"https://drs.faa.gov/browse/excelExternalWindow/FAA000000000LEGALINTPR2013001PDF.0001?modalOpened=true\", webdriver)\n",
    "\n",
    "df = pd.DataFrame(urls, columns=['urls'])\n",
    "\n",
    "with Pool(1) as p:\n",
    "    p.map_async(collect_drs_pdf, [(urls[0], webdriver)])\n",
    "\n",
    "'''\n",
    "with Pool() as pool:\n",
    "    pool.map(collect_drs_pdf,[urls[0], webdriver])\n",
    "\n",
    "with Pool() as pool:\n",
    "    pool.starmap(collect_drs_pdf,[(urls[0], webdriver)])\n",
    "    #pool.starmap(collect_drs_pdf, list(zip(iter(urls), repeat(webdriver))))\n",
    "    #pool.starmap(collect_drs_pdf, zip(urls, repeat(webdriver)))\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8c49a9e6d6b122d0bc46f085b0f9b84866f77188fdff3252b8a9eae176833897"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}