04_model_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3f5d1031-4a3c-4bd7-9089-4766917156f2",
   "metadata": {},
   "source": [
    "# Run inference on time to merge model trained previously\n",
    "\n",
    "\n",
    "## What we did previously\n",
    "\n",
    "In the previous [notebook](./03_model_training.ipynb) we trained machine learning models to classify a PR's `time_to_merge` into one of the 10 bins (or \"classes\"). We then deployed the model with the highest f1-score as a service using the model saved in s3.\n",
    "\n",
    "## In this step\n",
    "\n",
    "In this notebook, we are going to fetch the model that we previously trained, saved and stored in s3. We will send a payload to this model and see how it performs on the test data.\n",
    "# Time to Merge Prediction Inference Service\n",
    "\n",
    "In the previous notebook, we explored some basic machine learning models for predicting time to merge of a PR."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "091c5463",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import gzip\n",
    "import json\n",
    "import boto3\n",
    "import datetime\n",
    "import requests\n",
    "from github import Github\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "from io import BytesIO\n",
    "\n",
    "import joblib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import ceph_comm\n",
    "import process_pr\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from github_handling import connect_to_source, GITHUB_TIMEOUT_SECONDS, GitHubSingleton, GithubHandler\n",
    "\n",
    "\n",
    "load_dotenv(find_dotenv(), override=True)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') \n",
    "\n",
    "from warnings import simplefilter\n",
    "simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e4672acb",
   "metadata": {},
   "outputs": [],
   "source": [
    "## CEPH Bucket variables\n",
    "## Create a .env file on your local with the correct configs,\n",
    "\n",
    "ACTION = os.getenv(\"ACTION\", 0)\n",
    "ORG = os.getenv(\"GITHUB_ORG\")\n",
    "REPO = os.getenv(\"GITHUB_REPO\")\n",
    "TOKEN = os.getenv(\"GITHUB_ACCESS_TOKEN\") \n",
    "\n",
    "## S3 bucket credentials\n",
    "s3_endpoint_url = os.getenv(\"S3_ENDPOINT_URL\")\n",
    "s3_access_key = os.getenv(\"AWS_ACCESS_KEY_ID\")\n",
    "s3_secret_key = os.getenv(\"AWS_SECRET_ACCESS_KEY\")\n",
    "s3_bucket = os.getenv(\"S3_BUCKET\")\n",
    "\n",
    "s3_input_data_path = os.getenv(\"CEPH_BUCKET_PREFIX\")\n",
    "\n",
    "REMOTE = os.getenv(\"REMOTE\")\n",
    "RAW_DATA_PATH = os.path.join(\n",
    "    s3_input_data_path, \"srcopsmetrics/bot_knowledge\", ORG, REPO, \"PullRequest.json\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "66f1de79-b08e-48c2-9b01-3b5c5c8d23c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:github_handling: Github Handler __init__: 4982 remaining api calls\n",
      "INFO:github_handling: _is_api_exhausted: 4982 remaining api calls\n",
      "INFO:github_handling: _is_api_exhausted: 4982 remaining api calls\n"
     ]
    }
   ],
   "source": [
    "# Collect PRs and combine them\n",
    "cc = ceph_comm.CephCommunication(s3_endpoint_url, s3_access_key, s3_secret_key, s3_bucket)\n",
    "\n",
    "gs = GitHubSingleton()\n",
    "gh = GithubHandler(gs.github)\n",
    "repo = connect_to_source(ORG+'/'+REPO, gh)\n",
    "prs = repo.get_pulls(state='open')\n",
    "pr_ids = [pr for pr in prs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54f19596-766e-4b5e-ac23-f8da626af6f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_output(input_time):\n",
    "    \n",
    "    day1 = ''\n",
    "    day2 = ''\n",
    "    splitted = input_time.split(' - ')\n",
    "    start = splitted[0]\n",
    "    end = splitted[1]\n",
    "    \n",
    "    item1 = start.split(\", \")\n",
    "    item2 = end.split(\", \")\n",
    "    \n",
    "    if len(item1) == 2:\n",
    "        day1 = start.split(\", \")[0]\n",
    "        time1 = start.split(\", \")[1]\n",
    "    else:\n",
    "        time1 = start.split(\", \")[0]\n",
    "    \n",
    "    h1,m1,s1 = re.split(':', time1)\n",
    "\n",
    "    if len(item2) == 2:\n",
    "        day2 = end.split(\", \")[0]\n",
    "        time2 = end.split(\", \")[1]\n",
    "    else:\n",
    "        time2 = end.split(\", \")[0]\n",
    "    \n",
    "    h2,m2,s2 = re.split(':', time2)\n",
    "\n",
    "    if h1:\n",
    "        h1 = h1+\" hrs\"\n",
    "    else:\n",
    "        h1 = ''\n",
    "    if h2:\n",
    "        h2 = h2+\" hrs\"\n",
    "    else:\n",
    "        h2 = ''\n",
    "    if m1:\n",
    "        m1 = m1+\" mins\"\n",
    "    else:\n",
    "        m1 = ''\n",
    "    if m2:\n",
    "        m2 = m2+\" mins\"\n",
    "    else:\n",
    "        m2 = ''\n",
    "\n",
    "    s = \"Our model predicts that it will take between {} {} {} and {} {} {} for this PR to be merged from the time it was opened.\".format(day1,h1,m1,day2,h2,m2)\n",
    "        \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7cc6d73-9f74-4650-8aba-3e231c402218",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:github_handling: _is_api_exhausted: 4980 remaining api calls\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "collected PR jan/srcopsmetrics/bot_knowledge/aicoe-aiops/ocp-ci-analysis/PullRequest.json/PRs/622.json\n",
      "Our model predicts that it will take between  9 hrs 23 mins and  19 hrs 38 mins for this PR to be merged from the time it was opened.\n"
     ]
    }
   ],
   "source": [
    "if pr_ids:\n",
    "    d = process_pr.parse_pr_with_mi(pr_ids[0], gh)\n",
    "    pr_df = pd.DataFrame.from_dict(d, orient=\"index\")\n",
    "    pr_df = pr_df.transpose()\n",
    "\n",
    "    PR_FILENAME = os.path.join(\"PRs/\"+ str(pr_ids[0].number) + \".json\")\n",
    "    print(\"collected PR\", RAW_DATA_PATH+\"/\"+PR_FILENAME)\n",
    "\n",
    "    ## read model\n",
    "    MODEL_KEY = os.path.join(s3_input_data_path, ORG, REPO, \"ttm-model\")\n",
    "    MODEL_FILENAME = \"model.joblib\"\n",
    "\n",
    "    s3_resource = boto3.resource(\n",
    "        \"s3\",\n",
    "        endpoint_url=s3_endpoint_url,\n",
    "        aws_access_key_id=s3_access_key,\n",
    "        aws_secret_access_key=s3_secret_key,\n",
    "    )\n",
    "\n",
    "    buffer = BytesIO()\n",
    "    s3_object = s3_resource.Object(s3_bucket, f\"{MODEL_KEY}/{MODEL_FILENAME}\")\n",
    "    s3_object.download_fileobj(buffer)\n",
    "    model = joblib.load(buffer)\n",
    "\n",
    "    labels = cc.read_from_ceph(MODEL_KEY, \"label-map\")\n",
    "\n",
    "    prediction = model.predict(pr_df)\n",
    "    label = labels[labels[\"Class_Label\"] == prediction[0]][\"Class_Name\"].values\n",
    "    str_op = time_output(label[0])\n",
    "    \n",
    "    if ACTION:\n",
    "        pr = repo.get_pull(pr_ids[0].number)\n",
    "        pr.create_issue_comment(str_op)\n",
    "    else:\n",
    "        print(str_op)\n",
    "\n",
    "else:\n",
    "    print(\"No open PRs for running inference\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b3d3e60-469e-4dd9-aed9-853c05b76ba8",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "\n",
    "This notebook fetches the saved model from s3 and sends a payload to see how the model is performing on this new data. Additionally, we see that the evaluation scores in the classification report match the ones we saw in the training notebook. So, great, looks like our model are working as expected, and are ready to predict some times to merge for GitHub PRs! "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "675feb49d8b9f57a6c7838bbe31323f9ec5521e61f1f5118c23cd9b0b9101d34"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}