LLM_financial_analyst.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "624749b7-98e6-4333-9049-be15ec27134d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers datasets torch torchvision accelerate evaluate ipywidgets --quiet\n",
    "!pip install --upgrade transformers accelerate --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "38932267-6912-4941-8d3d-a183c9e594d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "from transformers import DataCollatorWithPadding\n",
    "from datasets import load_dataset, Dataset\n",
    "import numpy as np\n",
    "import evaluate\n",
    "import transformers, accelerate\n",
    "from accelerate import Accelerator\n",
    "\n",
    "accelerator = Accelerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "763b0b6d-2126-4142-8d04-3e7a2ba6c9d6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files removed: 4\n",
      "Filesystem      Size  Used Avail Use% Mounted on\n",
      "overlay          37G   83M   37G   1% /\n",
      "tmpfs            64M     0   64M   0% /dev\n",
      "tmpfs           7.7G     0  7.7G   0% /sys/fs/cgroup\n",
      "shm             4.0G   48K  4.0G   1% /dev/shm\n",
      "/dev/nvme2n1     25G   21G  4.1G  84% /home/studio-lab-user\n",
      "/dev/nvme0n1p1   50G   26G   25G  52% /mnt/sagemaker-nvme\n",
      "devtmpfs        7.7G     0  7.7G   0% /dev/tty\n",
      "tmpfs           7.7G   12K  7.7G   1% /proc/driver/nvidia\n",
      "tmpfs           7.7G     0  7.7G   0% /proc/acpi\n",
      "tmpfs           7.7G     0  7.7G   0% /sys/firmware\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!pip cache purge\n",
    "!df -h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5cef22a2-1564-4b69-a404-8ce2be7447a9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/sagemaker-distribution/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning: The repository for financial_phrasebank contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/financial_phrasebank\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset('financial_phrasebank', 'sentences_50agree')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c4e55ece-8b91-466c-99a8-13cbaf7a1eb5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'sentence': Value(dtype='string', id=None), 'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)}\n",
      "{'sentence': \"TeliaSonera TLSN said the offer is in line with its strategy to increase its ownership in core business holdings and would strengthen Eesti Telekom 's offering to its customers .\", 'label': 2}\n"
     ]
    }
   ],
   "source": [
    "# Inspect the dataset\n",
    "print(dataset['train'].features)\n",
    "print(dataset['train'][10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "551f6c60-9638-4a1d-8c17-1635fd125bfa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>According to Gran , the company has no plans t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Technopolis plans to develop in stages an area...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The international electronic industry company ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>With the new production plant the company woul...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>According to the company 's updated strategy f...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4841</th>\n",
       "      <td>LONDON MarketWatch -- Share prices ended lower...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4842</th>\n",
       "      <td>Rinkuskiai 's beer sales fell by 6.5 per cent ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4843</th>\n",
       "      <td>Operating profit fell to EUR 35.4 mn from EUR ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4844</th>\n",
       "      <td>Net sales of the Paper segment decreased to EU...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4845</th>\n",
       "      <td>Sales in Finland decreased by 10.5 % in Januar...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4846 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence  label\n",
       "0     According to Gran , the company has no plans t...      1\n",
       "1     Technopolis plans to develop in stages an area...      1\n",
       "2     The international electronic industry company ...      0\n",
       "3     With the new production plant the company woul...      2\n",
       "4     According to the company 's updated strategy f...      2\n",
       "...                                                 ...    ...\n",
       "4841  LONDON MarketWatch -- Share prices ended lower...      0\n",
       "4842  Rinkuskiai 's beer sales fell by 6.5 per cent ...      1\n",
       "4843  Operating profit fell to EUR 35.4 mn from EUR ...      0\n",
       "4844  Net sales of the Paper segment decreased to EU...      0\n",
       "4845  Sales in Finland decreased by 10.5 % in Januar...      0\n",
       "\n",
       "[4846 rows x 2 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the dataset into a DataFrame for preprocessing\n",
    "df = pd.DataFrame(dataset['train'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9dcffdaf-9500-4388-900e-4e8237fd3e7c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique labels in the dataset: [1 0 2]\n"
     ]
    }
   ],
   "source": [
    "# Check the unique values in the 'label' column\n",
    "print(\"Unique labels in the dataset:\", df['label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5ef60230-5496-45e2-80fd-5d6a9f56fb9c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['sentence', 'label'],\n",
       "    num_rows: 4846\n",
       "})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert to Hugging Face Dataset\n",
    "hf_dataset = Dataset.from_pandas(df[['sentence', 'label']])\n",
    "hf_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4fe26a43-6771-41cb-8af2-f440b4dfc175",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/sagemaker-distribution/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "/opt/conda/envs/sagemaker-distribution/lib/python3.10/site-packages/torch/_utils.py:776: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n"
     ]
    }
   ],
   "source": [
    "# OPTION 1\n",
    "# Load FinBERT model and tokenizer\n",
    "\n",
    "model_name = \"yiyanghkust/finbert-tone\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "19e2345f-df2b-434a-b5c9-6ab14b62445c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# OPTION 2\n",
    "# Load the fine-tuned finBERT model from your DIRECTORY\n",
    "\n",
    "model_dir = os.path.expanduser(\"~/LLM/finbert-finetuned/checkpoint-1455\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_dir)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "0b1bd744-1e98-4e4b-8111-2ae44aa373c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30873, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a50cee41-a827-465d-9af0-2ecd7985e4f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Tokenization function\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples['sentence'], truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "df0bc336-30cf-40a2-9f16-fe93e38bde5a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e67e7b383f5a44608293ccd9c7e690a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4846 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['sentence', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 4846\n",
       "})"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenize the dataset\n",
    "tokenized_dataset = hf_dataset.map(tokenize_function, batched=True)\n",
    "tokenized_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "43a7a81a-a4f2-4637-9721-f5a0a8dfb2d0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepare for PyTorch DataLoader\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b17e30fd-dd18-4e45-85b9-e86f293bd378",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Split the dataset into train and test\n",
    "train_test_split = tokenized_dataset.train_test_split(test_size=0.2)\n",
    "train_dataset = train_test_split['train']\n",
    "test_dataset = train_test_split['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ff8b72f8-b34a-4ba2-ba44-237c73cfcbd0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/sagemaker-distribution/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"~/LLM/finbert-finetuned\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='~/LLM/logs',\n",
    "    logging_steps=10,\n",
    "    save_steps=500,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "152ed68c-6957-4f1e-a193-3150f021bc37",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define the evaluation function\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "\n",
    "    # Load metrics using the evaluate library\n",
    "    accuracy_metric = evaluate.load(\"accuracy\")\n",
    "    precision_metric = evaluate.load(\"precision\")\n",
    "    recall_metric = evaluate.load(\"recall\")\n",
    "    f1_metric = evaluate.load(\"f1\")\n",
    "\n",
    "    # Compute the metrics\n",
    "    accuracy = accuracy_metric.compute(predictions=predictions, references=labels)[\"accuracy\"]\n",
    "    precision = precision_metric.compute(predictions=predictions, references=labels, average='weighted')[\"precision\"]\n",
    "    recall = recall_metric.compute(predictions=predictions, references=labels, average='weighted')[\"recall\"]\n",
    "    f1 = f1_metric.compute(predictions=predictions, references=labels, average='weighted')[\"f1\"]\n",
    "\n",
    "    return {\n",
    "        \"accuracy\": accuracy,\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": f1\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3c76e429-8f29-4fd1-a0f0-d2a06de176a0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define the Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,  # Include the compute_metrics function\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "4ec9268d-2b96-4206-92e7-c009506f3539",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1455' max='1455' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1455/1455 02:52, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>1.569430</td>\n",
       "      <td>0.819588</td>\n",
       "      <td>0.824238</td>\n",
       "      <td>0.819588</td>\n",
       "      <td>0.820912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.041200</td>\n",
       "      <td>1.372468</td>\n",
       "      <td>0.826804</td>\n",
       "      <td>0.826409</td>\n",
       "      <td>0.826804</td>\n",
       "      <td>0.826527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.362451</td>\n",
       "      <td>0.838144</td>\n",
       "      <td>0.838546</td>\n",
       "      <td>0.838144</td>\n",
       "      <td>0.838024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1455, training_loss=0.045306213430752675, metrics={'train_runtime': 172.8065, 'train_samples_per_second': 67.289, 'train_steps_per_second': 8.42, 'total_flos': 312101847191232.0, 'train_loss': 0.045306213430752675, 'epoch': 3.0})"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fine-tune the model\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "85baeab4-baa6-4654-8258-c49352b17f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='122' max='122' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [122/122 00:03]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Evaluate the model on the test set\n",
    "evaluation_results = trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "d838ba1c-99ba-4958-968f-160cf4b89497",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation Metrics:\n",
      "Accuracy: 0.838\n",
      "Precision: 0.839\n",
      "Recall: 0.838\n",
      "F1 Score: 0.838\n"
     ]
    }
   ],
   "source": [
    "# Print evaluation results\n",
    "print(\"Evaluation Metrics:\")\n",
    "print(f\"Accuracy: {evaluation_results['eval_accuracy']:.3f}\")\n",
    "print(f\"Precision: {evaluation_results['eval_precision']:.3f}\")\n",
    "print(f\"Recall: {evaluation_results['eval_recall']:.3f}\")\n",
    "print(f\"F1 Score: {evaluation_results['eval_f1']:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "7fa05066-0158-42bf-9ff6-f9f2baa66ed4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def analyze_sentiment_with_finbert(user_input):\n",
    "    # Assume that `finbert_model` and `tokenizer` are already defined and loaded\n",
    "    inputs = tokenizer(user_input, return_tensors=\"pt\", truncation=True, padding=True)\n",
    "    \n",
    "    # Move input tensors to the same device as the model\n",
    "    inputs = {key: value.to(device) for key, value in inputs.items()}\n",
    "    \n",
    "    # Get model outputs\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.logits\n",
    "    \n",
    "    # Apply softmax to get probabilities\n",
    "    probabilities = torch.softmax(logits, dim=1)\n",
    "    \n",
    "    # Move tensor to CPU before converting to NumPy\n",
    "    probabilities = probabilities.cpu().detach().numpy()\n",
    "    \n",
    "    # Get the sentiment with the highest probability\n",
    "    sentiment_idx = np.argmax(probabilities, axis=1)[0]\n",
    "\n",
    "    # Map the output to sentiment label\n",
    "    LABELS = {0: 'positive', 1: 'neutral', 2: 'negative'}\n",
    "    sentiment = LABELS[sentiment_idx]\n",
    "    \n",
    "    return sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ec782967-382b-47a6-b726-4641c031df94",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load gpt2-medium model and tokenizer for text generation\n",
    "\n",
    "gpt2_tokenizer = AutoTokenizer.from_pretrained(\"gpt2-medium\", cache_dir=\"/tmp\")\n",
    "gpt2_model = AutoModelForCausalLM.from_pretrained(\"gpt2-medium\", cache_dir=\"/tmp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "6da153a5-fba8-4e08-823e-367d13386bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example usage\n",
    "import torch \n",
    "# Check if CUDA is available\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Move model to the device\n",
    "gpt2_model = gpt2_model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "c5d6b0c8-0c59-4c08-beca-07658234de79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_advice_with_gpt2(user_question, sentiment):\n",
    "    # Construct the input prompt for GPT-2 without repetition\n",
    "    prompt = (\n",
    "        f\"You are a professional financial analyst.\\n Based on the '{user_question}' and the market sentiment being '{sentiment}'\\n \"\n",
    "        \"Provide concise financial advice. Respond in exactly three sentences, focusing on potential risks, opportunities, and actionable steps : \"\n",
    "    )\n",
    "\n",
    "    # Tokenize the prompt for GPT-2\n",
    "    input_ids = gpt2_tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
    "\n",
    "    # Move input_ids to the correct device\n",
    "    input_ids = input_ids.to(device)\n",
    "\n",
    "    # Create the attention mask and move it to the correct device\n",
    "    attention_mask = torch.ones_like(input_ids).to(device)\n",
    "\n",
    "    # Generate text using GPT-2 with controlled decoding parameters\n",
    "    gpt2_output = gpt2_model.generate(\n",
    "        input_ids,\n",
    "        max_length=200,  # Limit the total number of tokens\n",
    "        do_sample=True,\n",
    "        temperature=0.3,  # Lower temperature for coherence\n",
    "        top_p=0.85,  # Tighten nucleus sampling for relevance\n",
    "        top_k=40,  # Keep diversity but reduce randomness\n",
    "        attention_mask=attention_mask,  # Ensure attention mask is also on the right device\n",
    "        pad_token_id=gpt2_tokenizer.eos_token_id,  # Ensure proper padding\n",
    "        num_return_sequences=1,  # Only return one response\n",
    "        repetition_penalty=2.0,  # Penalize repetitive outputs\n",
    "        early_stopping=True  # Stop early if the response seems complete\n",
    "    )\n",
    "\n",
    "    # Decode the generated text\n",
    "    advice_with_reasoning = gpt2_tokenizer.decode(gpt2_output[0], skip_special_tokens=True)\n",
    "\n",
    "    return advice_with_reasoning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "8be31dad-b02c-44d3-bd72-21e1341d81f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_financial_advice_with_sentiment(user_question):\n",
    "    \n",
    "    # Step 1: Analyze sentiment using FinBERT\n",
    "    sentiment = analyze_sentiment_with_finbert(user_question)\n",
    "\n",
    "    # Step 2: Generate advice with GPT-2 based on the sentiment\n",
    "    advice_with_reasoning = generate_advice_with_gpt2(user_question, sentiment)\n",
    "\n",
    "    # Return both sentiment and advice\n",
    "    return {\n",
    "        \"user_question\": user_question,\n",
    "        \"sentiment\": sentiment,\n",
    "        \"advice_with_reasoning\": advice_with_reasoning\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "dce34fb2-853c-4b4e-8667-53e270f51188",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_question = \"Should I invest in real esatet in california now?, i am quite sure what to do, i have some money that is sufficient enough i think.\"\n",
    "financial_advice = get_financial_advice_with_sentiment(user_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "8f2c9381-a38f-4ba6-82cc-5b21a35f2420",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_95/970458050.py:1: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import display, HTML\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<!DOCTYPE html>\n",
       "<html>\n",
       "<head>\n",
       "    <style>\n",
       "        body {\n",
       "            font-family: Arial, sans-serif;\n",
       "            line-height: 1.6;\n",
       "        }\n",
       "        h3 {\n",
       "            color: #333;\n",
       "        }\n",
       "        p {\n",
       "            margin: 0;\n",
       "            padding: 0 0 10px 0;\n",
       "        }\n",
       "    </style>\n",
       "</head>\n",
       "<body>\n",
       "    <h3>User Question</h3>\n",
       "    <p>Should I invest in real esatet in california now?, i am quite sure what to do, i have some money that is sufficient enough i think.</p>\n",
       "\n",
       "    <h3>Sentiment</h3>\n",
       "    <p>Neutral</p>\n",
       "\n",
       "    <h3>Advice with Reasoning</h3>\n",
       "    <p>You are a professional financial analyst.\n",
       " Based on the 'Should I invest in real esatet in california now?, i am quite sure what to do, i have some money that is sufficient enough i think.' and the market sentiment being 'neutral'\n",
       " Provide concise financial advice. Respond in exactly three sentences, focusing on potential risks, opportunities, and actionable steps :  \"I would like you not only answer my questions but also provide me with your personal information.\" \"What should we buy? What will be our net income for this year?\" or something similar. This way it's easier if someone asks more detailed question about specific asset classes (e-commerce), stocks etc.. You can use any of these methods as long they're relevant: 1.) Ask them directly 2,) Use their own words 3.), Offer an alternative investment strategy 4.) Make reference from previous experience 5.). Be polite 6). Give feedback 7.): Send email 8 ). Post comments 9 ): Tweet 10</p>\n",
       "</body>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "\n",
    "# Format the output in HTML\n",
    "html_output = f\"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html>\n",
    "<head>\n",
    "    <style>\n",
    "        body {{\n",
    "            font-family: Arial, sans-serif;\n",
    "            line-height: 1.6;\n",
    "        }}\n",
    "        h3 {{\n",
    "            color: #333;\n",
    "        }}\n",
    "        p {{\n",
    "            margin: 0;\n",
    "            padding: 0 0 10px 0;\n",
    "        }}\n",
    "    </style>\n",
    "</head>\n",
    "<body>\n",
    "    <h3>User Question</h3>\n",
    "    <p>{financial_advice['user_question']}</p>\n",
    "\n",
    "    <h3>Sentiment</h3>\n",
    "    <p>{financial_advice['sentiment'].capitalize()}</p>\n",
    "\n",
    "    <h3>Advice with Reasoning</h3>\n",
    "    <p>{financial_advice['advice_with_reasoning']}</p>\n",
    "</body>\n",
    "</html>\n",
    "\"\"\"\n",
    "\n",
    "# Display the HTML formatted output\n",
    "display(HTML(html_output))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "680d3307-091c-47d2-b817-521eec941694",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sagemaker-distribution:Python",
   "language": "python",
   "name": "conda-env-sagemaker-distribution-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}