EmotionStack_EMO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00a958cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from utils import utils\n",
    "import pandas as pd\n",
    "import torch\n",
    "from datetime import datetime\n",
    "from collections import Counter\n",
    "from sklearn.metrics import f1_score, recall_score, precision_score\n",
    "import numpy as np\n",
    "from transformers.trainer_utils import set_seed\n",
    "from transformers import RobertaTokenizer\n",
    "from transformers import AutoConfig, AutoModelWithHeads\n",
    "from transformers.adapters.composition import Stack\n",
    "from transformers import TrainingArguments, EvalPrediction, AdapterTrainer\n",
    "\n",
    "\n",
    "dateTimeObj = datetime.now()\n",
    "\n",
    "RANDOM_SEED = 42\n",
    "set_seed(RANDOM_SEED)\n",
    "\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68ca93bf",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2ba95ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" load data \"\"\"\n",
    "\n",
    "train_data, val_data, test_data = utils.load_wassa_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c213ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" get task labels \"\"\"\n",
    "\n",
    "prediction_task = 'emotion'\n",
    "\n",
    "train_labels = list(train_data[prediction_task].values)\n",
    "val_labels = list(val_data[prediction_task].values)\n",
    "test_labels = list(test_data[prediction_task].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "31ce9dbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" count labels and convert labels to ids \"\"\"\n",
    "\n",
    "train_label_counts = Counter(train_labels)\n",
    "val_label_counts = Counter(val_labels)\n",
    "id2label = {id:lab for id, lab in enumerate(train_label_counts.keys())}\n",
    "label2id = {lab:id for id, lab in id2label.items()}\n",
    "\n",
    "\n",
    "train_labels = [label2id[lab] for lab in train_labels]\n",
    "val_labels = [label2id[lab] for lab in val_labels]\n",
    "test_labels = [label2id[lab] for lab in test_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ba0d9cdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" save random baseline predictions \"\"\"\n",
    "\n",
    "random_preds = utils.random_baseline_predictions(len(val_labels), train_label_counts)\n",
    "random_pred_counts = Counter(random_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c3446e4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "      <th>random_preds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sadness</td>\n",
       "      <td>34.78</td>\n",
       "      <td>36.30</td>\n",
       "      <td>42.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>anger</td>\n",
       "      <td>18.76</td>\n",
       "      <td>28.15</td>\n",
       "      <td>18.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neutral</td>\n",
       "      <td>14.78</td>\n",
       "      <td>9.26</td>\n",
       "      <td>12.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fear</td>\n",
       "      <td>10.43</td>\n",
       "      <td>11.48</td>\n",
       "      <td>5.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>surprise</td>\n",
       "      <td>8.82</td>\n",
       "      <td>5.19</td>\n",
       "      <td>7.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>disgust</td>\n",
       "      <td>8.01</td>\n",
       "      <td>4.44</td>\n",
       "      <td>7.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>joy</td>\n",
       "      <td>4.41</td>\n",
       "      <td>5.19</td>\n",
       "      <td>5.93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label  train    dev  random_preds\n",
       "0   sadness  34.78  36.30         42.59\n",
       "3     anger  18.76  28.15         18.52\n",
       "1   neutral  14.78   9.26         12.59\n",
       "2      fear  10.43  11.48          5.19\n",
       "5  surprise   8.82   5.19          7.78\n",
       "4   disgust   8.01   4.44          7.41\n",
       "6       joy   4.41   5.19          5.93"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" make table of label distributions \"\"\"\n",
    "\n",
    "train_total = sum(train_label_counts.values())\n",
    "val_total = sum(val_label_counts.values())\n",
    "rand_pred_total = sum(random_pred_counts.values())\n",
    "\n",
    "label_counts = {'label':[], 'train':[], 'dev':[], 'random_preds':[]}\n",
    "\n",
    "for lab in train_label_counts:\n",
    "    train_prop = round(train_label_counts[lab] / train_total, 4) * 100\n",
    "    val_prop = round(val_label_counts[lab] / val_total, 4) * 100\n",
    "    random_prop = round(random_pred_counts[lab] / rand_pred_total, 4) * 100\n",
    "    \n",
    "    label_counts['label'].append(lab)\n",
    "    label_counts['train'].append(train_prop)\n",
    "    label_counts['dev'].append(val_prop)\n",
    "    label_counts['random_preds'].append(random_prop)\n",
    "    \n",
    "label_counts = pd.DataFrame(label_counts, columns=label_counts.keys()).sort_values(by=['train'], ascending=False)\n",
    "label_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "55d6a743",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Prepare dataset for training: feature encodings \"\"\"\n",
    "\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "\n",
    "train_encodings = tokenizer(list(train_data['essay'].values), truncation=True, padding=True)\n",
    "val_encodings = tokenizer(list(val_data['essay'].values), truncation=True, padding=True)\n",
    "test_encodings = tokenizer(list(test_data['essay'].values), truncation=True, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5dfe14e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" setup torch dataset \"\"\"\n",
    "\n",
    "class EmoDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels, id2label):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "        self.id2label = id2label\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.long)\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = EmoDataset(train_encodings, train_labels, id2label)\n",
    "val_dataset = EmoDataset(val_encodings, val_labels, id2label)\n",
    "test_dataset = EmoDataset(test_encodings, test_labels, id2label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc7e85f0",
   "metadata": {},
   "source": [
    "# Stacking Emotion Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "263aafb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/adapters/models/roberta.py:250: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/adapters/models/roberta.py:228: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModelWithHeads: ['lm_head.bias', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing RobertaModelWithHeads from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModelWithHeads from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaModelWithHeads were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.embeddings.position_ids']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\"\"\" init model \"\"\"\n",
    "\n",
    "config = AutoConfig.from_pretrained(\n",
    "    \"roberta-base\",\n",
    "    id2label=id2label,\n",
    ")\n",
    "model = AutoModelWithHeads.from_pretrained(\n",
    "    \"roberta-base\",\n",
    "    config=config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "da9aa98c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" load emotion adapter, using: https://huggingface.co/AdapterHub/roberta-base-pf-emotion \"\"\"\n",
    "\n",
    "emotion_adapter = model.load_adapter('AdapterHub/roberta-base-pf-emotion', source=\"hf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "966bb7fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" add adapter for emotion prediction task \"\"\"\n",
    "\n",
    "adapter_name = \"EMO_emotion_stack\"\n",
    "\n",
    "model.add_adapter(adapter_name)\n",
    "model.add_classification_head(adapter_name, num_labels=len(id2label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c663e3fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" activate adapter stack \"\"\"\n",
    "\n",
    "model.active_adapters = Stack(emotion_adapter, adapter_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "252d330e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" activate adapter for training \"\"\"\n",
    "\n",
    "model.train_adapter([adapter_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a4a4f2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "\"\"\" training arguments \"\"\"\n",
    "\n",
    "approach = 'emotion-stack'\n",
    "training_output_dir = f\"./training_output/{approach}/{prediction_task}_{dateTimeObj.hour}{dateTimeObj.minute}-{dateTimeObj.day}-{dateTimeObj.month}\"\n",
    "num_train_epochs=20\n",
    "per_device_train_batch_size=32\n",
    "per_device_eval_batch_size=32\n",
    "metric_for_best_model='eval_f1_macro'\n",
    "warmup_steps=1000\n",
    "weight_decay=0.1\n",
    "learning_rate=1e-04\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    seed=RANDOM_SEED,\n",
    "    output_dir=training_output_dir,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    per_device_eval_batch_size=per_device_eval_batch_size,\n",
    "    learning_rate=learning_rate,\n",
    "     warmup_steps=warmup_steps,\n",
    "     weight_decay=weight_decay,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=50,\n",
    "    eval_steps=50,\n",
    "    save_steps=50, # NOTE: saving every 50 steps uses about 527M space\n",
    "    evaluation_strategy='steps',\n",
    "    disable_tqdm=False,\n",
    "    overwrite_output_dir=True,\n",
    "    remove_unused_columns=False,\n",
    "    save_strategy='steps',\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=metric_for_best_model,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f5f3a9c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" setup trainer \"\"\"\n",
    "\n",
    "def compute_metrics(p: EvalPrediction):\n",
    "    preds = np.argmax(p.predictions, axis=1) \n",
    "    metrics = {\"accuracy\": (preds == p.label_ids).mean()}\n",
    "    \n",
    "    for avg in ['weighted', \"macro\", \"micro\"]:\n",
    "        metrics[f\"f1_{avg}\"] = f1_score(p.label_ids, preds, average=avg)\n",
    "        \n",
    "    for id, lab in id2label.items():\n",
    "        metrics[f\"f1_{id2label[id]}\"] = f1_score(p.label_ids, preds, average='micro', labels=[id])\n",
    "        \n",
    "    return metrics\n",
    "\n",
    "\n",
    "trainer = AdapterTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "41e212f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 1860\n",
      "  Num Epochs = 20\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 1180\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1180' max='1180' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1180/1180 01:53, Epoch 20/20]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Weighted</th>\n",
       "      <th>F1 Macro</th>\n",
       "      <th>F1 Micro</th>\n",
       "      <th>F1 Sadness</th>\n",
       "      <th>F1 Neutral</th>\n",
       "      <th>F1 Fear</th>\n",
       "      <th>F1 Anger</th>\n",
       "      <th>F1 Disgust</th>\n",
       "      <th>F1 Surprise</th>\n",
       "      <th>F1 Joy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>2.008300</td>\n",
       "      <td>2.033924</td>\n",
       "      <td>0.062963</td>\n",
       "      <td>0.019977</td>\n",
       "      <td>0.037317</td>\n",
       "      <td>0.062963</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.157895</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.103321</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.956300</td>\n",
       "      <td>1.958373</td>\n",
       "      <td>0.322222</td>\n",
       "      <td>0.202139</td>\n",
       "      <td>0.097951</td>\n",
       "      <td>0.322222</td>\n",
       "      <td>0.512821</td>\n",
       "      <td>0.172840</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>1.885600</td>\n",
       "      <td>1.854987</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.193317</td>\n",
       "      <td>0.076087</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>1.822700</td>\n",
       "      <td>1.751091</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.193317</td>\n",
       "      <td>0.076087</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>1.743800</td>\n",
       "      <td>1.673881</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.193317</td>\n",
       "      <td>0.076087</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>1.754800</td>\n",
       "      <td>1.645303</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.193317</td>\n",
       "      <td>0.076087</td>\n",
       "      <td>0.362963</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>1.685600</td>\n",
       "      <td>1.474257</td>\n",
       "      <td>0.440741</td>\n",
       "      <td>0.323935</td>\n",
       "      <td>0.139617</td>\n",
       "      <td>0.440741</td>\n",
       "      <td>0.599369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.377953</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>1.459400</td>\n",
       "      <td>1.189146</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.504262</td>\n",
       "      <td>0.281949</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.721992</td>\n",
       "      <td>0.392157</td>\n",
       "      <td>0.216216</td>\n",
       "      <td>0.643275</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>1.240600</td>\n",
       "      <td>1.057548</td>\n",
       "      <td>0.648148</td>\n",
       "      <td>0.623490</td>\n",
       "      <td>0.480300</td>\n",
       "      <td>0.648148</td>\n",
       "      <td>0.754545</td>\n",
       "      <td>0.483871</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.723684</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>1.114700</td>\n",
       "      <td>0.983696</td>\n",
       "      <td>0.677778</td>\n",
       "      <td>0.657368</td>\n",
       "      <td>0.541851</td>\n",
       "      <td>0.677778</td>\n",
       "      <td>0.758929</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>0.133333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>1.086200</td>\n",
       "      <td>0.907723</td>\n",
       "      <td>0.703704</td>\n",
       "      <td>0.691775</td>\n",
       "      <td>0.590842</td>\n",
       "      <td>0.703704</td>\n",
       "      <td>0.794118</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.721893</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.421053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.978500</td>\n",
       "      <td>0.950327</td>\n",
       "      <td>0.655556</td>\n",
       "      <td>0.658327</td>\n",
       "      <td>0.574522</td>\n",
       "      <td>0.655556</td>\n",
       "      <td>0.773869</td>\n",
       "      <td>0.521739</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.648649</td>\n",
       "      <td>0.424242</td>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>0.983700</td>\n",
       "      <td>0.897566</td>\n",
       "      <td>0.696296</td>\n",
       "      <td>0.688617</td>\n",
       "      <td>0.602225</td>\n",
       "      <td>0.696296</td>\n",
       "      <td>0.772093</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.902400</td>\n",
       "      <td>0.898667</td>\n",
       "      <td>0.662963</td>\n",
       "      <td>0.663554</td>\n",
       "      <td>0.588288</td>\n",
       "      <td>0.662963</td>\n",
       "      <td>0.757282</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.677966</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.608696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>0.887200</td>\n",
       "      <td>0.936172</td>\n",
       "      <td>0.681481</td>\n",
       "      <td>0.676022</td>\n",
       "      <td>0.577018</td>\n",
       "      <td>0.681481</td>\n",
       "      <td>0.788732</td>\n",
       "      <td>0.439024</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.704225</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.465116</td>\n",
       "      <td>0.608696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.822200</td>\n",
       "      <td>0.897305</td>\n",
       "      <td>0.670370</td>\n",
       "      <td>0.671320</td>\n",
       "      <td>0.591155</td>\n",
       "      <td>0.670370</td>\n",
       "      <td>0.778325</td>\n",
       "      <td>0.491228</td>\n",
       "      <td>0.711864</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>0.813800</td>\n",
       "      <td>0.880630</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.685383</td>\n",
       "      <td>0.607083</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.774869</td>\n",
       "      <td>0.509804</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.695652</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>0.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.774600</td>\n",
       "      <td>0.927151</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.684239</td>\n",
       "      <td>0.602138</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.536585</td>\n",
       "      <td>0.676056</td>\n",
       "      <td>0.704403</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>0.762300</td>\n",
       "      <td>0.934565</td>\n",
       "      <td>0.674074</td>\n",
       "      <td>0.664400</td>\n",
       "      <td>0.562855</td>\n",
       "      <td>0.674074</td>\n",
       "      <td>0.780488</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.697674</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.516129</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.704700</td>\n",
       "      <td>0.948642</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.681577</td>\n",
       "      <td>0.600302</td>\n",
       "      <td>0.688889</td>\n",
       "      <td>0.787037</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.679245</td>\n",
       "      <td>0.684932</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.645161</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>0.607500</td>\n",
       "      <td>1.023208</td>\n",
       "      <td>0.655556</td>\n",
       "      <td>0.658636</td>\n",
       "      <td>0.571168</td>\n",
       "      <td>0.655556</td>\n",
       "      <td>0.773196</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.626866</td>\n",
       "      <td>0.671329</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.633700</td>\n",
       "      <td>0.995790</td>\n",
       "      <td>0.681481</td>\n",
       "      <td>0.684973</td>\n",
       "      <td>0.600565</td>\n",
       "      <td>0.681481</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.677966</td>\n",
       "      <td>0.688742</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.608696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>0.544500</td>\n",
       "      <td>0.963600</td>\n",
       "      <td>0.685185</td>\n",
       "      <td>0.680790</td>\n",
       "      <td>0.592835</td>\n",
       "      <td>0.685185</td>\n",
       "      <td>0.774510</td>\n",
       "      <td>0.490566</td>\n",
       "      <td>0.677419</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.620690</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-50/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-200/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-250/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-300/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-350/EMO_emotion_stack/pytorch_model_head.bin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-400/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-450/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-500/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-550/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-600/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-650/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-700/EMO_emotion_stack/pytorch_model_head.bin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-750/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-800/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-900/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-950/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1000/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1050/EMO_emotion_stack/pytorch_model_head.bin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1100/EMO_emotion_stack/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n",
      "Saving model checkpoint to ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/emotion/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-1150/EMO_emotion_stack/pytorch_model_head.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850 (score: 0.6070825546538183).\n",
      "Could not locate the best model at ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/pytorch_model.bin, if you are running a distributed training on multiple nodes, you should activate `--save_on_each_node`.\n",
      "Loading best adapter(s) from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850 (score: 0.6070825546538183).\n",
      "Loading module configuration from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/adapter_config.json\n",
      "Overwriting existing adapter 'emotion'.\n",
      "Loading module weights from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_adapter.bin\n",
      "Loading module configuration from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/head_config.json\n",
      "Overwriting existing head 'emotion'\n",
      "Adding head 'emotion' with config {'head_type': 'classification', 'num_labels': 6, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'anger': 3, 'fear': 4, 'joy': 1, 'love': 2, 'sadness': 0, 'surprise': 5}, 'use_pooler': False, 'bias': True}.\n",
      "Loading module weights from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/emotion/pytorch_model_head.bin\n",
      "Loading module configuration from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/adapter_config.json\n",
      "Overwriting existing adapter 'EMO_emotion_stack'.\n",
      "Loading module weights from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Loading module configuration from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/head_config.json\n",
      "Overwriting existing head 'EMO_emotion_stack'\n",
      "Adding head 'EMO_emotion_stack' with config {'head_type': 'classification', 'num_labels': 7, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'LABEL_0': 0, 'LABEL_1': 1, 'LABEL_2': 2, 'LABEL_3': 3, 'LABEL_4': 4, 'LABEL_5': 5, 'LABEL_6': 6}, 'use_pooler': False, 'bias': True}.\n",
      "Loading module weights from ./training_output/emotion-stack/emotion_1638-20-4/checkpoint-850/EMO_emotion_stack/pytorch_model_head.bin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1180, training_loss=1.1659034842151708, metrics={'train_runtime': 114.1296, 'train_samples_per_second': 325.945, 'train_steps_per_second': 10.339, 'total_flos': 4371713319448800.0, 'train_loss': 1.1659034842151708, 'epoch': 20.0})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" train \"\"\"\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c960fc71",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ede8e3a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='35' max='9' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [9/9 00:02]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eval_loss</td>\n",
       "      <td>0.880630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>eval_accuracy</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eval_f1_weighted</td>\n",
       "      <td>0.685383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>eval_f1_macro</td>\n",
       "      <td>0.607083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>eval_f1_micro</td>\n",
       "      <td>0.688889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>eval_f1_sadness</td>\n",
       "      <td>0.774869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>eval_f1_neutral</td>\n",
       "      <td>0.509804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>eval_f1_fear</td>\n",
       "      <td>0.727273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>eval_f1_anger</td>\n",
       "      <td>0.695652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>eval_f1_disgust</td>\n",
       "      <td>0.315789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>eval_f1_surprise</td>\n",
       "      <td>0.642857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>eval_f1_joy</td>\n",
       "      <td>0.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>eval_runtime</td>\n",
       "      <td>0.272600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>eval_samples_per_second</td>\n",
       "      <td>990.475000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>eval_steps_per_second</td>\n",
       "      <td>33.016000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>epoch</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     metric       value\n",
       "0                 eval_loss    0.880630\n",
       "1             eval_accuracy    0.688889\n",
       "2          eval_f1_weighted    0.685383\n",
       "3             eval_f1_macro    0.607083\n",
       "4             eval_f1_micro    0.688889\n",
       "5           eval_f1_sadness    0.774869\n",
       "6           eval_f1_neutral    0.509804\n",
       "7              eval_f1_fear    0.727273\n",
       "8             eval_f1_anger    0.695652\n",
       "9           eval_f1_disgust    0.315789\n",
       "10         eval_f1_surprise    0.642857\n",
       "11              eval_f1_joy    0.583333\n",
       "12             eval_runtime    0.272600\n",
       "13  eval_samples_per_second  990.475000\n",
       "14    eval_steps_per_second   33.016000\n",
       "15                    epoch   20.000000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" output eval metrics from best model \"\"\"\n",
    "\n",
    "trainer.model.cuda()\n",
    "eval_output = trainer.evaluate()\n",
    "eval_result = eval_output[metric_for_best_model]\n",
    "\n",
    "pd.DataFrame({'metric':list(eval_output.keys()), 'value': list(eval_output.values())}, columns=['metric', 'value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "48f4b37e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 270\n",
      "  Batch size = 32\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAG3CAYAAACnsh/VAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABmFklEQVR4nO3deZgU1fX/8fcBAdmRRQ2gQYjiioCIgqIoKvxMNKjEBb8xsqgBo6JRQxQUQYS4YOKSICSCEEmUoBDURJRdjSDKiIgLi4hCFEcMm2zC+f1R1WNPT88G3V1D9+f1PP309K1bt071eubeW1Xm7oiIiIhkg0pRByAiIiKSKkpsREREJGsosREREZGsocRGREREsoYSGxEREckaB0QdgJTMzHL2sLU2bdpEHYJEJNeP1qxcuXLUIUgEVq9eTX5+vmVqeyn+fXnZ3bulsL29psRGKqzXXnst6hAkIjt27Ig6hEjVrVs36hAkAu3bt486hH3RMOoAYpTYiIiI5Ciz1HQQVaReViU2IiIiOSobExtNHhYREZGsoR4bERGRHJWqHpuKRImNiIhIDjIzKlVKzcDN7t27U9JOKmgoSkRERLKGemxERERylIaiREREJGtkY2KjoSgRERHJGuqxERERyVHZ2GOjxEZERCRHZWNio6EoERERyRrqsREREclBZpaVPTZKbERERHJUqk7QV5EosREREZG0M7PxwOnAloRFNYAjgZPc/Z2w7sXAIII8pSowERjp7qWe4liJjYiISI6KYCiqr7vPSYjh10DvuKSmO/AM0M3dZ5rZYcCbQEPg5tI2kH19UCIiIlImsXk2+3oroyeAD5KUXwuMDuMx4GFgmrvPBHD3z4AHgBvMrEVpG1FiIyIiImnn7v9x9y/jy8zsLKApMCEsOhloBsxKWH0mUBm4pLTtaChKREQkB1WQo6KuAya5+8bwcevw/pOEeivD+xNLa1CJjYiISI5KYWLT0MwWxT0e4+5jStl2I+AioEN8O+H9pvi67v6tme2OW14sJTYiIiKyr/LdvV051+kF5MUmDaeKEhsREZEcFdVQVDhJ+Frg3oRF+eF97YT6NQjm2ORTCk0eljI7+uij2blzJ7Nnz446FJF9tnbtWi6++GLq168fdSgVxueff07Xrl2pXLly1KFkXK7ue6VKlVJy2wtdgPoEh3XHywvvmyeUxx6/W+o+7U00sv8bN24cy5cvZ/HixYVuH330Ee5OmzZtiqzz8MMPU6VKlQiizawZM2Zw/vnnc/rpp3PCCSfQrl07hg8fHnVYGZEr+z558mS6devGZ599VmK96dOn07lzZ04//XROOeUUHnroIXbvLvX8YPulp59+mtNPP51PP/006lAyLpf3PULXAePdfVtC+VvAp8BZCeVdgD3Ac6U1nLWJjZk1NrM8M9tiZnOijqci6tu3L23atCl0e+KJJ3j//fdZvHhxobrnn38+hxxyCGvWrIko2swYN24cd9xxB48//jivvfYaS5Ys4aKLLuL555+POrS0y5V937RpE+PHj2f69Om0b9++2Hovvvgiffr04Z577uG1115jypQpPPnkkwwePDiD0WbGxo0bGTt2LLNmzaJDhw6lr5BFcnnfU3UOm/IOZ5nZIcCFhOeuiefuTnASvu7hoeCYWVPgNuARd19RWvtZO8fG3dcBrZXUJPfEE0+watWqIuXXXnstjz32WKGyAw44gIceeohf/vKXjB8/PkMRZt7atWu57bbbeOGFFzjiiCOA4IN/0003ceqpp0YcXXrl0r7XqlWL6dOnl9h97u7ceeed/L//9/8488wzAWjatCm/+tWvuOuuu7jmmmsKnqdsULt2bWbNmpWV1w0qTS7vO0Q2x6Y3MN/dP0620N2fN7PLgFFmVpngkgqjgRFlaTw3X0nhzTffZP369YXKOnfuTNOmTZkwYUKh8uuvv55ly5Yxd+7cTIaYcZMmTaJmzZpFfshr1apFly5dIooqM3Jp38syJ+Cdd95hzZo1nHHGGYXKzzzzTHbv3s306dPTGWLG7cM8if1eLu97VNx9hLufU0qd59y9jbu3cvej3f3eslwnCrK4x0bK77rrrmPSpEls2vT96QMaNGjAwIEDOe200yKMLDPeeOMNmjVrxvTp03nsscf46quvqF69Ot27d2fAgAFZPb8ol/c9mffeew+Aww8/vFB5s2bNAFi6dGmmQxJJiwpwgr6UqxCJjZkdSXAdiGaAA98BLwL3E0wWuhs4L6xehWBW9O3u/t+EdroADwGNCCYf/SnJtiYAnYHDgLOBAUBLwMI2pyXUPxb4HXBCGMtKYKC7vx1X52SCLrIGYdG3BNe5uD9cXitcfiawm6Cn7DXgIXcvOh4UgYYNG3LRRRcVGWMeOnQoEyZMSDpslW0+++wz1qxZw+9+9zv+/ve/07RpU+bNm8fll1/O4sWLmTRpUtQhpk0u73syGzZsAIJhing1atSgcuXKfP3111GEJZJy2ZjYVJT+txeBxe7e2t3bADcCA4GDgXoE43EXu/tJQFtgAzDdzAriN7OjgZcIrifR1N07AscAx8VvyN2vAu4KH/4K6OnuRwPTgL+aWb24NlsArwPrgebu3jx8PNfMjgrr1AL+Dfw17DZrQ5Bc/S5us6MIkqeTwuVdgXOBwv3cEerVqxd5eXmFJg0fd9xx/PSnP+XeexNPM5CdduzYwdatWxkxYgRNmzYF4IwzzuDaa69l2rRpLFq0qJQW9l+5vO8iuSyKycPpFnliY2YNgSOBgpnO7v4f4E6CUyrnAx3d/dNw2XcEPTEnAfFnObyToKfn7nBWNcAwoFoJm/+ru28N/34WqEVwAa6YIUAN4DZ33xOW3QfsIEi8AI4mOBZ/eVz8z4X1YjoCn7r7rnD5FwQzvJclC8rMrjWzRQmnp06ra6+9lj/9qXAH18MPP8w999zD5s2bMxVGpGrVqgVA69atC5W3bdsWIKt/3HN535OJndtmy5Ythcq//fZbdu/eTYMGDZKtJiIVQEUYivoaeA94wszaAH9z97fd/aFYBTP7kZk9ChxOkLxUDRe1ABaGf3cEPnT3gm8id99mZrELZyUTPyN7Q3h/aFzZuWGbsWW4+04zW873x9h/CPwXmGpmjwPPuvsyd78zrp3ZwPVmVhsYD8xKHPKKF15fY0y4715cvVTp0qUL9evX55lnvj9PUu3atTnhhBM45JBD6N+/f0F548aNadiwYUHPTvv27dm1a1e6Q8yIo48+miVLlrBnz55C5QccEHxMEsuzSS7vezInnHACQJHzmqxevRqA448/PtMhiaScmWXlxOnI9yjsXTkTeAzoCSwys5Vm1gvAzLoB/wLmAq3cvTVwfrh6fG9MY+B/STaxMUlZzNa4v2Pf3PGnnWwAHBGeD6fgBvyA8LkLE6lTCHp8bgTeN7MlZnZBXDsDgBuAY8N9+dLMRppZSb1JGXPdddcxfvx4tm/fXlC2efNmfvCDH3DiiScWOs/NunXrWLRoUcHjbElqAH7yk58A308cjYk9bteuvJdB2X/k8r4n07ZtWw477DDmz59fqHzevHlUqlSp4PkS2d9pKCpN3P0bdx8INCWYf/IF8KSZnQNcBWxx99/FDQclsw44KEl5vX0ILR94P5z7E3/7obv/MC7+z9z9eoLenh4EydHz4bwf3H23uz/u7q0ILrk+DfgNEPmZvg4++GAuvPBCRo8ucp6knHPRRRfRsWNHhgwZwsaNQT78wQcf8Kc//YkLL7ywxJO57e9yed+TMTOGDx/OSy+9VJDcrF27lkcffZRrr72W5s0Tz/YuIhVF5ENRZnYwcKe73xQmLjPCuSVfA60Ihp0Sh2MOpag3gIvNrFZsOMrMqhNcX2Jvrxw6A7jAzA5094LujLA35iR3H2JmJwAXuvtwd98JTDGzT4C3CXpoPjSzvwA3uPu37r4EuNrM2ob7F6nevXszf/58li9fXmK9p556ilatWhUainrllVe4/fbbMxRp+lWqVIkpU6Zwzz330KFDB6pVq8aePXu49tprue2226IOL61ybd9vueUWFi1axOeffw5QcL6a6dOnU7duXSDoxfrLX/7CoEGD2L17N7t27aJXr17cfPPNkcWdTv369WPBggUFZxePza+aNWsW9erVizCy9Mvlfa9ovS2pYN/Ps40oALNmwEfA2e7+elh2ETCZYIjnBGAc0M/dR4fDN5OBC4Be7j4+XOdogsPAHwNudXc3s5EERz4tcvfOcdu8OmzzCHdfHRfHJwltHkGQoEwEbnH33WbWHHglfDzNzDoTXLuig7t/FK43gGDi8tHuvtbMVgOPufuDce2+Awxx9z+U8vxE+wJFaOvWraVXkqy0Y8eOqEOIVCy5ktzSvn17Fi1alLFMo2rVqn7oocn6Ccrvs88+e9vdK8SYdeQ9NsCXBOd4eTQ8fNsIzgPTPTxXzNtm9gNgoJndSDBR918Eic1QMzvR3W929w/N7MfAg8BaM1tLkJAsAtqFc2O6A7eE9wAvmVns6KaR4X18m5+YWcdw2admth7YRpA4xSb/LgX+Akw2s90Ez+l6oJu7rw3r3AP8wsyuIpjLUxkYDjySkmdQREREgArQYyMlU4+N5CL12KjHJhdlusemWrVqKeuxWbNmjXpsREREJFrZOMemQhwVJSIiIpIK6rERERHJUdl4gj4lNiIiIjlKQ1EiIiIiFZh6bERERHJQRbwcQioosREREclR2ZjYaChKREREsoZ6bERERHJUNvbYKLERERHJUdl4uHf27ZGIiIjkLPXYiIiI5CAdFSUiIiJZJRsTGw1FiYiISNZQj42IiEiOysbJw0psREREcpSGokREREQqMPXYiIiI5CAz01CUiIiIZA8NRYmIiIhUYOqxERERyVHZ2GOjxEZERCQHZescm+zbIxEREclZ6rGp4Nq2bcubb74ZdRiR+Prrr6MOIVKNGjWKOoTIZON/keWRy/u/Z8+eqEPIKRqKEhERkayRjUl09u2RiIiI5Cz12IiIiOQoDUWJiIhIVtBRUSIiIiL7wMy6mdlMM1tkZivM7H0zG5JQ5wwze8PMlprZR2b2kJlVL+s2lNiIiIjkKDNLya2M27oGeBC4xt3bAUcCk4EecXXaAzOAJ9z9eOAU4BxgUln3SUNRIiIiOSpTc2zMrCnwB+Acd18F4O5uZg8Cr8dVfQBY6u5PhXX+Z2Z3A8+bWWd3n1PattRjIyIiIul2FbDF3d+IL3T3Le7+CoCZHQp0AmYlrDsLcODSsmxIPTYiIiI5KMWThxua2aK4x2PcfUzc49OBT8ysO3AzcDDwLfAP4EF33wW0Agz4JL5hd99kZvnAiWUJRImNiIhIjkrhUFR+OG+mOIcDzYBBwEXu/pmZdQamAu2AS4CGYd1NSdbfFLe8RBqKEhERkXQ7EKgJ/NrdPwMI58s8DlwcThpOCfXYiIiI5KgMnsdmc3j/TkJ5bPiqPfBx+HftJOvXAZaXZUPqsREREclRGTzce1l4n5h3fBdXvoRgknDzhBhrEwxDvVuWDSmxERERkXSbFt4nTgCOPV7o7l8ArwFnJdQ5m2BS8eSybEiJjYiISA5KVW9NGXts/kGQtNxnZnXD7R8L3AA87+5vhvVuBVqZ2c/DOvWAe4Cp7j67LBvSHBsREZEclak5Nu6+x8x+DAwH8sxsB0Hnyh+B++LqLTSzc4GRZvYboCrwAnBnWbelxEZERETSzt03EfTQ3FBKvXlAx73djhIbERGRHJWpSypkkhIbERGRHJTiMw9XGNm3RyIiIpKz1GMjIiKSozQUJSIiIllDiY2IiIhkDc2xEREREanAlNhITlu3bh2XXXYZP/jBD6IORUQi9Pnnn9O1a1cqV64cdSgZk+EzD2eMhqKkwMqVKxk7dixz5szB3dmyZQt169bl+uuv58orr4w6vJSbMmUK9913HwceeGDS5bt372bcuHG88sorbNiwge+++46dO3dyzjnncPPNN1OvXr3MBpxGH3zwAWPHjuXVV1/FzNi9ezfHHnssd911F8cff3zU4aXdp59+ykknnUSLFi2KLJs4cSJHHXVUBFFlzrZt2xg8eDDTp0+nSpUq1KlThxEjRnDmmWdGHVpGPP3009x5553Ffhdks2wcilJik0FmNhLoAbQAznL3OdFGVNjLL7/M3/72N1555ZWCL/JHH32UXr16YWb07Nkz4ghTZ9OmTUycOJHnnnuOUaNGsWrVqiJ1tm/fzuDBgxk+fHjBc7By5Uq6d+/Ohx9+yDPPPBNB5OnRp08fqlWrxuzZs2nQoAFbt27l5z//OR06dGD+/Pm0bt066hDTrm3btsyYMSPqMCLRs2dPVq9ezYIFC6hXrx5PPvkkXbt2Zd68ebRv3z7q8NJq48aNjB07llmzZjFs2DCWL18edUiyj7IvVdtHZtbZzIako213Hwj0TUfbqdC4cWMGDRpU6L/TG264gbp16/L8889HGFnq1apVi+eee44f/vCHxdapVKkS3bp1o3fv3gVdrS1atOCnP/0p8+fPZ9OmTZkKNyOGDh1KgwYNAKhZsyb3338/27ZtY/To0RFHJuk0e/Zspk6dyrBhwwp6IXv37s3RRx/N7bffHm1wGVC7dm1mzZpF8+bNow4lEhqKyg2dgbuBIdGGkXndu3cvUrZz5062b99Oo0aNMh9QGpWl+7V69eqMGzeuSPnWrVupXr06NWrUSEdokZgzZw5Vq1YtVNakSRMAvvnmmyhCkgyZPHkyZkbnzp0LlXfp0oWHH36YL7/8kkMOOSSa4DIgG4diykpnHpack5+fT//+/WnatCkDBw6MOpzI7d69m2nTpjF9+nSGDx/OAQdkz/8FiUkNwEcffQRQ5AcvW61fv57evXvTqVMnjj/+eK644goWLFgQdVhpl5eXR6NGjahVq1ah8hYtWuDuLFmyJKLIRPZOhUxszKyHmeWZmZvZMDMbbmYLzWytmT1uZlXj6lYzs5FmtsrMPjSzZWZ2Y9zyc+LaGhJXnmdmW8xsTlzZ08Av45bnmdkjZtYsvr6ZXWFm88zsi7DdemZ2hJn9xczeNbPF4f1QM6uWkScthdatW8dxxx1HkyZNWLFiBVOmTOHwww+POqxI3XLLLRx11FH89re/ZdiwYVx++eVRh5R2o0eP5thjj6VXr15Rh5J2lSpV4oADDqB///7Mnz+fBQsW8IMf/IBzzjmHf//731GHl1b5+fnUqVOnSHmsLD8/P9MhSQZl41BUhUxs3P0f7t46fHgV8E93bw90A64Ly2ImA1cAnd39aKAnMMzM7gjbejWurfhttAYWJZRdCYyOLQ9vN7r76rj6xwJHufsZwJHA1+HqpwDNgVPdvQ1wJtAFGFne/Teza81skZktiuJLpXHjxrz//vt88803dOvWjVNPPZVnn30243FUJKNGjWL58uX85S9/4cEHH6Rv377s2bMn6rDS5oUXXmDq1Kk888wzOXGkyGGHHcaiRYto164dEMwxeuCBB2jatGlOzDOR3KXEJhp57r4AwN3fAz4EzgIwsy7ABcAD7r4mrJMHTAR+a2Y10xBPFcJkxd03EyQ0m4CXgcvcfVu47H9hHNdaOV91dx/j7u3cvV3Dhg1TGXu51KhRg4EDB3LaaafRv39/tm/fHlksFUGlSpXo0KEDd9xxBy+++CLTpk2LOqS0mD9/Pv3792f69Okcc8wxUYcTmcqVK9O2bVuWL1/O119/XfoK+6mGDRuyefPmIuWxyfFRfgeJ7I39IbH5OOHxBuDQ8O9zw/vXE+osBWoB7dIQz0p33xF74O4r3X0PQXLTw8zmm9lSM8sD7gBqxMVboW3btg13L1J+4oknsmnTppw7DDJ23ppEsfO6vPvuu5kOKe1mz57NL37xC6ZNm8bJJ58cdTgZs3HjxqSvdexkbbt37850SBnTunVr1q9fz9atWwuVr1q1CjOjVatWEUUmmaAem2hsTXi8B4idGrJBeD8xbk5MHjAA+BI4KA3xbCmmfBgwChjs7seHQ1d3hcv2i3k2F1xwQdLJkqtXrwYoOBQ4V0yZMoU777yzSPlnn30GQP369TMdUlrNmDGDvn37Mm3aNNq0aVNQnuxouWxz66238txzzxUqc3fy8vJo0qQJBx98cESRpV+PHj1wd+bMmVOofObMmXTq1Cmrj4jKdalKaipaYrO/H9YRm4Byibt/VErdPUDis1+L4hOV8roaeKWinXSvvIYMGcLEiRMLDu+ePHkyU6dOpUePHjRu3Dji6DLv+eef5+KLL6ZDhw4ArFmzhuHDh3PQQQfxs5/9LOLoUufFF1/ksssuY8CAASxZsqTQkTDvvfdehJFlzsMPP8wZZ5xB48aNcXfuvfdeli9fzpgxY6IOLa3OPvtsunfvzuDBgzn99NOpW7cu48ePZ9myZcyfPz/q8ETKbX9PbGYAA4HWQEFiY2a1gKeAa9x9Q1j8JVA/rk4Vgsm+iccy7gqXx5KgnwL/dvfSJphUJUie4u0XQ1Axw4YNY9y4cZx33nlUrlyZHTt2ULNmTUaNGkXfvhX2vIJ77fbbb+edd95h7dq1AJxzzjlA0FNTt25dunTpwn//+1+GDh3Kzp072bNnD9999x0dOnTghhtuyKrrSw0YMIDt27czcmTRue4lncQwW9x00008+eSTdO/eHTPjq6++4ogjjuCZZ57hwgsvjDq8tJs0aRKDBg3i5JNPpmrVqtSuXZsZM2Zk/VmHY/r168eCBQtYs2YNEJyFGmDWrFlZdemUZCpab0sqWLI5FRWFmTlwj7sPiSubA+DuncPHzwMtgW7uviY8vPpRoIG7XxK33iSCOTcnuftmM7sF+A3wQaytsF5P4GngcOBA4D2gtrvvStx2QqzjgUuB09x9sZk1Jpj70ww4wt1Xh/U6A7OhbJdUOOmkk/zNN98srVpWyuYJm2WRbSdFLI9du3ZFHUKkcuFItOJk89GGpWnfvj2LFi3KWKZRv359P/fcc0uvWAbPPvvs2+6ejnmt5VYh59hYeO6Z8OEvzezPYXkeQXLSLpxPUxu4DHgOmG1m7wMLgP8B/5fQ7O3AJ8CHZjYP+AL4IK6tw8J6zwHTgLnh/Q1AzSTbTrwq5ADgb8CLZraQ4LDxqeGyl8zsSguuFfXnsOzPZjZ8L54eERERKUaFHIpy91cJhpcSy4uUhQaFt5La/BzomlA8KUm97UD3JE0Ut+3Yev8D+iRZdHPCY53CV0REKoRsHIqqkImNiIiIpF82JjYVcihKREREZG+ox0ZERCQHWZZe3VuJjYiISI7SUJSIiIhIBaYeGxERkRyVjT02SmxERERyVDYmNhqKEhERkayhHhsREZEcVBGvzJ0KSmxERERyVDYmNhqKEhERkayhHhsREZEclY09NkpsREREclQ2JjYaihIREZGsoR4bERGRHJWNPTZKbERERHJQpg/3NrNmwFJgRZLFl7n7R3F1LwYGEeQpVYGJwEh3313adpTYiIiISKYscvfOJVUws+7AM0A3d59pZocBbwINgZtL24Dm2IiIiOSoWK/Nvt5SGI8BDwPT3H0mgLt/BjwA3GBmLUprQ4mNiIhIjqpoiQ1wMtAMmJVQPhOoDFxSWgNKbERERCRTDjGziWa2wMyWm9k/zOzUuOWtw/tPEtZbGd6fWNoGlNiIiIjkqBT22DQ0s0Vxt2uTbG438B3wiLufQpDErAPmm9n5YZ2G4f2m+BXd/dtw/YaUQpOHRUREclQKh5Hy3b1dSRXCuTInxD3eamY3Az8hmFfzUioCUY+NiIiIRCI8fHsRcJSZNQDyw0W14+uZWQ2COTb5lEI9NvuBPXv2RB1CJA4++OCoQ4jUW2+9FXUIkWnXrsR//CSLVaqk/7czJYLz2NQFtrn7zoRFsXPTVAbywr+bJ9SJPX63tO3oHSQiIpKjMnxU1B+AHgnbN6At8Lm7rwfeAj4FzkpYtwuwB3iutI0osREREZFMud3MGkNBUjMEOAoYDODuTnASvu5mdlZYrylwG8Gk42RnLS5EQ1EiIiI5KsPXinoIuAb4l5k50AhYBVzk7lNjldz9eTO7DBhlZpUJLqkwGhhRlo0osREREclRmUxs3P094MYy1n2OMgw7JaOhKBEREcka6rERERHJURkeisoIJTYiIiI5KNOHe2eKhqJEREQka6jHRkREJEdlY4+NEhsREZEclY2JjYaiREREJGuox0ZERCRHZWOPjRIbERGRHKXERkRERLKCDvcWERERqeDUYyMiIpKjsrHHRomNiIhIjqpUKfsGbrJvj0RERCRnqcdGREQkR2koSkRERLKCjooSERERqeDUYyMiIpKj1GNTAjM7OVVtiYiISPrFhqP29VaRpHIo6okUtiUVxF133cWBBx7IhAkTog4loz7//HO6du1K5cqVow6lQvjmm28455xz6N69e9ShiIiUqNihKDNbVc62Gu9jLFLBrF69mkceeSTqMDLu6aef5s477+TAAw+MOpS0+OSTT5g6dSoLFy7EzNi9ezfNmzenb9++tGjRIuk6Y8aMYcuWLdSqVSvD0WbGBx98wNixY3n11VcLnpNjjz2Wu+66i+OPPz7q8NJu27ZtDB48mOnTp1OlShXq1KnDiBEjOPPMM6MOLe1yed8h94ai6gJzy3ibB2xJa6RpZmZNzGyqmX1gZh+Z2Wwzqx51XFG644476NKlS9RhZNTGjRsZO3Yss2bNokOHDlGHkxb33nsvH3/8MaNHj2bSpEmMHz+e3bt307t3bz7++OMi9ZcvX87rr7/OMcccE0G0mdGnTx8WL17M7Nmzeffdd1mwYAHfffcdHTp0IC8vL+rw0q5nz57MnDmTBQsWsHTpUvr27UvXrl1ZuHBh1KGlXS7vO+TeUNQad+9VxtvVwGcZijld/gg0AI4Pb42BapFGFKH58+fz3nvv0a9fv6hDyajatWsza9YsmjdvHnUoaXXddddRt25dAKpXr84NN9zAjh07mDJlSpG6f/jDH7j22muztgcrZujQoTRo0ACAmjVrcv/997Nt2zZGjx4dcWTpNXv2bKZOncqwYcOoV68eAL179+boo4/m9ttvjza4NMvlfc9mJSU27cvZVnnrVzRnAq+7+2533wWc6O7/izimSOzZs4fbbruNESNGUKVKlajDyahKlSpl5SnG440ePZrWrVsXKmvUqBEAmzdvLlQ+d+5ctmzZwo9//ONMhReJOXPm0KlTp0JlTZo0AYL5Rdls8uTJmBmdO3cuVN6lSxfmzZvHl19+GU1gGZDL+w6p663Zb3pswh/3IszsdDMbbGYj4x7XLK7+fqQusD32wN23l1A3qz311FPUr1+fn/zkJ1GHImmQLFlds2YNAG3bti0o27VrF48++igDBgyocF9cqVa1atUiZR999BFAkR+9bJOXl0ejRo2KzJ9q0aIF7s6SJUsiiiz9cnnfY3IqsUlkZjXN7N8Ec2ruAa4KF10MvGdmh6chvrQzsx5mlhc+/KWZ5ZnZS+GyY81supmtNrNVZvaKmZ2UsP5ZZvZPM1scrrvIzK5MqDPBzNaYmYf1p5rZe+HjARnZ0TLavHkzw4YN4/777486FMmgKVOmcMQRR3DBBRcUlP3973+nZcuWRXp3csXo0aM59thj6dWrV9ShpFV+fj516tQpUh4ry8/Pz3RIGZPL+57NytPnfh9wMHApcCzwFYC73wLcC4xIeXQZ4O7/cPfW4cPR7t7a3c83sxbA68B6oLm7Nw8fzzWzo+KauAx4H2gbtvNz4A9m9tO4bVwF3BU+vB3o7e4nAGPSuGt7ZeTIkZx//vk5cSSIBF577TXmzp3LfffdR7VqwbSyDRs2MGnSJK6//vqIo4vGCy+8wNSpU3nmmWeyfm6R5LZs7LEpz5mHfwy0i807MbPdsQXu/qSZZds34BCgBnCbu+8Jy+4DbgAGAr3jyr5ydwdw9w/M7FXgOmBaknbHufuG8O/fAkXeEWZ2LXAtwGGHHZaSnSmLVatWMXHiRN5+++2MbVOilZeXx+9+9ztGjRrFEUccUVA+evRoLrzwQho3zr2zOMyfP5/+/fszffr0rD4SLKZhw4asWlX07B6bNm0qWJ6tcnnfYypaUpIK5UlsdpUymTbbTnBxLvBhXBKCu+80s+XAWXH1NgNDzOxsgqOo9gCHE/ZoJbEsrr0NySq4+xjC3pyTTjrJ92UnymP27NlUq1at0ETRLVuCo/iHDh3KY489xnnnnce9996bqZAkjd5++22GDRvGgw8+SMuWLQsty8vLo3LlyrzxxhsFZZ9//jm7du3i5z//OQC/+93vsi7xmT17Nn369GHatGm0adMm6nAyonXr1rz55pts3bqVmjVrFpSvWrUKM6NVq1YRRpdeubzv2aw8ic13ZtbZ3eckLjCzM4BtKYuqYmgA1IibfxNzUOwPC1LdF4CGQFd3Xx2Wjwc6F9NuhT3fT58+fejTp0+hsrlz59K1a1fuuusurrrqqmLWlP3NggULGDFiBA8++CA/+tGPCspvvfVWHnzwQZ599tki6/Tr14///ve/TJw4MZOhZsyMGTPo168f06ZN44QTTigo7969O1OnTo0usDTr0aMHf/rTn5gzZ06hf2pmzpxJp06dOOSQQyKMLr1yed8h6K3JxqNAy5PYjAJeMbPngDeAembWF2hDMJH4l2mIL0r5wGp3L+ksbUcCHYFbY0mNSEX3+uuvc8cdd3D55ZezfPlyli9fXrBsxYoVEUYWnRdffJHLLruMAQMGsGTJkkJHw7z33nsRRpZ+Z599Nt27d2fw4MGcfvrp1K1bl/Hjx7Ns2TLmz58fdXhplcv7HpPTQ1HuPs7MDiI4IupnYfEYgh6IO9396TTEF6UZwAVmdmD8od9mdgFwkrsPAWLHiO5JWPfQzISYPqtXr+bSSy8tMhQ1atQoTj/99IijS69+/fqxYMGCIodAz5o1q+AkXvuzhx56iB07dvDUU08VWXbooUXfuo8//jhvvvlmoaGoI444gqFDh2Yi3IwYMGAA27dvZ+TIkUWW/fCHP4wgosyaNGkSgwYN4uSTT6Zq1arUrl2bGTNm0L79/n56stLl8r5nKwvnvJZ9BbPaBL0UDQh6Nd5w9wo7vFJWZubAPWHCgpkdAbwNTARucffdZtYceCV8PM3MqhDMmdkBnOnuX4fDcrOAz929WVz7VwPjgCPK07tz0kknefw8h1ySaycHTPTWW29FHUJk2rVrF3UIkdLFV3NTu3btWLRoUca6UJo0aeKpOrv84MGD33b3CvHBLffgmrtvdveX3X2Su8/Y35OaYs5jc6G7f0KQwP0Q+NTM3iFIcm5192lQcBLDC4H/Au+b2TzgF8DLQOOwrZZm9ggQ+/f2JTN7JmM7KCIiUoxcP9wbM6sH3Ax0AJoAawnm2/x+f738gLv/A/hHMcs+BLqXsv4HBEdQleTG8CYiIiJpVObExsxaAzMJjgr6H/ANcDJwDvArMzvb3bP//NMiIiJZoqL1tqRCeYaiHgNmA0e6e313b+HuBwEtCS6z8Hg6AhQREZHUS9UwVEVLjsozFHUkcEbcWXgBcPflZnYZwbCUiIiISGTK02OzKjGpiXH374DPUhOSiIiIZEKUPTZmNjy8GPTVqdyn8iQ2z5nZdckWhOX/Tk1IIiIikglRJTZm1gy4pYTlZ5jZG2a21Mw+MrOHzKx6WdoudijKzJ5MUny+md0MLAU2AXWB4wiOkJpSlg2KiIhIznuA4LxwFyQuMLP2BCfJvc7dnwqPyJ4LNAcuKq3hkubYXAmsSyjbRnChx5MSyvOBy4GrS9ugiIiIVAxRTPwNT2TbCvgVSRIbgqRnqbs/BeDu/zOzu4HnrZhrVsYrKbFZ5u5lvrytmS0ua10RERGJXqYTGzOrBPweuA3YlWT5oUAn4MGERbMABy4F5pS0jZLm2PQve6h7VV9ERESyQ0MzWxR3u7aYer2Ar939n8UsbwUY8El8obtvIhgdOrG0QIrtsXH3/5S2coIbgfKuIyIiIhEwMypVKveVlYqTX9q1osJrTQ4FupVQrWF4vynJsk1xy4tV3ksq1ADOB1oQzLWJd2Z52hIREZFoZXgoahAw3d3fS+dGynNJhSOBV4HDCMa5Ep+N8l0mXERERHKCmTUnOMDo+FKq5of3tZMsqwMsL21b5emxuR94lODSCv+JTSwOJ/rcDbxbjrZEREQkYhnssekCbAdeidtmrfB+qJkNIDgf3u8JOkqax68cDmM1pJiLVscrT2LT3N0vCjdQ0Dvj7l+YWX+C3pzR5WhPREREIpSpxMbdxwJjE7bdmeAalHe5+/i48teAsxKaOJtgpGhyadsqz6yhHXF/H2BmBUmRuztweDnaEhEREUnmVqCVmf0cIDxB3z3AVHefXdrK5Uls3MxOCP/+GHjQzOqZWV0zGw58V764RUREJCpRXd3bzJqZWR7w57BoqJnlmVknAHdfCJwL9DOzpcBCgvPY9CxL++UZipoGzDWzUwnm28wFbohb3qccbYmIiEjEojjzsLuvBlqXUmce0HFv2i9zYuPu9wH3xR6H13K4AqgCvFjaKY5FRERE0q1c57GJFx6HXnAsuplZONdGRERE9gNR9Nik214nNkm8DbRNYXsiIiKSRjmV2JjZk+VsS0dFiYiISKRK6rG5ElhXjrZqlV5FREREKoqc6rEBlsXOLlwWZrY4BfGIiIhIBuzNodr7g5ISm/7lbKu89aUM3J1du3ZFHYZE4OSTT446hMhcfPHFUYcQqcmTSz25qmQhHX+TGsUmNu7+n/I0VN76IiIiEq1Klcpznt79QyqPihIREZH9SDYORWVfqiYiIiI5Sz02IiIiOSobe2yU2IiIiOSgXDwqqlhmVhk4yN3zUxyPiIiIZEg2Th4u1x6Z2elm9iqwBXg3LHvczHSot4iIiESuzImNmXUBZgGHAFOBbeGiCcDVZtY75dGJiIhI2sSGo/b1VpGUp8dmKHCtu5/g7lcAmwDcfQHwE+C6NMQnIiIiaZLriU1Ddx+fbIG7rweqpCQiERERkb1UnsnD1cysirsXOb+/mR0INEhdWCIiIpJOFbG3JRXK02PzOjDVzI6JLzSzQwjm2cxOZWAiIiKSXtk4FFWeHpvbCZKbpWa2ETjQzNYSTCZeDZye+vBEREREyq7MiY27rzWzNsAtwDkEQ0/5wFjg9+7+v7REKCIiImmRjeexKdcJ+tz9G2BweBMREZH9VEUcRkqFlKVqZjYsVW2JiIiI7I0y99iY2RmlVLkM9eSIiIjsN7Kxx6Y8Q1FzAE9THCIiIpJhuZ7YrAT6JpTVAo4BfgqMSlVQIiIiInujPInNg+4+N0n5i2Y2AXgIeD41YYmIiEi65fRRUe7+RAnL1ptZq9SEJCIiIumWrUdFletw72TM7CDgZwTDUiIiIiKRKc9RUXsofvKwA79KSUQiIiKSEbneY/MlMDqhbHdYPs/dP0pZVCIiIpJ22ZjYlGfW0Gvufk/C7V53H6ukRrLRXXfdxYEHHsiECROiDiWjPv/8c7p27UrlypWjDkVEpNzK02NzgJk9B9zo7p+nKyCJ1nXXXcebb75JzZo1C5V/++23rFy5kvnz59O6detogsug1atX88gjj0QdRsY9/fTT3HnnnRx44IFRh5I2Bx98MH/4wx/44osviix74IEHWLduXcHjgw46iCuuuIIf/ehHVK5cmRo1arBy5UrGjBnDhg0bMhl22qxcuZKxY8cyZ84c3J0tW7ZQt25drr/+eq688sqow0u7XN//bOyxKU9icz5wFcHQk2Sxxx57jE6dOhUqe+SRR5g4cWJOJDUAd9xxB126dOHFF1+MOpSM2bhxI2PHjmXWrFkMGzaM5cuXRx1S2qxcuZJBgwaVWKdRo0YMHz6cv/71r/zxj38E4Ic//CEjR47koIMOyprE5uWXX+Zvf/sbr7zyCkcddRQAjz76KL169cLM6NmzZ8QRplcu77+ZZeXh3uXZo3fd/Rl335VsoZnVTFYu+5fevXvTsmXLIuXjxo2jT58+EUSUefPnz+e9996jX79+UYeSUbVr12bWrFk0b9486lAqhGuuuYa8vDzmzZtXUPbpp58yfPjwpL09+6vGjRszaNCggh91gBtuuIG6devy/PPZf2qyXN//bFSeHpuFZtbK3ZcUs3w+0DYFMUmETjnllCJl8+bNY926dVxxxRURRJRZe/bs4bbbbmPEiBFUqVIl6nAyKhv/c9tb9evXp127dgwfPrzIsqVLl0YQUfp07969SNnOnTvZvn07jRo1ynxAGZbr+5/rQ1HvA8+Y2avAB8CWhOX1UxaVVChPPvkkP/vZz6hbt27UoaTdU089Rf369fnJT37C3LnJTrQt2aBu3boMGDCAxo0bU6tWLVavXs3UqVP5+OOPATjmmGOoVKkS7s6NN95IixYtOOCAA/joo4949tlns6rHJlF+fj4DBw6kadOmDBw4MOpwMi7X9j8bE5vy/Iv2ONASuB54DBifcDsspZGVk5m1NrO/m9m7ZrY4vN1kZpXC5SPNbIWZuZn1MLOnzWypma0ys+uStNfGzF4zs/Vm9paZDTWzp8xsp5nlmVnHuLq9wrY+Ctt7xMxqxS1/1cy+CLfdxsxeNrOPw8fdM/IE7aWvvvqK6dOn07dv4mXCss/mzZsZNmwY999/f9ShSBrt2bOHPXv28MILL3D77bdz8803s2HDBu677z5OOukkABo2bAjAgAEDWLBgATfddBMDBw6kQYMG3H///Rx88MFR7kJarFu3juOOO44mTZqwYsUKpkyZwuGHHx51WBmT6/ufTcqT2HwAHFHMrTnwYcqjK59uwHfASe7eBvgxwUkDbwBw94F8fxHPG4Gb3f144EHgj2ZWMMBqZvWBmcA6oLG7nwz8F+gBrHP31u7+Rlj3FmAscJO7twTaAZ2A52Ltufs5fH8OoJuA7gRJ4svJdsTMrjWzRWa2KD8/f5+elH3117/+lVatWuXEpOGRI0dy/vnnc/zxx0cdiqRRfn4+N910EytWrABgx44dPPnkk+Tn59O7d28AqlatCsBbb73FggULgCDxHT16NHXq1OGiiy6KJvg0aty4Me+//z7ffPMN3bp149RTT+XZZ5+NOqyMydX9j11WYV9vZdhOCzN7IPxtezvsCFhoZv+XpO7FZvaOmS0xsw/N7E4zK/P5J8qT2Ix290+Lua0GRpajrXQYD/zK3b8DcPd1wBSgSG8M8Ly7rw//fpbgeTgzbvnNwEHAwLj2/gSsjW/EzOoAQ4Gp7j4zrLcBGA6ca2ZnJNn2H919m7s7cDVBAlWIu49x93bu3i72n2MU3J3x48fnxKThVatWMXHiRO6+++6oQ5EI7Nmzh5UrV9KkSRNq167Ntm3bgOB9Ee+///0vW7du5cgjj4wizIyoUaMGAwcO5LTTTqN///5s37496pAyKpf2P3ZUVCpuZfD/gCuBnu5+UtgR8DQw0cwKjqsPRzGeAW5z91bAuUB/gk6IMikxGjObFd5GuvujJdV194ll3WiabAT6m9l/zOw9M8sjSBxaJKn7cdzfsWM2D40r6whscPfC32rBPCMS6tUEXk8oj80uPCvJtpfF/nD3L9x9c5I6FcKcOXP45ptvuOSSS6IOJe1mz55NtWrV+PGPf0z79u1p3759wVFRQ4cOpX379qUeHiz7hxo1anDAAUWnF+7evRsIJlF/9tlnQPL5B7t3786qeQnbtm0j+D+rsBNPPJFNmzZl9WH/oP3PoLXAPe5e8Pvr7n8g+O2+BMCCD9bDwLS4zoLPgAeAG8ws2e95EaWlWYcB9xBkVRXdXwh6Wq519xPcvTXB8E/VJHW3xv5w9z3hn/HdXI2B/yVZb2PC4wbh/U3hvJu8MKF6luB8P0UOgXf3xEnXFdaTTz5Jz549qV69etShpF2fPn1Yvnw5CxcuLLj96U9/AoIzEC9cuJB777034iglFfr27UvHjh2LlLdo0YL8/Hw2btzIsmXL2Lx5M82aNStUp2HDhtSpUyerfuwuuOCCguG2eKtXrwagQYMGRZZlk1zf/0wNRbn78+7+RMK2qwIHAl+FRScDzYBZCavPJPiNLtN/2aUlNlvcfa67v1eWxqJiZtWBS4G/pyjWdQRDUYnqJTyOTYAZEc67id1aufuh7n57CmKJxPr163nppZdyYhhKcs9FF13EQQd9/xG//PLLadKkCZMmTQJg165dTJw4kU6dOnHssccCcMABB9CnTx82bdrEc889l7Td/dWQIUP46quvCh5PnjyZqVOn0qNHDxo3bhxhZJmRy/ufqcQmyXYbAk8AnxNM3wBoHd5/klB9ZXh/YlnaLu1w7+Ku5l2EmY1396vLWj/FDiDI5vYklB+apG5ZvAGcbWbNE4ajjk1SbwvfvxgFzOxhgrk88xKX7Q8mTJhAx44ds3ouQXFWr17NpZdeypYtQefa0KFDeeyxxxg1ahSnn356xNGlV79+/ViwYAFr1qwBoG3b4NRUs2bNol69ehFGljrTpk3j3HPP5a677gKgTp06fPnll4wYMaLQf+4zZsxg586dXHPNNVStWpUqVaqwfPlyfvOb37B+/frimt/vDBs2jHHjxnHeeedRuXJlduzYQc2aNRk1alROHA2Z6/ufQg3NbFHc4zHuPiaxkpk1BmYDRxL8hl7o7mtibYT3m+LXcfdvzWx33PISlZbY1DazTkBZ0rGifbsZ4u6bzWwucJmZjXL3T83sGGBvzyj3e4LD2keaWU93/87M+gF1gW0J2x0E3Gdmf3f3uQBm1hu4CLhrH3YrUrfeeiu33npr1GFEolmzZixcuDDqMCIRG37LZp9++il//vOfy1R3zpw5zJkzJ70BRaxDhw506NAh6jAik+v7n8L5Yvnu3q60SuGBPS3NrAYwAHjbzK5292dSFUhpiU0LYE4Z2jHK0buTJlcCjwALzGwlQffWVOCqcN7LM0BsXOXPZvYgsAiIfcP90syOcffL3P1rM+tCcL6edWb2CTAZ+BeFj57C3f9gZhuAR8IXagvBoe+dYxODzewfwOnh33nAbHe/OQ3PgYiISJlFNRHe3b8l6BQ4CxhjZtP4fnpH7fi64W9r5bjlJSotsfmMsvU6GDCkLBtMF3dfS/KJRb+I+3tEkuWti2lvMXBafJmZ/RMo0gcdHhFW7FFh7t6juGUiIiLZLpwLu92LHoKWB5xDMDSVF5YlXrAu9vjdsmyrtMTma3d/qiwNmdnlZam3PwhfgNHu/ou4MiOYY6Pz7IuIyH4vdh6bDPkX8FvgPwnlzcL7rwlOhPspwalS/hhXpwvBHNoyzdpP2R65e7dUtVUBVAb+z8z+X1zZjQSHv0d9IkIREZGUyPBRUcPMrODKomZ2KXAx8Ky7rwt7c24GuodDVJhZU+A24BF3X1GWjZTWY1PVzA4j6D76qpS62WQ7wQTiB81sJMFh3muAru6ePSewEBERyYw7COa5zgqPcKpGcE65m4CCo6fc/XkzuwwYFV5GoSrBOemSTSVJqrTEphbB0MtcoFd59mB/Fl5G4dfhTUREJCtlavJweH3FN8pY9znKOOyUTImJjbs329uGRUREpGLLpsuDxGRs1pCIiIhIupU2FCUiIiJZKMNHRWWMEhsREZEcpaEoERERkQpMiY2IiIhkDQ1FiYiI5CgNRYmIiIhUYOqxERERyVHZ2GOjxEZERCQHlfM6T/sNDUWJiIhI1lCPjYiISI7Kxh4bJTYiIiI5KhsTGw1FiYiISNZQj42IiEiOysYeGyU2IiIiOSobExsNRYmIiEjWUGIjIiIiWUNDUSIiIjkoW0/Qp8SmgqtUqRI1atSIOgyJQKVKuduh+te//jXqECQiVapUiTqEyESRZGRjYpO735wiIiKSddRjIyIikqOyscdGiY2IiEiOysbERkNRIiIikjXUYyMiIpKjsrHHRomNiIhIDsrWw701FCUiIiJZQz02IiIiOUo9NiIiIiIVmBIbERERyRoaihIREclR2TgUpcRGREQkR2VjYqOhKBEREcka6rERERHJUdnYY6PERkREJAfpBH0iIiIiFZx6bERERHJUNvbYKLERERHJUdmY2GgoSkRERNLKzI4xs9+b2ftmttTMPjCzf5jZ8UnqnmFmb4T1PjKzh8yselm3pcRGREQkR8UmEO/rrQzGAW2AM9z9eKAdwajRQjNrHRdPe2AG8ERY7xTgHGBSWfdJiY2IiEiOymBiAzDI3b8GcPetwK1AdaB/XJ0HgKXu/lRY73/A3UB3M+tclo0osREREZF0O8Pd5yeUrQ3vDwIws0OBTsCshHqzAAcuLcuGlNiIiIhIWrn7ziTFLcP72eF9K8CATxLW3QTkAyeWZVtKbCSpzz//nK5du1K5cuWoQ4lEru+/iGS/VA1DhUNRDc1sUdzt2jKE0B9YBjwZPm4Y3m9KUndT3PIS6XBvKeLpp5/mzjvv5MADD4w6lEjk6v5v27aNwYMHM336dKpUqUKdOnUYMWIEZ555ZtShpd11113Hm2++Sc2aNQuVf/vtt6xcuZL58+fTunXraIJLs5UrVzJ27FjmzJmDu7Nlyxbq1q3L9ddfz5VXXhl1eGmXy+/7FMt393ZlrWxmPwEuIhii2p7KQParHhsza2dmeWa208zGRx1PNtq4cSNjx45l1qxZdOjQIepwMi6X979nz57MnDmTBQsWsHTpUvr27UvXrl1ZuHBh1KFlxGOPPcYbb7xR6Na7d2+OPvrorE1qAF5++WX+9re/MWHCBBYsWMD777/PFVdcQa9evZg0qcwHouy3cv19n+HJw7FtdgKeAM539w/iFuWH97WTrFYnbnmJ9qvExt0XuXtrYF18uZlNNrP94l1oZleb2dVRx1Gc2rVrM2vWLJo3bx51KJHI1f2fPXs2U6dOZdiwYdSrVw+g4Ef99ttvjza4DOjduzctW7YsUj5u3Dj69OkTQUSZ07hxYwYNGsRRRx1VUHbDDTdQt25dnn/++QgjS79cf99D5hMbMzsL+CvwE3d/K2HxEoJJws0T1qlNMAz1blm2sV8lNiVYB3wadRBldHV4q5AqVapEpUrZ8rYov1zd/8mTJ2NmdO7cuVB5ly5dmDdvHl9++WU0gWXIKaecwsEHH1yobN68eaxbt44rrrgioqgyo3v37lxzzTWFynbu3Mn27dtp1KhRRFFlRq6/7zPNzM4jOJ/NT9x9cVz5PwHc/QvgNeCshFXPJphUPLks28mKb3B3v8ndfxZ1HCL7q7y8PBo1akStWrUKlbdo0QJ3Z8mSJRFFFp0nn3ySn/3sZ9StWzfqUDIqPz+f/v3707RpUwYOHBh1OGml933memzM7MfANOBp4EQz+7/YjeBoqJhbgVZm9vNwvXrAPcBUd59NGVT4xMbMLjezj83sEzObb2bdEpa/ZGZfmJknlHc1s9fN7G0ze9fMZppZ74Q6XcI5O2vNbKGZ3WBmc8xsS1jezILTOm8ws9Vx6w01szVm5mbWLK78UDObEG5vcXh7wMwamVl1M8sjONtibK5QnpldlYanTaRc8vPzqVOnTpHyWFl+fpmGtrPGV199xfTp0+nbt2/UoWTMunXrOO6442jSpAkrVqxgypQpHH744VGHlVZ632d0KOpR4EDgDmBiwq2Auy8EzgX6mdlSYCHBeWx6lnWfKnRiY2ZnE5xG+XF3P4Kge+oSoEGsjrufD4xOWO8IgszwTnc/yd1PBF4A7oqrczTwEjATaOru7YEmwMnAIndv7e6r3b0j8M/49t39rvi24kwAvgPauHsb4DKCYafj3H1bOD9oUVz7rd19wt49OyKSLn/9619p1apVVk8aTtS4cWPef/99vvnmG7p168app57Ks88+G3VYkiXcvbm7WzG3Zgl157l7R3c/3t2Pcvdb3H1bWbdVoRMbgu6nNcAjAO7+HTAEqFXCOgAnAdWA5XFlf+T7Y+UB7iRIQu5291hvzz3A7n2ItyOw0t33hPF+DNwOfF6eRszsWgvPBfDVV1/tQzgiZdOwYUM2b95cpHzTpk0Fy3OFuzN+/PisnzRcnBo1ajBw4EBOO+00+vfvz/btKT0St0LJ9fd9qnpryjN5OBMqbGJjZpUJLn71dlzigbuvBf5XyuoLga3AG2Z2q5k1c/cd7j40rk5H4EN33xLX9jZg5T6EPRu428zGWHB10sruPs7dV5SnEXcf4+7t3L1dtk/ek4qhdevWrF+/nq1btxYqX7VqFWZGq1atilkz+8yZM4dvvvmGSy65JOpQMmLbtm3EfcUWOPHEE9m0aRPLly9PslZ20Ps+O1XYxIbg0K4qJE9iNpa0oruvAdoDcwh6eD4J59t0jKvWeG/aLsWl4fa6AHOBz83sNjOryM+zCD169MDdmTNnTqHymTNn0qlTJw455JBoAovAk08+Sc+ePalevXrUoWTEBRdcwIIFC4qUr169GoAGDRoUWZYt9L7PThX5Bzcf2EV4cawE9Upb2d2XufsvgEOAXsDhwL/NrH5YZV052t5NcKhZvCLDYeE8mvvcvQVwGsF8mvuB3ol1RSqSs88+m+7duzN48GA2bgxy+/Hjx7Ns2TIeeOCBiKPLnPXr1/PSSy/l3DDUkCFDiB/2njx5MlOnTqVHjx40btw4wsjSS+/7aE7Ql24V9pIK7r7bzBYAbc3MYsNRZtYUKPH4SzPrAhzh7n8OL40+3sw2AVOAZsAG4A3gYjOrFRuOMrPqBCcGeiehyS+B+gllRyfZ7t/d/fIw/jfMrDtBghbfn7kLqBrWbwSc6O6vlvhkZFi/fv1YsGABa9asAaBt27YAzJo1q+AkVtksV/d/0qRJDBo0iJNPPpmqVatSu3ZtZsyYQfv27aMOLWMmTJhAx44dOfLII6MOJWOGDRvGuHHjOO+886hcuTI7duygZs2ajBo1KieOCsv1931FS0pSwZKNrVYU4VFRrwID3P0RMzsA+AvBkM8z7n51WG8IwSRgCx9fDQwCOrn7f8OhoIcIjlJq7u7bw6Oi3gUeA251dzez+4BrgaXu3jkujvOAl4H/5+7/NrOjgOeBYwkSqNVhPQcudffJ4eN2wJvAJe4+LSwbQ3CyoSOBvsAV7n52cc9Bu3btPFdO7S2F5eKJAmO2bNlSeqUsVq1atahDiEyVKlWiDiEy7dq1Y9GiRRnLNE444QSfOnVqStr60Y9+9HZ5rhWVThX6m9PdY8eu/yo8j8zrBIdofwlcGJ4H5iXglwDh44sI5re8ArxqZouB9wh6Ys6NXWzL3T8EfkwwH+ZzM3sD+ABYSnBK5/g4ZgD3AmPC9gYQJEQAL5nZ9eHftwO/jp2jBhgD9I0lNaEHCeb2LCO4sukd+/g0iYiISKjCDkXFuPvfgb8nFD9ThlX7laHtV4HW8WVmdhtBgpNYdzAwOKH4Twl1HgBKHJgNDwGvEFmtiIjktmwciqrQPTbpZGZHmNnIhLIawBEEvTYiIiKyn8nZxAaoCdxgZm0BLEhbhwE7gcejDExERCQTdFRUdllLcI2KSWa2g+Cop6XA2e6+IdLIRERE0qwiJiWpkLOJjbt/QzjpWERERLJDLg9FiYiISJbJ2R4bERGRXJeNQ1HqsREREZGsoR4bERGRHKUeGxEREZEKTImNiIiIZA0NRYmIiOSobByKUmIjIiKSo5TYiIiISFbI1jMPa46NiIiIZA0lNiIiIpI1NBQlIiKSozQUJSIiIlKBqcdGREQkR2Vjj40SGxERkRyVjYmNhqJEREQkayixERERkayhoSgREZEcpaEoERERkQpMPTZSYVWqlNt59549e6IOITLVqlWLOoRIValSJeoQIrNx48aoQ4jM7t27M7o9XVJBREREpIJTYiMiIiJZQ0NRIiIiOSobh6KU2IiIiOSobExsNBQlIiIiWUM9NiIiIjlKPTYiIiIi+8DMmprZDDPzdLSvxEZEREQywsyuBN4AmpVS72Ize8fMlpjZh2Z2p5lVLss2NBQlIiKSozI5FGVmdYHrgM7AXcCRxdTrDjwDdHP3mWZ2GPAm0BC4ubTtqMdGREQkB8XOPJyKWxltBjq7+6oSYjLgYWCau88EcPfPgAeAG8ysRWkbUWIjIiIiaefue9y9tGvFnEwwTDUroXwmUBm4pLTtaChKRERE9lVDM1sU93iMu4/Zi3Zah/efJJSvDO9PLK0BJTYiIiI5KoVzbPLdvV0K2mkY3m+KL3T3b81sd9zyYmkoSkRERLKGemxERERyVAU8QV9+eF87vtDMahDMsckvskYC9diIiIhIRZEX3jdPKI89fre0BpTYiIiISEXxFvApcFZCeRdgD/BcaQ0osREREclRGT6PTanc3QlOwtfdzM4KY2wK3AY84u4rSmtDc2xEREQkI8xsNHAqcHj4OC9c1Nnd/wfg7s+b2WXAqPAyClWB0cCIsmxDiY2IiIhkhLv/soz1nqMMw07JKLERERHJQakeRqooNMdGREREsoYSGxEREckaGooSERHJURqKkpzx+eef07VrVypXrhx1KJJheu1FZH+mxEaKePrppzn99NP59NNPow4l47Zt28att95Ky5YtOf744+nYsSNz586NOqyMydXXfuXKlQwcOJBTTz2VU045heOOO46OHTvy9NNPRx1aRuTy+/6tt97i4osv5sQTT+TEE0+kW7duzJs3L+qwZB8osZFCNm7cyNixY5k1axYdOnSIOpyM69mzJzNnzmTBggUsXbqUvn370rVrVxYuXBh1aGmXy6/9yy+/zN/+9jcmTJjAggULeP/997niiivo1asXkyZNijq8tMvV9/2bb77J+eefT9u2bXn33Xd59913ueCCC7jkkkt47bXXog4vIyraCfpSQYlNGZnZYWb2lZn9KupY0ql27drMmjWL5s0TL9OR/WbPns3UqVMZNmwY9erVA6B3794cffTR3H777dEGlwG5/No3btyYQYMGcdRRRxWU3XDDDdStW5fnn38+wsjSL5ff98OGDaNOnTr85je/KSjr378/hx9+OIMGDYowMtkXSmzKbifB9Ss2RB1IOlWqVIlKlXLzbTF58mTMjM6dOxcq79KlC/PmzePLL7+MJrAMyeXXvnv37lxzzTWFynbu3Mn27dtp1KhRRFFlRi6/79955x2OPPJIqlSpUlBmZhx33HHk5eWxdu3aCKOTvZWb32J7wd2/dPd27p79/dI5Ki8vj0aNGlGrVq1C5S1atMDdWbJkSUSRSabl5+fTv39/mjZtysCBA6MOJ61y+X1vZuzevbtIeWzi/Mcff5zpkDJOQ1EpZGZHmtlUM8szs8Vm9paZDTGzY8OynWY2Pq7+S2a2wcxWx5XdaGbLzMzNrL+ZPR62szNse4KZrQmXdzezV83sAzP7wszuNbNKYTvNwm1uMbM5ZnaFmc0L67mZtSsmpspmNszMlpjZO+H9eDNrnbCvvcxsqZl9ZGarzOwRMyv8LSKRy8/Pp06dOkXKY2X5+fmZDkkybN26dRx33HE0adKEFStWMGXKFA4//PCow0qrXH7ft2nTho8//pht27YVlLk77733HgCbNm2KKjTZB1H22LwILHb31u7eBrgRGAh86+6tgXXxld39fOCfCWWPAOeHD28Cxrr7ycAvwuVXAXeFy38LXOHuxwAXA78G7g7rrQ63uQg4FjjK3c8AjgS+BlYkiwn4DXAR0MHd2wKnAy2A7rEKZnYLMBa4yd1bAu2ATpRwDQwzu9bMFpnZoq+++qq4aiKSYo0bN+b999/nm2++oVu3bpx66qk8++yzUYclaTJw4EC2bt3KnXfeybZt29i1axcPPPAAX3zxBQAHHnhgxBHK3ogksTGzhgRJQ8Hlx939P8CdwN6myDPdPS/8+zmCRCneH9z9q3BbbwD/AG4xs7oJ9aoAI8N6m4FTSoipI/Bfd98a1t8E3AG8CWBmdYChwFR3nxnW2QAMB841szOSNeruY8Jhr3bZPr5fkTRs2JDNmzcXKY/919awYcNMhyQRqVGjBgMHDuS0006jf//+bN++PeqQ0iaX3/edOnXihRdeYO3atXTs2JEuXbqwffv2gsnETZs2jTjC9NNQVOp8DbwHPGFmD5rZSQDu/lD4w783lsX+cPcd7r4mYfnShMeLgFrACQnlK919R1xbK919TzHbnA2cY2b/MrNLzKymu89393+HyzsCNYHXi4nlrJJ3STKpdevWrF+/nq1btxYqX7VqFWZGq1atIopM0m3btm24e5HyE088kU2bNrF8+fIIosqMXH/fn3rqqTzzzDMsXryYefPmcdddd7F+/XoOOuggWrZsGXV4shciSWw8+AY5E3gM6AksMrOVZtZrH5rdUsryxF6Xb8L7JuVsp4C7PwRcCdQl6AFab2Zj4nqBGoT3N4VzdPLMLA94FviSIOmRCqJHjx64O3PmzClUPnPmTDp16sQhhxwSTWCSdhdccAELFiwoUr569WoAGjRoUGRZtsjl9/1nn31GXl5eobI9e/bw8ssv83//938ccICuOrQ/imyOjbt/4+4DgaZAV+AL4EkzOyesshtI7N/alwm3iUNO9cP7fTqez90nuXtH4EfAaKAP8Hi4ODbrbkQ4lyh2a+Xuh7p7dp8kYj9z9tln0717dwYPHszGjRsBGD9+PMuWLeOBBx6IODpJtyFDhhA/p23y5MlMnTqVHj160Lhx4wgjS69cft+/8cYbXHXVVQWHtO/atYu7776bKlWqZP05fCB1w1AVbSgqknTUzA4G7nT3m8JhnhlmtohgiKoV8CpBj0b9hFWP3ofNHg+8G/f4ZILemcQhqjIzsxHAGHf/xN1XAr82syMJ9gHgjXAbrZOs+zDwvLtXuHN39+vXjwULFrBmTTCa17ZtWwBmzZpVcAKvbDVp0iQGDRrEySefTNWqValduzYzZsygffv2UYeWEbn62g8bNoxx48Zx3nnnUblyZXbs2EHNmjUZNWoUffv2jTq8tMvV933Lli1p0aIFXbp0oW7dupgZZ511Fi+99BK1a9eOOjzZS5ZsXDntGzVrBnwEnO3ur4dlFwGTgVPc/W0zuw/4JXCcu//XzH4G/AHY6e7NEtr6BOjl7uOTbOtqYBzBfJjL3P0rM+sIzATud/e74+rOAXD3zsXEvRqY4+5Xx9V/F7jF3XebWQNgIfBPd785rHMTcB9wvrvPDct6ExytdUI4QblY7dq182w/rXlxcvVkcTF79hQ3tSv7JTu3SC6JP2Fcron1GuWizp07s3jx4ox1f5x00kn++uuJU0D3TvXq1d9293YpaWwfRTWA+CUwAng0PJeMAd8C3d397bDO/QSHTr8dJhTPAjOAnuE8leuAlgSHiAMMNbMBQKdikoX7gKfCRKg+MAq4B8DM6gFzCIaTCNt/wN2fDh+3A/4MNAYuDJefBDxMkHy9a2bfERxR9UysXQB3/4OZbQAeMbMaBD04HwKdS0tqREREpHwiSWzcfRswJLwVV+d/wGVJFl0d9/cCYEIZN7siPBdOcdtqXUIsi4pZPi28lcjdJwITyxKkiIhIplS0+TGpkNt9/SIiIpJVlNiIiIhI1sjqxMbMJhCc+RfgJTP7ZZTxiIiIVCQ63Hs/E14rSkRERHJEVvfYiIiISG7J6h4bERERKV5FG0ZKBfXYiIiISNZQYiMiIiJZQ0NRIiIiOUpDUSIiIiIVmBIbERERyRoaihIREclBFfHkeqmgHhsRERHJGkpsREREJGtoKEpERCRHaShKREREpAJTYiMiIiJZQ0NRIiIiOUpDUSIiIiJ7wcyqm9mDZvaRmS01szfM7MxUb0eJjYiIiGTCJKALcIq7Hw/8GXjZzNqnciNKbERERHJU7CR9+3orw3bOAroDg939fwDu/iTwIXB/KvdJiY2IiIik288AB+YklM8EzjCzQ1K1ISU2IiIikm6tga/cfUtC+UrAgFap2pCOiqrg3n777fzKlSt/GtHmGwL5EW27Isjl/c/lfYfc3v9c3neIdv9/mMmNvf322y+bWcMUNXegmS2KezzG3cfEPW4IbEqy3qa45SmhxKaCc/dGUW3bzBa5e7uoth+1XN7/XN53yO39z+V9h9zaf3fvFnUM6aChKBEREUm3fKB2kvI6cctTQomNiIiIpFsecLCZ1Uwob04wqXhJqjakxEZKMqb0Klktl/c/l/cdcnv/c3nfQfufLv8gmCTcOaG8CzDf3b9M1YbM3VPVloiIiEhSZvY8wQTps9x9o5ldDTwBdHL3hanajiYPi4iISCb0BO4F3jKzncBm4LxUJjWgHhsRERHJIppjk4PMrLGZ5ZnZFjObE3U8ucrMRprZCjNzM+ucpm00MbOpZvZBeOG52WZWPR3bkswws3bh53enmY2POh6RikaJTQ5y93Xu3hpYVFrdXGdmnc1sSDradveBQN90tB3nj0AD4Pjw1hioluZtShq5+6Lw87suvtzMJptZSrv008XMrg7nV+zXzOwwM/vKzH4VdSzyPSU2IiXrDNwddRD74EzgdXff7e67gBNjF6CTrLMOiOos5eV1dXjb3+0keM43RB2IfE+Th0WyW11ge+yBu28voa7sx9z9pqhjyDXhIco5cZbi/Yl6bCo4MzsynCORZ2aLzewtMxtiZjXM7EAzG2Fmb4e3JWY20cx+kKSdLmEba83sDTP7eZI6E8xsTTjn4ywzm2ZmH4ZzM36apP6xZjbdzFab2Soze8XMTkqoc7KZvRrGvtjMXjez2+OW1zKzR8PYF5vZu2b2uJk1j6vTI4zdzWyYmQ03s4XhvjxuZlXj6lYL566sCmNfZmY3xi0/J66tIXHlReYcmdnTwC/jlueZ2SNm1iy+vpldYWbzzOyLsN16ZnaEmf0l3J/Yfg01s4wMA8Wes/DhL8N4XwqXleV1O8vM/hnGnmdmi8zsyoQ6ie+XqWb2Xvh4QAb2sbWZ/T3uOV5sZjeZWaVwefwcph5m9rSZLQ33+bok7bUxs9fMbL0Fn7OhZvaUBXNZ8sysY1zdXmFbH4XtPWJmteKWvxr3fmhjZi+b2cfh4+57sa+Xh+t/YmbzzaxbwvKXYttLKO9qwWfu7fB5mmlmvRPqxH83LDSzG8L39ZawvJkF3xkbzGx13HpD417/ZnHlh4bvjfjX5QEza2Rm1cP3ZTsgNlcoz8yuCtct7vvuWEsyryjc78S4brTgc+9m1t+C74i3wnWnJrxvu4ev1Qfh83dv3PuntM950rlOZlbZgu+pJWb2Tng/3sxaJzzvJb6HZB+4u24V+AZ8DNwd97gDwX/gzYBDgS+BH4bLDgD+QDB3plLcOkcDO4CH+P5IuPuAr4A5Cdu7muAskFOAmmHZ/QSH5dWLq9cC+Ab4S2xbwBBgC3BU+LgW8DVwddx6Fwdvu4LHY4AZQJXw8aHhPl+d5Llwgm7fU8LHJwDfAX3j6vwzrHN4+Lg1sBG4I0lbQxLK5iR5PobEx5uk/vrY60NwuvB8oB5wOTAbqB4uqwe8Djyc0EbnMJbOaXr/FNrPsrxuYdloYETc++WYcN9+Wsz75V9A/bDsCWBABj4bA4G/AgeEjxsDy4Gbkjy/84CDw7L+wO6E/a1PMJzwbFx7/YCtwOqE7d4Svu+6xK27GJiR7L0DjAeqE5yc7N9A93Lu59nAnth+EXzOxxJ8JscX914FjiD4rugcV3Zz/P6Q/LthZLjfiZ+F8Umei9jr3yyubAbwZNz76yiC75r4OOYkth+WF/t9Fz5eHb/PJcTVLIzrI6B1WHYFMDUh7gVAo7CsI7ANuCehrTkU8zlPFhNwB7CU778/6wDzKfw5LNN7SLe9/G6IOgDdSnhxgqudOnBlQvmvww/CAUCLhGVHh+u0jyubGH5R1Yorq05wVdU5CevHPvAXxZW1C8vOTWhzB+GPWVhWlSCReTJhvdMStjE87u+lwNiE5T+Njz+u3IFpCWVLgafDv7uEdX6VUOcxgh+BmgltDUmoNyfJ8zGEkhObb4BqcWUtCHpCDyL8IY1b9svwdbC4ss5kNrEp9XULyw4nTMriyv4OvFTM++XSuLL6QIMMfD4OJS7ZDstGAsuSPL83J/lcXRNXNiwsa57Q3scUTgTqECSB/0io1yNc/4zE9w6FP4uHArXLuZ/zCX484983TcK2xxf3Xo2LqUlcWTXgroT3Q1m/G8ZTtsRmC3BnQr1ewI8SPjuJ7Zf4fRf+vZryJTZ/TNj3wxPi7pmw3kSC74q6CbEm/Zwniwl4AXglod1OQLfyvod027ubhqIqtq+B94AnzOxBC4cL3P0hd9/g7t8BPwq7Y5eGXbzPheu2iGunI/Chu2+JFbj7NmBlCdv+OO7v2MS4Q+PKzg3bLJg05+47Cf5jPiss+hD4LzA11p0c1rszrp3ZQB8LhhS6mVlVd5/mxZ+w6eOExxvi4jo3vH89oc5Sgt6jdIyFr3T3HbEH7r7S3fcQ/DD0CIcNYq/NHUANCj+PmVaW1w2CL/chYRf+kjD+8yj8voq3LK69De7+dcojL2oj0N/M/mPBEFgewQ9WshhLez93BDa4+6qE9d5PeNwRqEny9xgUfg5j4p+bL9x9c5I6SZlZZeAU4G0Pf/3CdtYC/ytl9YUEScsbZnarmTVz9x3uPjSuzt58N5RmNnC3mY0xszPMrLK7j3P3FaWsV+L33V7GEv/c73D3NQnLlyY8XkTwXXFCQnlxn/NkZgPnmNm/zOwSM6vp7vPd/d/h8r15D0k5KLGpwMIvsjMJehx6AovMbKWZ9QIIx9n/BcwFWnlwCOj54erxczkak/xLcGMJm98a93fsA1w5rqwBcETcGHle+MPyA8L3VfhleQpB9/6NwPvhj+QFce0MAG4Ajg335UsL5kYUNxdla8LjPXFxNQjvJybENIBgyO6gEvZ3b20ppnwYMAoY7O7Hh6/NXeGyKA+3LvV1MzMj+K+zO/Azd4+9t/5J8bEX9zyk018IhlaudfcTwhhHE/RAJSp438T9IMW/n8v6GYm9x25KeP6eJXiPJV7gj/ikYS80BKqUMbbE7a4B2hP0OAwBPrFgvk3HuGp7891QmkvD7XUh+G763Mxui81dKSHeEr/v9lJpz/2mhMffhPdNytlOAXd/CLiSYOL+P4D1YZJXN6xS7veQlI8SmwrO3b/x4HwnTYGuwBfAk2Z2DnAVsMXdf1fCfw8QHAaa7Ee93j6Elg+87+6tE24/dPcfxsX/mbtfT/DfcQ+CH5PnzezocPlud3/c3VsBJwLTgN8Ag/cyJoBLEmI62t0PdfepcXX3EMx5iJfKiXtXE3RHz0lhm6lQltftSIL/Kse4++rIIi2BBScZvBT4u7u/l4Imy/oZib3HRiQ8f63C99jtpFY+sKuMsRXh7svc/RfAIQTDQYcD/zaz+mGV8nw37KYMnxl33+bu97l7C+A0gl6Q+4HeiXWTrFvS912ZYyiHugmPY8/L2n1oE3ef5O4dgR8RJNt9gMfDxZl+D+UcJTYVmJkdbGZ/gOC/THefAcR6O1oR/GfqCaslG+Z4A2hphY/aqE5wufi9NSNs88CEmC+w8GgjMzvBzO4M49/p7lOAnxMkN8eGdf5iZjXCOkvc/WqC7uhWexkTBBOG42OqZWZT4r7MIfjPqH5cnSokfz52hctjuifuczGq8n1PV0yUQ1Axpb5ufN/jURHjjzmA4H2UqhjfAA6yuKPxQscmqbeFhPcYgJk9bGZn7OX2k3L33QQTXNuGPWmxbTWl6I9yYjxdzKxv2M5Wdx8P3EQw+bVZWK083w2FPjOho5Ns9+9x8b9B0PP3Pwp/pncRJigWHC11Thm+78ocQzkcn/D4ZILXN3GIqswsOFL1CCgYsvo18CLf70NG30O5SIlNxVaD4FDd0+LKziT4r2UuwXBBHTOLHZJcjWAeR6LhBD8E98R9Od7Nvr3+Q8L734XzAAh/FH5PMLsfgi7XX5tZy7j1ziD4UC8IH3chOEqFsI0jgMOAmeUNyN1nA1MJxvcPD9urRjAkRMI4/Rygq5nVDh/fQJjEJPgkvG9K8N/X3wme/9K8AJxrZm3COBoDRQ4xjsCQ8L6k1+0jYAXB3KcGYZ0zgHOoIMJ5KnOBy8zshwBmdgzBkS974/cEwxAjzeyAsL1+JCQP4XYHAVeZ2ZmxcgsOob6I75/DVLqboKflhnBbBxB8pks7J9FhwEALT/8QDgV1Ipj3Fpt7kuy7YTDBiecSzQFqhUPgmNlRJJ8PcpmZ/SzucRuCZCr+M/0J0CTcZne+n39W0vddLIbT4vbpZxRNdMqjj5k1CtvqSHDU5ijft5NYdgAGxH2+GgDHEe5/RO+h3OIVYAazbslvBEcnDAHeAfKAd4H/AD+Jq/Nbgln5ywg+OLcS9OKsIe7QYoIfpTyCrue3COa8zCFIMvII/oN7JFzPw/YuDG/LimnzaIJE4vMwxtcpfDRVQ+ABYAnBh/W9MMbT4ur0CuNYEsbxXrgPliR2J+ia/nNYnhfGH9uH2gS9DfcSTH58Pyy/n6JH+DQFXibocp5HMKYf/3wcFtY7MNzHVeHzcA1BN33ithOP5KhHMAdkHcEkzn8CD8c9t1cSHMGzIixbQdzRYil47/RIeM7ygAvL8rqFdY4BXgnXnRfuy4sEP3h5QEuKvl+eyfDnownBaQm+CPfhGeCpMJ48gs9G/PP7S4IJ5PHPyzNx7bUJ21lPkHjfSnDEzSdJtv1zgs/jcoL39t8ofGTQP8L2Y7E8vI/7ejnhEVphbJeFf28I238pYXsXERzu/SeCz8Hi8H4acFxC27HP11qC3oSfE3wWZieJY1j4mi8muFxHv7jX//qwzm3Am2GbeeF77OqEdo4iGKL6IGzrVMr2fVcvfJ3XhbEOCF+j2PvyFIIh+vjvrDwSjkTj+6Oizgmfu2Xh8zec7492qkcJn3O+fy/tjHsdKhMc1fkvgl6fvPB5v4+4o6rK8h7Sbe9vurq3iEgxzOyfwCHufkrUsWSSmS0BPnD3y6KOJR0suE7VOOAIr6DzyGTvaShKRHKeBWfEfSqhzAjm2Oz1fIuKzoIzZI9MKKtB0NuTtfst2U2JjYhIMITwf2b2/+LKbiSYpzIy+SpZoSZwg5m1hYJkbhjB8MrjJa0oUlFpKEpEcl44Ifd3QDeCU93XI5ifMdgr3iH7KWNmBxFcOqMz4RmpCXpqBrr7uxGGljZmNoFgfw8jmOPziLuPjjQoSSklNiIiIpI1NBQlIiIiWUOJjYiIiGQNJTYiIiKSNZTYiEiJzGyoma0wMzezzlHHk4yZXRheTNDjLg1RXN1XzewLMyv3BMN9WbeY9kZW9OdWZH+jxEakgjKzw8If6w3hD1/sSsArzOzD8Eexdukt7Rt3vwvoW0x8X5nZr8rbZnjNrQGpiA/A3f/pwdW9y1L3HIILE+7NdvZ63WLaG0iS51ZE9p4SG5EKyoMro7cmuBwD/v1VgH9EcN2pm4GXw2sARWEn8CnB6eTLqzvB6fBFRFJKiY3Ifsjd5xJcJ6kDcFop1dMVw5fu3s7dJ0WxfRGRZJTYiOy/Pgvvm5pZj7g5JsPM7F4z+4+ZbTWzvNgKZvYTM3vLzJab2Woze8rMDo5vNLy8wJ/MLN/MlpjZM0CjhDrtwu3tNLPxCcvqmNmjYftLzOw9M3vCzE4Il88muLhq47jhtYFx6x9rZtPD9VeZ2StmdlLizpvZTWb2qZl9bGazzKzdvjyZZnaWmf3TzBaHMS0ysytLqH+ymc0Ot7/GzG5KUqdM+yIiqaPERmT/dWR4v8rd/xE3x6QXMMfdOxBcvRgAM+tBMKz1mLsfSXCV70OAmWZWLa7d0QRXh+7o7q2AO8JbAXdfFG5vXXy5mVUluCr4KUCbcP3zgLOAPuG6Z4VxrIsbXhsZrt+C76+w3dzdm4eP55rZUXHb6UVwtfSb3P0o4AKCq3Hvi8sIrsTcNty3nwN/MLOfFlP/1wRXnj6KYFjt92FcsRjLtC8iklpKbET2M2ZWycwuBX4K/MvdFyRUWeLur4Z/LwAuCa8B9BDwjrs/BeDu24FBwPEEP+qEP7j/Bzzh7h+H9VYC/yhjeFcC7YFB7v5NuP5/CZKQXWVYfwhQA7jN3feEZfcRnO5/YGz/w3rz3X1quI2twKNljLE49wFDPTwdu7t/ALxKMJ8pmUfC7eLuzwGvAUPC57pM+yIiqXdA1AGISNnEDSnVAL4k6DFIdqHCZbE/wh/UlWZ2NHA4MDWhbuwKzmcBEwh6WioBi4qpV5rzwvuF8YXu/qcyrn8u8KG7F0xIdvedZrY8jBGgCcG+JCZb+3o16s0EicnZQDVgT7idr4qpn7i9RcDpYXyfU7Z9EZEUU2Ijsp8o6+HMwJYkZQ3C+5+Z2ZkJy74ADgz/bhze/y+hzsYybrshsMfdE9cvqwZAjfh5QaGD4v5OGqO7b/y+s6R8wl6WFwji7+ruq8Py8QQXTCzC3TclFH0T3scSm7Lsi4ikmBIbkdyQH94/5e6/LaFebM5M4o9vvXJsp5KZ1XX3siZDieuvDucHFSdpjGZWby+2F3Mk0BG4NZbUlCbJPtYP79eG92XZFxFJMc2xEckNHxOcc6Z14gIz+204sRiCOTl7gMQjjI4r43ZmhPcnJ2yjt5k9FFe0C7BwWU0zuzBu/ZZmdmDC+hfY92cUXgus2YcYk6ka3u9JKD+0hHWOT3h8chhXLLEpy76ISIopsRHJAeGE2JuBc8OJxwCYWTfgRuCtsN7HwF+Ba83syLBOC+AXZdzU0wTza+6N9aCYWTPgbuDluHqfAA3Do7E6Ar8Py4eE978zs8rh+s3D5YvDGPeE9U43s+5hnRrs24Tcj4AVQB8zaxC2eQZxR5UlMTDcLmZ2cbgfQ2KTj8uyLyKSBu6um266VcAbcBiQR3BmXw///kcxdc8JlzvBnJk8oGWSev8PeJMgsXgbmA6cmFCnBvAngqGU98I6vwjbXgGMJOgtySM4+/CG8O/K4fp1gccIeoiWECQ6Fyds42BgNkFP0lLgwrhlRxNMcv4ceIfgEOmLkuzLTQQ9JMvDOmfH7f+rJTyvr4Z1Ys/pRWH5MQSHqn8BzAP+ArwY7mMe0DJh3S7AG+H21xAcep64rRL3JXwuV8Q9t8Ojft/pptv+fjP3lFzLTURERCRyGooSERGRrKHERkRERLKGEhsRERHJGkpsREREJGsosREREZGsocRGREREsoYSGxEREckaSmxEREQkayixERERkazx/wHXwx2Ks3iiEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\" plot confusion matrix of dev set preditions \"\"\"\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.rcParams.update({\n",
    "    'font.family': 'serif',\n",
    "    'font.size': 16.4,\n",
    "    'figure.figsize': [10,7]\n",
    "})\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "p = trainer.predict(val_dataset)\n",
    "preds = np.argmax(p.predictions, axis=1)\n",
    "\n",
    "cm = confusion_matrix(p.label_ids, preds, labels=[0,1,2,3,4,5])\n",
    "\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n",
    "                               display_labels=[id2label[i] for i in range(6)])\n",
    "\n",
    "disp.plot(cmap='Greys')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "04eaf892",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "      <th>random_preds</th>\n",
       "      <th>model_preds_eval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sadness</td>\n",
       "      <td>34.78</td>\n",
       "      <td>36.30</td>\n",
       "      <td>42.59</td>\n",
       "      <td>34.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>anger</td>\n",
       "      <td>18.76</td>\n",
       "      <td>28.15</td>\n",
       "      <td>18.52</td>\n",
       "      <td>31.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neutral</td>\n",
       "      <td>14.78</td>\n",
       "      <td>9.26</td>\n",
       "      <td>12.59</td>\n",
       "      <td>9.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fear</td>\n",
       "      <td>10.43</td>\n",
       "      <td>11.48</td>\n",
       "      <td>5.19</td>\n",
       "      <td>12.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>surprise</td>\n",
       "      <td>8.82</td>\n",
       "      <td>5.19</td>\n",
       "      <td>7.78</td>\n",
       "      <td>5.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>disgust</td>\n",
       "      <td>8.01</td>\n",
       "      <td>4.44</td>\n",
       "      <td>7.41</td>\n",
       "      <td>2.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>joy</td>\n",
       "      <td>4.41</td>\n",
       "      <td>5.19</td>\n",
       "      <td>5.93</td>\n",
       "      <td>3.70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label  train    dev  random_preds  model_preds_eval\n",
       "0   sadness  34.78  36.30         42.59             34.44\n",
       "3     anger  18.76  28.15         18.52             31.48\n",
       "1   neutral  14.78   9.26         12.59              9.63\n",
       "2      fear  10.43  11.48          5.19             12.96\n",
       "5  surprise   8.82   5.19          7.78              5.19\n",
       "4   disgust   8.01   4.44          7.41              2.59\n",
       "6       joy   4.41   5.19          5.93              3.70"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" output label distribution \"\"\"\n",
    "\n",
    "pred_counts = Counter(preds)\n",
    "pred_total = sum(pred_counts.values())\n",
    "\n",
    "l = []\n",
    "for lab in label_counts.label:\n",
    "    lab = label2id[lab]\n",
    "    if lab in pred_counts:\n",
    "        prop = round(pred_counts[lab] / pred_total, 4) * 100\n",
    "    else:\n",
    "        prop = 0.0\n",
    "    l.append(prop)\n",
    "\n",
    "label_counts['model_preds_eval'] = l\n",
    "label_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a6cccea",
   "metadata": {},
   "source": [
    "# Test predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7a108eb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 525\n",
      "  Batch size = 32\n"
     ]
    }
   ],
   "source": [
    "\"\"\" make predictions on test dataset \"\"\"\n",
    "\n",
    "p = trainer.predict(test_dataset)\n",
    "preds = np.argmax(p.predictions, axis=1)\n",
    "preds = [id2label[p] for p in preds]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c1935153",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "      <th>random_preds</th>\n",
       "      <th>model_preds_eval</th>\n",
       "      <th>model_preds_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sadness</td>\n",
       "      <td>34.78</td>\n",
       "      <td>36.30</td>\n",
       "      <td>42.59</td>\n",
       "      <td>34.44</td>\n",
       "      <td>34.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>anger</td>\n",
       "      <td>18.76</td>\n",
       "      <td>28.15</td>\n",
       "      <td>18.52</td>\n",
       "      <td>31.48</td>\n",
       "      <td>25.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neutral</td>\n",
       "      <td>14.78</td>\n",
       "      <td>9.26</td>\n",
       "      <td>12.59</td>\n",
       "      <td>9.63</td>\n",
       "      <td>10.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fear</td>\n",
       "      <td>10.43</td>\n",
       "      <td>11.48</td>\n",
       "      <td>5.19</td>\n",
       "      <td>12.96</td>\n",
       "      <td>11.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>surprise</td>\n",
       "      <td>8.82</td>\n",
       "      <td>5.19</td>\n",
       "      <td>7.78</td>\n",
       "      <td>5.19</td>\n",
       "      <td>9.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>disgust</td>\n",
       "      <td>8.01</td>\n",
       "      <td>4.44</td>\n",
       "      <td>7.41</td>\n",
       "      <td>2.59</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>joy</td>\n",
       "      <td>4.41</td>\n",
       "      <td>5.19</td>\n",
       "      <td>5.93</td>\n",
       "      <td>3.70</td>\n",
       "      <td>4.76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label  train    dev  random_preds  model_preds_eval  model_preds_test\n",
       "0   sadness  34.78  36.30         42.59             34.44             34.10\n",
       "3     anger  18.76  28.15         18.52             31.48             25.90\n",
       "1   neutral  14.78   9.26         12.59              9.63             10.29\n",
       "2      fear  10.43  11.48          5.19             12.96             11.81\n",
       "5  surprise   8.82   5.19          7.78              5.19              9.14\n",
       "4   disgust   8.01   4.44          7.41              2.59              4.00\n",
       "6       joy   4.41   5.19          5.93              3.70              4.76"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" output label distribution \"\"\"\n",
    "\n",
    "pred_counts = Counter(preds)\n",
    "pred_total = sum(pred_counts.values())\n",
    "\n",
    "l = []\n",
    "for lab in label_counts.label:\n",
    "    if lab in pred_counts:\n",
    "        prop = round(pred_counts[lab] / pred_total, 4) * 100\n",
    "    else:\n",
    "        prop = 0.0\n",
    "    l.append(prop)\n",
    "\n",
    "label_counts['model_preds_test'] = l\n",
    "label_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1c108d62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved predictions to ./predictions/emotion/emotion-stack-eval_f1_macro-60.7083_1638-20-4.tsv\n"
     ]
    }
   ],
   "source": [
    "\"\"\" save predictions \"\"\"\n",
    "\n",
    "pred_file = f\"{approach}-{metric_for_best_model}-{round(eval_result * 100, 4)}_{dateTimeObj.hour}{dateTimeObj.minute}-{dateTimeObj.day}-{dateTimeObj.month}.tsv\"\n",
    "pred_path = f'./predictions/{prediction_task}/{pred_file}'\n",
    "os.makedirs(f'./predictions/{prediction_task}/', exist_ok=True)\n",
    "pd.Series(preds).to_csv(pred_path, sep='\\t', header=False, index=False)\n",
    "print(\"saved predictions to\",pred_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79aa8f64",
   "metadata": {},
   "source": [
    "# Save the adapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f24ff109",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in ./trained_adapters/EMO_emotion_stack/adapter_config.json\n",
      "Module weights saved in ./trained_adapters/EMO_emotion_stack/pytorch_adapter.bin\n",
      "Configuration saved in ./trained_adapters/EMO_emotion_stack/head_config.json\n",
      "Module weights saved in ./trained_adapters/EMO_emotion_stack/pytorch_model_head.bin\n"
     ]
    }
   ],
   "source": [
    "# trainer.model.save_adapter(f\"./trained_adapters/EMO_emotion_stack\", \"EMO_emotion_stack\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}