demo/fastText example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qYJbw_-Fv1TH"
   },
   "source": [
    "## Download & install packages and models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 580,
     "status": "ok",
     "timestamp": 1650389087723,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "EiZKkBZXvbzk"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 70,
     "status": "ok",
     "timestamp": 1650389148354,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "sTjeYBFGuvwM"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/uh/projects/paper-293/sentence-embedding-evaluation-german/.venv/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "import sentence_embedding_evaluation_german as seeg\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 2913,
     "status": "ok",
     "timestamp": 1650389151205,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "af7fcl2HvsjY"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 69,
     "status": "ok",
     "timestamp": 1650389151206,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "Ws1lQPEzaAUQ"
   },
   "outputs": [],
   "source": [
    "# !wget -nc \"https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.de.300.bin.gz\"\n",
    "# !gzip -d cc.de.300.bin.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 626
    },
    "executionInfo": {
     "elapsed": 607,
     "status": "error",
     "timestamp": 1650389151752,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "DoEUr6HTZZuN",
    "outputId": "7f816b11-287b-4e8c-e566-edd4c9f69cef"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "import fasttext\n",
    "import fasttext.util\n",
    "fasttext.util.download_model('de', if_exists='ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bLKzlBlrv7fW"
   },
   "source": [
    "## Specify & run sentence embedding evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 566,
     "status": "aborted",
     "timestamp": 1650389151737,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "D0rKxkCCv0gj"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18min 57s, sys: 1min 44s, total: 20min 41s\n",
      "Wall time: 21min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# (1) Instantiate your Embedding model\n",
    "ft = fasttext.load_model('cc.de.300.bin')\n",
    "\n",
    "# (2) Specify the preprocessing\n",
    "def preprocesser(batch: List[str], params: dict=None) -> List[List[float]]:\n",
    "    \"\"\" Specify your embedding or pretrained encoder here\n",
    "    Paramters:\n",
    "    ----------\n",
    "    params : dict\n",
    "        The params dictionary\n",
    "    batch : List[str]\n",
    "        A list of sentence as string\n",
    "    Returns:\n",
    "    --------\n",
    "    List[List[float]]\n",
    "        A list of embedding vectors\n",
    "    \"\"\"\n",
    "    features = np.vstack([\n",
    "        ft.get_sentence_vector(str(sent).replace(\"\\n\", \" \")) for sent in batch])\n",
    "    return features\n",
    "\n",
    "# (3) Training settings\n",
    "params = {\n",
    "    'datafolder': '../datasets',\n",
    "    'bias': True,\n",
    "    'balanced': True,\n",
    "    'batch_size': 128, \n",
    "    'num_epochs': 500,\n",
    "    # 'early_stopping': True,\n",
    "    # 'split_ratio': 0.1,  # if early_stopping=True\n",
    "    # 'patience': 10,  # if early_stopping=True\n",
    "}\n",
    "\n",
    "# (4) Specify downstream tasks\n",
    "downstream_tasks = ['FCLAIM', 'VMWE', 'OL19-C', 'ABSD-2', 'MIO-P', 'ARCHI', 'LSDC']\n",
    "\n",
    "# (5) Run experiments\n",
    "results = seeg.evaluate(downstream_tasks, preprocesser, **params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 568,
     "status": "aborted",
     "timestamp": 1650389151740,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "qBDaaXo70nNF"
   },
   "outputs": [],
   "source": [
    "# save results\n",
    "import json\n",
    "with open(\"seeg-results-fasttext.json\", 'w') as fp:\n",
    "    json.dump(results, fp, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pLr2z5YEyCd0"
   },
   "source": [
    "## Display the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 569,
     "status": "aborted",
     "timestamp": 1650389151741,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "eUomhnL1wz8J"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task | Epochs | N train | N test\n",
      " FCLAIM:   500   3244    944\n",
      "   VMWE:   500   6652   1447\n",
      " OL19-C:   500   1921    930\n",
      " ABSD-2:   500  19432   2555\n",
      "  MIO-P:   500   4668   4668\n",
      "  ARCHI:   500  18809   4743\n",
      "   LSDC:   500  74140   8602\n"
     ]
    }
   ],
   "source": [
    "print(\"Task | Epochs | N train | N test\")\n",
    "for res in results:\n",
    "    print(f\"{res['task']:>7s}: {res['epochs']:5d} {res['train']['num']:6d} {res['test']['num']:6d}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 571,
     "status": "aborted",
     "timestamp": 1650389151743,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "l8FEvnsZyUKB"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Task | train | test\n",
      " FCLAIM:  0.671  0.661\n",
      "   VMWE:  0.857  0.833\n",
      " OL19-C:  0.770  0.757\n",
      " ABSD-2:  0.648  0.643\n",
      "  MIO-P:  0.793  0.789\n",
      "  ARCHI:  0.604  0.477\n",
      "   LSDC:  0.712  0.568\n"
     ]
    }
   ],
   "source": [
    "metric = 'acc'  # 'acc', 'acc-balanced'\n",
    "print('  Task | train | test')\n",
    "for res in results:\n",
    "    print(f\"{res['task']:>7s}: {res['train'][metric]:6.3f} {res['test'][metric]:6.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Task | train | test\n",
      " FCLAIM:  0.678  0.660\n",
      "   VMWE:  0.768  0.735\n",
      " OL19-C:  0.736  0.675\n",
      " ABSD-2:  0.650  0.580\n",
      "  MIO-P:  0.829  0.824\n",
      "  ARCHI:  0.603  0.477\n",
      "   LSDC:  0.408  0.404\n"
     ]
    }
   ],
   "source": [
    "metric = 'acc-balanced'  # 'acc', 'acc-balanced'\n",
    "print('  Task | train | test')\n",
    "for res in results:\n",
    "    print(f\"{res['task']:>7s}: {res['train'][metric]:6.3f} {res['test'][metric]:6.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Task | train | test\n",
      " FCLAIM:  0.671  0.661\n",
      "   VMWE:  0.857  0.833\n",
      " OL19-C:  0.770  0.757\n",
      " ABSD-2:  0.648  0.643\n",
      "  MIO-P:  0.793  0.789\n",
      "  ARCHI:  0.604  0.477\n",
      "   LSDC:  0.712  0.568\n"
     ]
    }
   ],
   "source": [
    "metric = 'f1'  # 'f1', 'f1-balanced'\n",
    "print('  Task | train | test')\n",
    "for res in results:\n",
    "    print(f\"{res['task']:>7s}: {res['train'][metric]:6.3f} {res['test'][metric]:6.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 573,
     "status": "aborted",
     "timestamp": 1650389151745,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "-9pvzgqmyPfj"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Task | train | test\n",
      " FCLAIM:  0.658  0.643\n",
      "   VMWE:  0.758  0.725\n",
      " OL19-C:  0.650  0.623\n",
      " ABSD-2:  0.559  0.515\n",
      "  MIO-P:  0.729  0.725\n",
      "  ARCHI:  0.603  0.472\n",
      "   LSDC:  0.357  0.334\n"
     ]
    }
   ],
   "source": [
    "metric = 'f1-balanced'  # 'f1', 'f1-balanced'\n",
    "print('  Task | train | test')\n",
    "for res in results:\n",
    "    print(f\"{res['task']:>7s}: {res['train'][metric]:6.3f} {res['test'][metric]:6.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "executionInfo": {
     "elapsed": 573,
     "status": "aborted",
     "timestamp": 1650389151747,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "rg_m8DuZycuN"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FCLAIM 944\n",
      "{'0': '630', '1': '314'}\n",
      "{'1': '418', '0': '526'}\n",
      "VMWE 1447\n",
      "{'1': '256', '0': '1191'}\n",
      "{'1': '283', '0': '1164'}\n",
      "OL19-C 930\n",
      "{'0': '796', '1': '134'}\n",
      "{'0': '688', '1': '242'}\n",
      "ABSD-2 2555\n",
      "{'1': '1670', '0': '780', '2': '105'}\n",
      "{'0': '989', '1': '1266', '2': '300'}\n",
      "MIO-P 4668\n",
      "{'1': '812', '0': '3856'}\n",
      "{'1': '1598', '0': '3070'}\n",
      "ARCHI 4743\n",
      "{'2': '1177', '3': '1176', '1': '1199', '0': '1191'}\n",
      "{'2': '1395', '1': '1414', '3': '810', '0': '1124'}\n",
      "LSDC 8602\n",
      "{'13': '280', '6': '346', '12': '940', '3': '925', '2': '944', '1': '934', '11': '931', '0': '453', '10': '511', '5': '924', '4': '65', '8': '923', '9': '83', '7': '343'}\n",
      "{'8': '1159', '11': '990', '12': '1235', '5': '1231', '3': '1779', '0': '891', '1': '1316', '7': '1'}\n"
     ]
    }
   ],
   "source": [
    "# class label distributions (inference)\n",
    "for res in results:\n",
    "    print(res['task'], res['test']['num'])\n",
    "    print(res['test']['distr-test'])\n",
    "    print(res['test']['distr-pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 574,
     "status": "aborted",
     "timestamp": 1650389151749,
     "user": {
      "displayName": "Ulf Hamster",
      "userId": "17569698098126794188"
     },
     "user_tz": -120
    },
    "id": "MlJCwluYXivN"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPXbjGyus9cZkRag231IL3s",
   "collapsed_sections": [],
   "name": "FastText example (SSH problems).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}