exposure_calc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "406cf3a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_filing(file):\n",
    "    fileA = open(\"filing.txt\",'r')\n",
    "    fileX = open(file,'r')\n",
    "    s = fileX.readlines()\n",
    "    l = \"\"\n",
    "    for ele in s:\n",
    "        l +=ele.replace(\"\\n\",\"\")\n",
    "    lst = l.split()\n",
    "    # print(lst)\n",
    "    s = fileA.readlines()\n",
    "    l = \"\"\n",
    "    for ele in s:\n",
    "        l +=(ele.replace(\"\\n\",\"\"))+\" \"\n",
    "    fr = l.split(\".\")\n",
    "    ans = []\n",
    "    for ele in fr:\n",
    "        ele = ele.strip()\n",
    "        if any(word in ele for word in lst):\n",
    "            ans.append(ele)\n",
    "    fileA.close()\n",
    "    fileX.close()\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d387578",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write(file,data):\n",
    "    f = open(file,\"w\")\n",
    "    for ele in data:\n",
    "        f.write(ele+\".\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6229e868",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = [\"theme.txt\"]\n",
    "for ele in l:\n",
    "    ans = get_filing(ele)\n",
    "    write(\"uni\"+ele,ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f442583",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2587c233",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('unitheme.txt', 'r') as file:\n",
    "    dataA = file.read().replace('\\n', '')    \n",
    "with open('theme.txt', 'r') as file:\n",
    "    dataB = file.read().replace('\\n', '')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "604d0d68",
   "metadata": {},
   "outputs": [],
   "source": [
    "bagOfWordsA = dataA.split()\n",
    "bagOfWordsB = dataB.split()\n",
    "uniqueWords = set(bagOfWordsA).union(set(bagOfWordsB))\n",
    "numOfWordsA = dict.fromkeys(uniqueWords, 0)\n",
    "for word in bagOfWordsA:\n",
    "    numOfWordsA[word] += 1\n",
    "numOfWordsB = dict.fromkeys(uniqueWords, 0)\n",
    "for word in bagOfWordsB:\n",
    "    numOfWordsB[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c4e10079",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "043daa77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTF(wordDict, bagOfWords):\n",
    "    tfDict = {}\n",
    "    bagOfWordsCount = len(bagOfWords)\n",
    "    for word, count in wordDict.items():\n",
    "        tfDict[word] = count / float(bagOfWordsCount)\n",
    "    return tfDict    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5c10857d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfA = computeTF(numOfWordsA, bagOfWordsA)\n",
    "tfB = computeTF(numOfWordsB, bagOfWordsB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "500021f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeIDF(documents):\n",
    "    import math\n",
    "    N = len(documents)\n",
    "    \n",
    "    idfDict = dict.fromkeys(documents[0].keys(), 0)\n",
    "    for document in documents:\n",
    "        for word, val in document.items():\n",
    "            if val > 0:\n",
    "                idfDict[word] +=1\n",
    "                \n",
    "    for word, val in idfDict.items():\n",
    "        idfDict[word] = math.log(N / float(val))\n",
    "    return idfDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "72a70ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "idfs = computeIDF([numOfWordsA, numOfWordsB])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fb227b80",
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTFIDF(tfBagOfWords, idfs):\n",
    "    tfidf = {}\n",
    "    for word, val in tfBagOfWords.items():\n",
    "        tfidf[word] = val * idfs[word]\n",
    "    return tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "23c3aa63",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfA = computeTFIDF(tfA, idfs)\n",
    "tfidfB = computeTFIDF(tfB, idfs)\n",
    "df = pd.DataFrame([tfidfA, tfidfB])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f100afc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "listA = list(tfidfA.values())\n",
    "listB = list(tfidfB.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d47fda1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "csim = cosine_similarity([listA],[listB])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0be6759",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}