ngram_model_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['20110123-000.json',\n",
       " '20110123-002.json',\n",
       " '20110123-004.json',\n",
       " '20110123-006.json',\n",
       " '20110123-007.json',\n",
       " '20110123-009.json',\n",
       " '20110123-010.json',\n",
       " '20110123-012.json',\n",
       " '20110123-014.json',\n",
       " '20110123-015.json']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names= os.listdir(os.getcwd()+\"\\\\data\\\\\")\n",
    "names[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# named entity is generally absent in words\n",
    "#words = set(nltk.corpus.words.words())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Below code is used to merge all json files into one, so takes long time\n",
    "## Do not run the below blocks untill i say : "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus=[]\n",
    "for name in names:\n",
    "    for tweet in open(os.getcwd()+\"\\\\data\\\\\"+name, 'r'):\n",
    "        data=json.loads(tweet)\n",
    "        try:\n",
    "            corpus.append(data['text'])\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Chef salad is calling my name, I'm so hungry!\",\n",
       " 'Stadd',\n",
       " 'Ngapain lo online sih nif RT @hanifpeng: Iyaaaa gituuu RT @renyfebry: @hanifpeng oooooh gituuuuu',\n",
       " \"@Biebercrombie haha I'm on my iPod and I'm on the tumblr app and it's working fine now ;)\",\n",
       " '塾だ～英語の単語やばいなΩÅΩ;',\n",
       " 'Chamo hay ir yo como me rasco con una sola Promoción de 2 botellas de Ron ó vodka nacional X 350 bsf de 6:00pm a 10:00pm en @discoverybar',\n",
       " 'Photo: seco http://tumblr.com/xjd1bnfg4w',\n",
       " 'Jungle Abduction, Get Out Alive 22 Jan, 2011 http://bsstracing.com/tv-shows/get-out-alive/jungle-abduction-get-out-alive-22-jan-2011.html',\n",
       " 'Jam belapa cekalang? Akuh males mandi, nantian ajadah mandinyah .',\n",
       " '@Chalk_Flew_Up Yup. Maybe not for much longer.']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file= open(\"tweet_text.txt\",'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for item in corpus:\n",
    "    file.write(\"%s\\n\" %item.encode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords=['','(',')','{','}','\\\\','--',':']\n",
    "stopwords.append(nltk.corpus.stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Sun Jan 23 00:00:01 +0000 2011',\n",
       " 'favorited': False,\n",
       " 'geo': None,\n",
       " 'id': 28965135238303744,\n",
       " 'id_str': '28965135238303744',\n",
       " 'in_reply_to_screen_name': 'BeasBookNook',\n",
       " 'in_reply_to_status_id': 28962147899805696,\n",
       " 'in_reply_to_status_id_str': '28962147899805696',\n",
       " 'in_reply_to_user_id': 47618028,\n",
       " 'in_reply_to_user_id_str': '47618028',\n",
       " 'place': None,\n",
       " 'retweet_count': 0,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://www.tweetdeck.com\" rel=\"nofollow\">TweetDeck</a>',\n",
       " 'text': \"@BeasBookNook every ARC I've ever read hasn't had final copy edits. Isn't that the nature of the beast? I guess some are better than others.\",\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Tue Apr 03 18:05:30 +0000 2007',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': 'Truth be told, I think I thrive under a lack of accountability. /Michael Scott',\n",
       "  'favourites_count': 0,\n",
       "  'follow_request_sent': None,\n",
       "  'followers_count': 95,\n",
       "  'following': None,\n",
       "  'friends_count': 168,\n",
       "  'geo_enabled': False,\n",
       "  'id': 3345521,\n",
       "  'id_str': '3345521',\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 11,\n",
       "  'location': 'Vancouver Island, BC',\n",
       "  'name': 'Heller',\n",
       "  'notifications': None,\n",
       "  'profile_background_color': 'ffffff',\n",
       "  'profile_background_image_url': 'http://a0.twimg.com/profile_background_images/21939138/Roboto3_vectorized.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_image_url': 'http://a3.twimg.com/profile_images/142506464/robot_normal.jpg',\n",
       "  'profile_link_color': '000000',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '215ca2',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': True,\n",
       "  'protected': False,\n",
       "  'screen_name': 'Heller',\n",
       "  'show_all_inline_media': False,\n",
       "  'statuses_count': 2245,\n",
       "  'time_zone': 'Pacific Time (US & Canada)',\n",
       "  'url': None,\n",
       "  'utc_offset': -28800,\n",
       "  'verified': False}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets=[]\n",
    "for name in names[:10]:\n",
    "    for line in open(os.getcwd()+\"\\\\data\\\\\"+name, 'r'):\n",
    "        try:\n",
    "            tweets.append(json.loads(line))\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "tweets[14]        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'in_reply_to_status_id': None, 'text': \"Chef salad is calling my name, I'm so hungry!\", 'created_at': 'Sun Jan 23 00:00:00 +0000 2011', 'favorited': False, 'retweet_count': 0, 'source': '<a href=\"http://itunes.apple.com/us/app/twitter/id409789998?mt=12\" rel=\"nofollow\">Twitter for Mac</a>', 'in_reply_to_screen_name': None, 'in_reply_to_status_id_str': None, 'id_str': '28965131362770944', 'contributors': None, 'retweeted': False, 'in_reply_to_user_id_str': None, 'place': None, 'coordinates': None, 'geo': None, 'in_reply_to_user_id': None, 'truncated': False, 'user': {'listed_count': 27, 'notifications': None, 'created_at': 'Sat Mar 28 01:05:42 +0000 2009', 'profile_sidebar_border_color': 'fcfcfc', 'show_all_inline_media': True, 'following': None, 'geo_enabled': False, 'description': \"A career woman who's focusing on accomplishing my goals to the fullest!!! #ImpressiveAngel\", 'contributors_enabled': False, 'profile_use_background_image': True, 'profile_image_url': 'http://a3.twimg.com/profile_images/1337763812/Cheeeeese_normal.jpg', 'verified': False, 'profile_background_color': 'fcfcfc', 'default_profile_image': False, 'default_profile': False, 'statuses_count': 12660, 'profile_background_image_url': 'http://a0.twimg.com/profile_background_images/234555842/Superstar.jpg', 'screen_name': 'LovelyThang80', 'friends_count': 279, 'id_str': '27144739', 'profile_text_color': '050000', 'follow_request_sent': None, 'lang': 'en', 'profile_sidebar_fill_color': 'ffffff', 'followers_count': 320, 'protected': False, 'location': 'Virginia/ATL', 'profile_background_tile': True, 'favourites_count': 143, 'name': 'Miss ImpressiveAngel', 'url': None, 'id': 27144739, 'is_translator': False, 'time_zone': 'Eastern Time (US & Canada)', 'utc_offset': -18000, 'profile_link_color': '13f034'}, 'id': 28965131362770944}\n",
      "en\n"
     ]
    }
   ],
   "source": [
    "tag = tweets[0]\n",
    "print(tag)\n",
    "print(tag['user']['lang'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample= open(\"sample.txt\", \"w\")\n",
    "for tweet in tweets:\n",
    "    if(tweet['user']['lang'] != 'en'):\n",
    "        continue\n",
    "    result = re.sub(r\"(http|Http|https)\\S+\", \"\", tweet['text'])\n",
    "    result = re.sub(\"(@[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)\",\"\",result)\n",
    "    #result= re.sub(r\"(@\\S+)\",\"\", result)\n",
    "    \n",
    "    listing= nltk.wordpunct_tokenize(result)\n",
    "    strin=\"\"\n",
    "    for w in listing:\n",
    "        if(w.lower() not in stopwords):\n",
    "            strin= strin +\" \"+w\n",
    "    \n",
    "    if (len(strin) < 10):\n",
    "        continue\n",
    "        \n",
    "    #item= strin.encode(\"utf-8\")\n",
    "    sample.write(\"%s\\n\"%strin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_tweets= []\n",
    "for line in open(\"20110123-006.json\", 'r'):\n",
    "    try:\n",
    "        test_tweets.append(json.loads(line))\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test= open(\"test_data.txt\", \"w\")\n",
    "for tweet in test_tweets:\n",
    "    result = re.sub(r\"(http|Http|https)\\S+\", \"\", tweet['text'])\n",
    "    listing= nltk.wordpunct_tokenize(result)\n",
    "    strin=\"\"\n",
    "    for w in listing:\n",
    "        #w= w.encode('utf-8')\n",
    "        if(w.lower() in words and w.lower() not in stopwords):\n",
    "            strin= strin +\" \"+w\n",
    "    #item= strin.encode(\"utf-8\")\n",
    "    if (len(strin) < 10):\n",
    "        continue\n",
    "    test.write(\"%s\\n\"%strin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now trying to remove non english words like chinese and other stuff "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.corpus import brown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent= \"サッカー名言”『アッズーリが必要とするのは”労働者”であって、ファンタジスタではない。Jungle https://bsstracing.com/tv-shows/get-out-alive/jungle-abduction-get-out-alive-22-jan-2011.html Ngapain sih nif RT @hanifpeng: Iyaaaa gituuu RT @renyfebry: @hanifpeng http://bit.ly/h6HH3t gituuuuu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'サッカー名言”『アッズーリが必要とするのは”労働者”であって、ファンタジスタではない。Jungle  Ngapain sih nif RT @hanifpeng: Iyaaaa gituuu RT @renyfebry: @hanifpeng  gituuuuu'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = re.sub(r\"(http|Http|https)\\S+\", \"\", sent)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'サッカー名言”『アッズーリが必要とするのは”労働者”であって、ファンタジスタではない。Jungle  Ngapain sih nif RT  Iyaaaa gituuu RT    gituuuuu'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = re.sub(r\"(@\\S+)\", \"\", result)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1161192"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(brown.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testing= \"Jungle Abduction, Get Out Alive 22 Jan, 2011 http://bsstracing.com/tv-shows/get-out-alive/jungle-abduction-get-out-alive-22-jan-2011.html\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "listing= nltk.wordpunct_tokenize(testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Jungle',\n",
       " 'Abduction',\n",
       " ',',\n",
       " 'Get',\n",
       " 'Out',\n",
       " 'Alive',\n",
       " '22',\n",
       " 'Jan',\n",
       " ',',\n",
       " '2011',\n",
       " 'http',\n",
       " '://',\n",
       " 'bsstracing',\n",
       " '.',\n",
       " 'com',\n",
       " '/',\n",
       " 'tv',\n",
       " '-',\n",
       " 'shows',\n",
       " '/',\n",
       " 'get',\n",
       " '-',\n",
       " 'out',\n",
       " '-',\n",
       " 'alive',\n",
       " '/',\n",
       " 'jungle',\n",
       " '-',\n",
       " 'abduction',\n",
       " '-',\n",
       " 'get',\n",
       " '-',\n",
       " 'out',\n",
       " '-',\n",
       " 'alive',\n",
       " '-',\n",
       " '22',\n",
       " '-',\n",
       " 'jan',\n",
       " '-',\n",
       " '2011',\n",
       " '.',\n",
       " 'html']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "strin=\"\"\n",
    "\n",
    "for w in listing:\n",
    "    #w= w.encode('utf-8')\n",
    "    if(w.lower() in words and w.lower() not in stopwords):\n",
    "        strin= strin +\" \"+w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Jungle Abduction Get Out Alive get out alive jungle abduction get out alive'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "strin=\"!! this is my home, and that's yours' how    about doing all stuff     \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\" this is my home, and that's yours' how    about doing all stuff     \""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strin.strip(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making language model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.util import ngrams\n",
    "import string\n",
    "from nltk import SimpleGoodTuringProbDist\n",
    "from math import log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PlaintextCorpusReader in 'C:\\\\Users\\\\Sudhanshu\\\\Downloads\\\\twitter_bot'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<PlaintextCorpusReader in 'C:\\\\Users\\\\Sudhanshu\\\\Downloads\\\\twitter_bot'>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = nltk.corpus.reader.plaintext.PlaintextCorpusReader(\".\", \"sample.txt\")\n",
    "test = nltk.corpus.reader.plaintext.PlaintextCorpusReader(\".\", \"test_data.txt\")\n",
    "print(test)\n",
    "corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unigram perplexity with add 1 smoothing\\ Laplacian smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43784"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram= [w.strip(string.punctuation).lower() for w in corpus.words()]\n",
    "len(unigram)\n",
    "#unigram[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "spl= int(95*len(unigram)/100)\n",
    "train_corpus= unigram[:spl]\n",
    "test_corpus= unigram[spl:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "freq_1gram= nltk.FreqDist(train_corpus)\n",
    "len_train= len(train_corpus)\n",
    "vocab= len(set(train_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unigram_prob_with_add1smoothing(word):\n",
    "    return (freq_1gram[word]+1)/(len_train+vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def entropy(n, text):\n",
    "        e = 0.0\n",
    "        text = [\"<s>\"] + text + [\"</s>\"]\n",
    "        for i in range(n - 1, len(text)):\n",
    "            context = text[i - n + 1:i]\n",
    "            token = text[i]\n",
    "            #print(str(context)+\"    \"+token)\n",
    "            e += logprob(token, context)\n",
    "        return e / float(len(text) - (n - 1))\n",
    "\n",
    "\n",
    "def logprob(word, context):\n",
    "    if len(context)==0:\n",
    "        p=unigram_prob_with_add1smoothing(word)\n",
    "    elif len(context)==1:\n",
    "        p=bigram_prob_with_add1smoothing(context[0], word)\n",
    "    else:\n",
    "        p=trigram_prob_with_add1smoothing(context[0], context[1], word)\n",
    "    return -p*log(p , 2)\n",
    "\n",
    "\n",
    "def perplexity(n, text):\n",
    "      return pow(2.0, entropy(n, text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.024138089875898"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(1, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.025133170322091"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(1, train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bigram Perplexity with add 1 smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43785"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram= [a for a in ngrams(unigram, 2, pad_left=True,pad_right=True,left_pad_symbol='<s>', right_pad_symbol=\"</s>\")]\n",
    "len(bigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cfreq_2gram = nltk.ConditionalFreqDist(bigram)\n",
    "cprob_2gram = nltk.ConditionalProbDist(cfreq_2gram, nltk.MLEProbDist)\n",
    "def bigram_prob_with_add1smoothing(word1, word2):\n",
    "    cprob_2gram_add1=float((((1+cfreq_2gram[word1][word2])/(len(cfreq_2gram)+sum(cfreq_2gram[word1].values())))))\n",
    "    return cprob_2gram_add1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0081485615099057"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(2, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0085341354101802"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(2, train_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bigram_prob_with_gtsmoothing(word1, word2):\n",
    "    return SimpleGoodTuringProbDist(word1, word2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def entropy(n, text):\n",
    "        e = 0.0\n",
    "        text = [\"<s>\"] + text + [\"</s>\"]\n",
    "        for i in range(n - 1, len(text)):\n",
    "            context = text[i - n + 1:i]\n",
    "            token = text[i]\n",
    "            #print(str(context)+\"    \"+token)\n",
    "            e += logprob_gt(token, context)\n",
    "        return e / float(len(text) - (n - 1))\n",
    "\n",
    "\n",
    "def logprob_gt(word, context):\n",
    "    if len(context)==0:\n",
    "        p=unigram_prob_with_gtsmoothing(word)\n",
    "    elif len(context)==1:\n",
    "        p=bigram_prob_with_gtsmoothing(context[0], word)\n",
    "    else:\n",
    "        p=trigram_prob_with_gtsmoothing(context[0], context[1], word)\n",
    "    return -p*log(p , 2)\n",
    "\n",
    "\n",
    "def perplexity(n, text):\n",
    "      return pow(2.0, entropy(n, text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trigram perplexity with add 1 smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43786"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigram= [a for a in ngrams(unigram, 3, pad_left= True, pad_right=True, left_pad_symbol= \"<s>\", right_pad_symbol=\"</s>\")]\n",
    "len(trigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trigrams_as_bigrams=[]\n",
    "trigrams_as_bigrams.extend([((t[0],t[1]), t[2]) for t in trigram])\n",
    "cfreq_3gram = nltk.ConditionalFreqDist(trigrams_as_bigrams)\n",
    "cprob_3gram = nltk.ConditionalProbDist(cfreq_3gram, nltk.MLEProbDist)\n",
    "def trigram_prob_with_add1smoothing(w1, w2, w3):\n",
    "    cprob_3gram_add1=float((((1+cfreq_3gram[(w1,w2)][w3])/(len(cfreq_3gram)+sum(cfreq_3gram[(w1,w2)].values())))))\n",
    "    return cprob_3gram_add1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0007427952147212"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(3, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.000768607503031"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(3, train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying to generate text from corpus:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter, OrderedDict\n",
    "from operator import itemgetter\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_unigram = [w.strip(string.punctuation).lower() for w in test.words()]\n",
    "\n",
    "test_bigram= [a for a in ngrams(test_unigram, 2)]\n",
    "len(test_bigram)\n",
    "\n",
    "test_trigram= [a for a in ngrams(test_unigram, 3)]\n",
    "len(test_trigram)\n",
    "\n",
    "test_frequency= nltk.FreqDist(test_bigram)\n",
    "test_sorted_freq= OrderedDict(reversed(sorted(test_frequency.items(), key= itemgetter(1))))\n",
    "\n",
    "test_cfreq_2gram= nltk.ConditionalFreqDist(test_bigram)\n",
    "\n",
    "test_cprob_2gram= nltk.ConditionalProbDist(test_cfreq_2gram, nltk.MLEProbDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "counts = Counter(corpus.words())\n",
    "total_count = len(corpus.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 1173),\n",
       " ('a', 1152),\n",
       " ('to', 940),\n",
       " ('the', 913),\n",
       " ('you', 652),\n",
       " ('de', 560),\n",
       " ('me', 537),\n",
       " ('in', 463),\n",
       " ('is', 427),\n",
       " ('s', 420),\n",
       " ('it', 415),\n",
       " ('my', 413),\n",
       " ('and', 407),\n",
       " ('of', 374),\n",
       " ('on', 373),\n",
       " ('no', 348),\n",
       " ('for', 344),\n",
       " ('t', 305),\n",
       " ('that', 291),\n",
       " ('o', 290)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts.most_common(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for word in counts:\n",
    "    counts[word] /= float(total_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "para do Tell slap SO turn N Just NO NOW I a on After live de A mention going period patience would tau some en school So virtually much conceited would para Store far gave look store on being bit Leaves home They Pouch D el the saved twitter S se Thanks whipped THING the las Improve better Your He malo take if GANE ne well around Opening for R Thanks to would is lo What chamar how NO party in one who m So q just around m helping morning han de urgence look them Ultimate no should how\n"
     ]
    }
   ],
   "source": [
    "# generating 100 words at random of the language\n",
    "text=[]\n",
    "for _ in range(100):\n",
    "    r = random.random()\n",
    "    accumulator = .0\n",
    " \n",
    "    for word, freq in counts.items():\n",
    "        accumulator += freq\n",
    " \n",
    "        if accumulator >= r:\n",
    "            text.append(word)\n",
    "            break\n",
    "print(' '.join(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation by Perplexity and Cross-Entropy :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def entropy(n, text):\n",
    "    #here n is ngram order, text is list of ngrams\n",
    "    e=0.0\n",
    "    text= [\"<s>\"]+text+[\"</s>\"]\n",
    "    for i in range(n-1, len(text)):\n",
    "        context= text[i-n+1:i]\n",
    "        token= text[i]\n",
    "        e += logprob(token, context)\n",
    "    return e/float(len(text) - (n-1))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logprob(word, context):\n",
    "    if (len(context) == 0):\n",
    "        p=unigram_prob(word)\n",
    "    elif(len(context) == 1):\n",
    "        p= bigram_prob_with_add1smoothing(context[0], word)\n",
    "    else:\n",
    "        p= trigram_prob_with_add1smoothing(context[0], context[1], word)\n",
    "    return -p*log(p,2)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def perplexity(n, text):\n",
    "    return pow(2.0, entropy(n, text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bigram_prob_with_add1smoothing(word1, word2):\n",
    "    cprob_2gram_add1=float((((1+cfreq_2gram[word1][word2])/(len(cfreq_2gram)+sum(cfreq_2gram[word1].values())))))\n",
    "    return cprob_2gram_add1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trigram_prob_with_add1smoothing(w1, w2, w3):\n",
    "    cprob_3gram_add1=float((((1+cfreq_3gram[(w1,w2)][w3])/(len(cfreq_3gram)+sum(cfreq_3gram[(w1,w2)].values())))))\n",
    "    return cprob_3gram_add1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from math import log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00099976292282454"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy_value_2gram= entropy(2, test_bigram)\n",
    "entropy_value_2gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.000368608494961"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#bigram perplexity with Good-Turing smoothing( or add 1 smoothing )\n",
    "per_value_2gram= perplexity(2, test_bigram)\n",
    "per_value_2gram"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}