Wikidata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STARTING\n",
      "START PARSING GRAPH\n",
      "DONE PARSING GRAPH\n",
      "START SERIALIZING GRAPH\n",
      "_:ub3bL6C13 <http://example.org/stuff/1.0/fullname> \"Dave Beckett\" .\n",
      "<http://www.w3.org/TR/rdf-syntax-grammar> <http://purl.org/dc/elements/1.1/title> \"RDF/XML Syntax Specification (Revised)\" .\n",
      "_:ub3bL6C13 <http://example.org/stuff/1.0/homePage> <http://purl.org/net/dajobe/> .\n",
      "<http://www.w3.org/TR/rdf-syntax-grammar> <http://example.org/stuff/1.0/editor> _:ub3bL6C13 .\n",
      "\n",
      "\n",
      "DONE SERIALIZING GRAPH\n",
      "EXECUTION TIME: 0.020387172699 s\n"
     ]
    }
   ],
   "source": [
    "# read, parse and serialize graph to nt format\n",
    "import time\n",
    "import rdflib\n",
    "from rdflib.graph import Graph, ConjunctiveGraph\n",
    "import rdflib.namespace\n",
    "from rdflib.namespace import OWL, RDF, RDFS\n",
    "from rdflib import Namespace, URIRef, Literal\n",
    "\n",
    "print ('STARTING')\n",
    "start = time.time()\n",
    "\n",
    "s = '../../../../Volumes/Public/SeminarPaper/example1.ttl'\n",
    "g = rdflib.ConjunctiveGraph()\n",
    "print ('START PARSING GRAPH')\n",
    "g = g.parse(s, format=\"n3\")\n",
    "print ('DONE PARSING GRAPH')\n",
    "print ('START SERIALIZING GRAPH')\n",
    "#g.serialize(destination='Wikidata/test-30.nt', format='nt')\n",
    "print g.serialize(format='nt')\n",
    "print ('DONE SERIALIZING GRAPH')\n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "Wikidata/wikidata-20160801-all-BETA.ttl has 1933695292 lines\n",
      "EXECUTION TIME: 403.305052042 s\n"
     ]
    }
   ],
   "source": [
    "# get file lines\n",
    "import time\n",
    "\n",
    "file = 'Wikidata/wikidata-20160801-all-BETA.ttl'\n",
    "start = time.time()\n",
    "print('START')\n",
    "\n",
    "# from http://stackoverflow.com/questions/845058/how-to-get-line-count-cheaply-in-python\n",
    "num_lines = sum(1 for line in open(file))\n",
    "print ('{} has {} lines'.format(file, num_lines))\n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "EXECUTION TIME: 0.00662589073181 s\n"
     ]
    }
   ],
   "source": [
    "# get remove invalid escape `\\a' at line 544253967\n",
    "import time\n",
    "\n",
    "file = '/Volumes/Samsung/wikidata-20160801-all-BETA.ttl'\n",
    "start = time.time()\n",
    "print('START')\n",
    "counter = 0\n",
    "#with open(file) as f:\n",
    "#    for i, line in enumerate(f):\n",
    "#        if (i > 544253960):\n",
    "#            print i\n",
    "#            print line\n",
    "#            if '\\a' in line:\n",
    "#                print ('found \\a')\n",
    "#        if (i > 544253970):\n",
    "#            break            \n",
    "    \n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))\n",
    "#\n",
    "#output\n",
    "#544253966\n",
    "#\twdt:P487 \"\\a\" ;\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "0 lines found containing \u0007\n",
      "set([])\n",
      "EXECUTION TIME: 0.000699996948242 s\n"
     ]
    }
   ],
   "source": [
    "# delete lines with escape character \\a\n",
    "import time\n",
    "\n",
    "oldfile = '/Volumes/Samsung/wikidata-20160801-all-BETA.ttl'\n",
    "newFile = 'Wikidata/wikidata-20160801-all-BETA_v2.ttl'\n",
    "\n",
    "deletedLines = set()\n",
    "lineCounter = 0\n",
    "start = time.time()\n",
    "print('START')\n",
    "\n",
    "#f = open(oldfile, 'r')\n",
    "#fn = open(newFile, 'w')\n",
    "#lines = f.readlines()\n",
    "#for line in lines:\n",
    "#    if '\\a' not in line:\n",
    "#        fn.write(line)\n",
    "#    else:\n",
    "#        deletedLines.add(line)\n",
    "#        print ('line deleted')\n",
    "#    lineCounter += 1\n",
    "#    if (lineCounter % 100000000 == 0):\n",
    "#        print ('{} lines read'.format(lineCounter))\n",
    "#f.close()\n",
    "#fn.close()\n",
    "\n",
    "print ('{} lines found containing \\a'.format(len(deletedLines)))\n",
    "print deletedLines\n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "1000000 lines processed\n",
      "DONE\n",
      "EXECUTION TIME: 6.44111514091 s\n"
     ]
    }
   ],
   "source": [
    "# create sample\n",
    "import os\n",
    "import string\n",
    "import itertools\n",
    "import time\n",
    "\n",
    "#sampleFile = '../../../../Volumes/Public/SeminarPaper/wikidata-20160801-all-BETA_sample.ttl'\n",
    "#sampleFile = 'Wikidata/wikidata-20160801-all-BETA_sample_1m.ttl'\n",
    "sampleFile = '/Volumes/Samsung/wikidata_sample_from_nt.nt'\n",
    "#readFile = '../../../../Volumes/Public/SeminarPaper/wikidata-20160801-all-BETA.ttl'\n",
    "#readFile = 'Wikidata/wikidata-20160801-all-BETA.ttl'\n",
    "readFile = '/Volumes/Samsung//wikidata.nt'\n",
    "\n",
    "\n",
    "sampleSize = 1000000\n",
    "start = time.time()\n",
    "print('START')\n",
    "try:\n",
    "    # read file\n",
    "    f = open(readFile, 'r')\n",
    "    # write file\n",
    "    fw = open(sampleFile, 'w')\n",
    "    lineCounter = 0\n",
    "    for line in f:\n",
    "        if (lineCounter < sampleSize):\n",
    "            fw.write(line)\n",
    "            lineCounter += 1\n",
    "        else:\n",
    "            print ('{} lines processed'.format(sampleSize))\n",
    "            break\n",
    "    f.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "598 classes found\n",
      "[('<http://www.wikidata.org/prop/novalue/P37>', 8), ('<http://www.wikidata.org/prop/novalue/P102>', 4), ('<http://www.wikidata.org/prop/novalue/P155>', 1), ('<http://www.wikidata.org/prop/novalue/P2555>', 1)]\n",
      "DONE\n",
      "EXECUTION TIME: 7.11636400223 s\n"
     ]
    }
   ],
   "source": [
    "# get top 10 classes\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "readFile = 'Wikidata/wikidata_sample_1m.nt'\n",
    "#readFile = '../../../../Volumes/Public/SeminarPaper/wikidata-20160801-all-BETA.ttl'\n",
    "\n",
    "classCount = {} #key:class, value:count\n",
    "\n",
    "lineProgress = 1000000\n",
    "\n",
    "# ADAPT TO WIKIDATA\n",
    "rdfType = '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>'\n",
    "owlClass = '<http://www.w3.org/2002/07/owl#Class>'\n",
    "\n",
    "# <http://www.wikidata.org/prop/novalue/P16> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class> .\n",
    "\n",
    "\n",
    "allClasses = set()\n",
    "seenClasses = set()\n",
    "classCountDic = {} #key:class, value:count\n",
    "\n",
    "def getSPO(splittedLine):\n",
    "    word_position = 0\n",
    "    for word in splittedLine:\n",
    "        if (word_position == 0):\n",
    "            subj = word\n",
    "        elif (word_position == 1):\n",
    "            pred = word\n",
    "        elif (word_position == 2):\n",
    "            obj = word\n",
    "        else:\n",
    "            return subj, pred, obj\n",
    "        word_position += 1\n",
    "    return subj, pred, obj\n",
    "\n",
    "def addToClasses(s):\n",
    "    if s not in allClasses:\n",
    "        allClasses.add(s)\n",
    "        \n",
    "def countClasses(o):\n",
    "    #print ('countClasses')\n",
    "    if o in seenClasses:\n",
    "        classCountDic[o] += 1\n",
    "    else:\n",
    "        classCountDic[o] = 1\n",
    "        seenClasses.add(o)\n",
    "        \n",
    "print('START')\n",
    "try:\n",
    "    f = open(readFile, 'r')\n",
    "    lineCounter = 0\n",
    "    for line in f:\n",
    "        splittedLine = line.rstrip('\\n').split()\n",
    "        s, p, o = getSPO(splittedLine)\n",
    "        if (p == rdfType and o == owlClass):\n",
    "            #print ('{} {} {}'.format(s,p,o))\n",
    "            addToClasses(s)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "        #if (lineCounter > 100):\n",
    "        #    break\n",
    "    f.close()\n",
    "    print ('{} classes found'.format(len(allClasses)))\n",
    "    f = open(readFile, 'r')\n",
    "    lineCounter = 0\n",
    "    for line in f:\n",
    "        splittedLine = line.rstrip('\\n').split()\n",
    "        s, p, o = getSPO(splittedLine)\n",
    "        if (p == rdfType and o in allClasses):\n",
    "            #print ('{}'.format(o))\n",
    "            countClasses(o)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    #print allClasses\n",
    "    #print classCountDic\n",
    "    top10dic = dict(sorted(classCountDic.items(), key=operator.itemgetter(1), reverse=True)[:10])\n",
    "    print (sorted(top10dic.items(), key=operator.itemgetter(1), reverse=True))\n",
    "    f.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "print ('EXECUTION TIME: {} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STARTING\n",
      "set(['<http://www.wikidata.org/entity/Q4167410>', '<http://www.wikidata.org/entity/Q5>', '<http://www.wikidata.org/entity/Q13406463>', '<http://www.wikidata.org/entity/Q532>', '<http://www.wikidata.org/entity/Q79007>', '<http://www.wikidata.org/entity/Q16521>', '<http://www.wikidata.org/entity/Q13100073>', '<http://www.wikidata.org/entity/Q486972>', '<http://www.wikidata.org/entity/Q4167836>', '<http://www.wikidata.org/entity/Q11266439>'])\n",
      "DONE\n",
      "INDEGREE RESULT:\n",
      "[('<http://www.wikidata.org/entity/Q5>', 265), ('<http://www.wikidata.org/entity/Q16521>', 44), ('<http://www.wikidata.org/entity/Q4167836>', 22), ('<http://www.wikidata.org/entity/Q4167410>', 12), ('<http://www.wikidata.org/entity/Q13406463>', 10), ('<http://www.wikidata.org/entity/Q532>', 8), ('<http://www.wikidata.org/entity/Q79007>', 4)]\n",
      "OUTDEGREE RESULT\n",
      "[]\n",
      "EXECUTION TIME: 7.01688504219 s\n"
     ]
    }
   ],
   "source": [
    "# calculate class indegree & outdegree\n",
    "\n",
    "import operator\n",
    "import time\n",
    "\n",
    "print ('STARTING')\n",
    "start = time.time()\n",
    "\n",
    "readFile = '/Volumes/Samsung/wikidata_sample_from_nt.nt'\n",
    "#readFile = '/Volumes/Samsung/wikidata.nt'\n",
    "\n",
    "indegreeDic = {} # key:class, value: indegree\n",
    "outdegreeDic = {} #key: class, value: outdegree\n",
    "\n",
    "lineProgress = 10000000\n",
    "\n",
    "top10set = set()\n",
    "top10set.update(['<http://www.wikidata.org/entity/Q5>', '<http://www.wikidata.org/entity/Q4167836>', '<http://www.wikidata.org/entity/Q16521>', '<http://www.wikidata.org/entity/Q4167410>', '<http://www.wikidata.org/entity/Q11266439>', '<http://www.wikidata.org/entity/Q13100073>', '<http://www.wikidata.org/entity/Q486972>', '<http://www.wikidata.org/entity/Q532>', '<http://www.wikidata.org/entity/Q79007>', '<http://www.wikidata.org/entity/Q13406463>'])\n",
    "\n",
    "def getSPO(splittedLine):\n",
    "    word_position = 0\n",
    "    for word in splittedLine:\n",
    "        if (word_position == 0):\n",
    "            subj = word\n",
    "        elif (word_position == 1):\n",
    "            pred = word\n",
    "        elif (word_position == 2):\n",
    "            obj = word\n",
    "        else:\n",
    "            return subj, pred, obj\n",
    "        word_position += 1\n",
    "\n",
    "try:\n",
    "    f = open(readFile, 'r')\n",
    "    lineCounter = 0\n",
    "    for line in f:\n",
    "        splittedLine = line.rstrip('\\n').split()\n",
    "        s, p, o = getSPO(splittedLine)\n",
    "        if s in top10set:\n",
    "            if outdegreeDic.has_key(s):\n",
    "                outdegreeDic[s] += 1\n",
    "            else:\n",
    "                outdegreeDic[s] = 1\n",
    "        if o in top10set:\n",
    "            if indegreeDic.has_key(o):\n",
    "                indegreeDic[o] += 1\n",
    "            else:\n",
    "                indegreeDic[o] = 1\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    f.close()\n",
    "    print ('DONE')\n",
    "    print ('INDEGREE RESULT:')\n",
    "    print sorted(indegreeDic.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    print ('OUTDEGREE RESULT')\n",
    "    print sorted(outdegreeDic.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    print ('EXECUTION TIME: {} s'.format(time.time()-start))\n",
    "except:\n",
    "    print ('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "START COUNTING INSTANCES\n",
      "10000000 lines read\n",
      "20000000 lines read\n",
      "30000000 lines read\n",
      "40000000 lines read\n",
      "50000000 lines read\n",
      "60000000 lines read\n",
      "70000000 lines read\n",
      "80000000 lines read\n",
      "90000000 lines read\n",
      "100000000 lines read\n",
      "110000000 lines read\n",
      "120000000 lines read\n",
      "130000000 lines read\n",
      "140000000 lines read\n",
      "150000000 lines read\n",
      "160000000 lines read\n",
      "170000000 lines read\n",
      "180000000 lines read\n",
      "190000000 lines read\n",
      "200000000 lines read\n",
      "210000000 lines read\n",
      "220000000 lines read\n",
      "230000000 lines read\n",
      "240000000 lines read\n",
      "250000000 lines read\n",
      "260000000 lines read\n",
      "270000000 lines read\n",
      "280000000 lines read\n",
      "290000000 lines read\n",
      "300000000 lines read\n",
      "310000000 lines read\n",
      "320000000 lines read\n",
      "330000000 lines read\n",
      "340000000 lines read\n",
      "350000000 lines read\n",
      "360000000 lines read\n",
      "370000000 lines read\n",
      "380000000 lines read\n",
      "390000000 lines read\n",
      "400000000 lines read\n",
      "410000000 lines read\n",
      "420000000 lines read\n",
      "430000000 lines read\n",
      "440000000 lines read\n",
      "450000000 lines read\n",
      "460000000 lines read\n",
      "470000000 lines read\n",
      "480000000 lines read\n",
      "490000000 lines read\n",
      "500000000 lines read\n",
      "510000000 lines read\n",
      "520000000 lines read\n",
      "530000000 lines read\n",
      "540000000 lines read\n",
      "550000000 lines read\n",
      "560000000 lines read\n",
      "570000000 lines read\n",
      "580000000 lines read\n",
      "590000000 lines read\n",
      "600000000 lines read\n",
      "610000000 lines read\n",
      "620000000 lines read\n",
      "630000000 lines read\n",
      "640000000 lines read\n",
      "650000000 lines read\n",
      "660000000 lines read\n",
      "670000000 lines read\n",
      "680000000 lines read\n",
      "690000000 lines read\n",
      "700000000 lines read\n",
      "710000000 lines read\n",
      "720000000 lines read\n",
      "730000000 lines read\n",
      "740000000 lines read\n",
      "750000000 lines read\n",
      "760000000 lines read\n",
      "770000000 lines read\n",
      "780000000 lines read\n",
      "790000000 lines read\n",
      "800000000 lines read\n",
      "810000000 lines read\n",
      "820000000 lines read\n",
      "830000000 lines read\n",
      "840000000 lines read\n",
      "850000000 lines read\n",
      "860000000 lines read\n",
      "870000000 lines read\n",
      "880000000 lines read\n",
      "890000000 lines read\n",
      "900000000 lines read\n",
      "910000000 lines read\n",
      "920000000 lines read\n",
      "930000000 lines read\n",
      "940000000 lines read\n",
      "950000000 lines read\n",
      "960000000 lines read\n",
      "970000000 lines read\n",
      "980000000 lines read\n",
      "990000000 lines read\n",
      "1000000000 lines read\n",
      "1010000000 lines read\n",
      "1020000000 lines read\n",
      "1030000000 lines read\n",
      "1040000000 lines read\n",
      "1050000000 lines read\n",
      "1060000000 lines read\n",
      "1070000000 lines read\n",
      "1080000000 lines read\n",
      "1090000000 lines read\n",
      "1100000000 lines read\n",
      "1110000000 lines read\n",
      "1120000000 lines read\n",
      "1130000000 lines read\n",
      "1140000000 lines read\n",
      "1150000000 lines read\n",
      "1160000000 lines read\n",
      "1170000000 lines read\n",
      "1180000000 lines read\n",
      "1190000000 lines read\n",
      "1200000000 lines read\n",
      "1210000000 lines read\n",
      "1220000000 lines read\n",
      "1230000000 lines read\n",
      "1240000000 lines read\n",
      "1250000000 lines read\n",
      "1260000000 lines read\n",
      "1270000000 lines read\n",
      "1280000000 lines read\n",
      "1290000000 lines read\n",
      "1300000000 lines read\n",
      "1310000000 lines read\n",
      "1320000000 lines read\n",
      "1330000000 lines read\n",
      "1340000000 lines read\n",
      "1350000000 lines read\n",
      "1360000000 lines read\n",
      "1370000000 lines read\n",
      "1380000000 lines read\n",
      "1390000000 lines read\n",
      "1400000000 lines read\n",
      "1410000000 lines read\n",
      "1420000000 lines read\n",
      "1430000000 lines read\n",
      "1440000000 lines read\n",
      "1450000000 lines read\n",
      "1460000000 lines read\n",
      "1470000000 lines read\n",
      "1480000000 lines read\n",
      "1490000000 lines read\n",
      "1500000000 lines read\n",
      "1510000000 lines read\n",
      "1520000000 lines read\n",
      "1530000000 lines read\n",
      "1540000000 lines read\n",
      "1550000000 lines read\n",
      "1560000000 lines read\n",
      "1570000000 lines read\n",
      "1580000000 lines read\n",
      "1590000000 lines read\n",
      "1600000000 lines read\n",
      "1610000000 lines read\n",
      "1620000000 lines read\n",
      "1630000000 lines read\n",
      "DONE COUNTING INSTANCES\n",
      "iSet1, #instances: 0\n",
      "iSet2, #instances: 0\n",
      "START COUNTING INSTANCE DEGREES\n",
      "10000000 lines read\n",
      "20000000 lines read\n",
      "30000000 lines read\n",
      "40000000 lines read\n",
      "50000000 lines read\n",
      "60000000 lines read\n",
      "70000000 lines read\n",
      "80000000 lines read\n",
      "90000000 lines read\n",
      "100000000 lines read\n",
      "110000000 lines read\n",
      "120000000 lines read\n",
      "130000000 lines read\n",
      "140000000 lines read\n",
      "150000000 lines read\n",
      "160000000 lines read\n",
      "170000000 lines read\n",
      "180000000 lines read\n",
      "190000000 lines read\n",
      "200000000 lines read\n",
      "210000000 lines read\n",
      "220000000 lines read\n",
      "230000000 lines read\n",
      "240000000 lines read\n",
      "250000000 lines read\n",
      "ERROR\n"
     ]
    }
   ],
   "source": [
    "# class instances indegree & outdegree\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "import numpy\n",
    "\n",
    "start = time.time()\n",
    "readFile = '/Volumes/Samsung/wikidata.nt'\n",
    "rdfType = '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>'\n",
    "lineProgress = 10000000\n",
    "\n",
    "\n",
    "top10set = set()\n",
    "top10set.update(['<http://www.wikidata.org/entity/Q215627>', '<http://www.wikidata.org/entity/Q386724>'])\n",
    "        #'<http://www.wikidata.org/entity/Q5>', '<http://www.wikidata.org/entity/Q4167836>', '<http://www.wikidata.org/entity/Q16521>', '<http://www.wikidata.org/entity/Q4167410>', '<http://www.wikidata.org/entity/Q11266439>', '<http://www.wikidata.org/entity/Q13100073>', '<http://www.wikidata.org/entity/Q486972>', '<http://www.wikidata.org/entity/Q532>', '<http://www.wikidata.org/entity/Q79007>', '<http://www.wikidata.org/entity/Q13406463>'])\n",
    "\n",
    "\n",
    "class ClassInstances:\n",
    "    def __init__(self, name, uri):\n",
    "        self.name = name\n",
    "        self.uri = uri\n",
    "        self.allClassInstances = set()\n",
    "        self.countDict = dict()\n",
    "        self.min = 9999999\n",
    "        self.max = 0\n",
    "        self.avg = 0.0\n",
    "        self.median = 0.0\n",
    "    def getURI(self):\n",
    "        return self.uri\n",
    "    def getClassInstances(self):\n",
    "        return len(self.allClassInstances)\n",
    "    def addInstance(self, instance):\n",
    "        if instance in self.allClassInstances:\n",
    "            self.countDict[instance] += 1\n",
    "        else:\n",
    "            self.countDict[instance] = 1\n",
    "            self.allClassInstances.add(instance)\n",
    "    def calculateDegrees(self):\n",
    "        allValueList = []\n",
    "        for k,v in self.countDict.iteritems():\n",
    "            allValueList.append(v)\n",
    "            if (v < self.min):\n",
    "                self.min = v\n",
    "            if (v > self.max):\n",
    "                self.max = v\n",
    "            self.avg += v\n",
    "        if (len(self.countDict) != 0):\n",
    "            self.avg = self.avg / len(self.countDict)\n",
    "        if (len(allValueList) != 0):\n",
    "            self.median = numpy.median(numpy.array(allValueList))\n",
    "    def printResults(self):\n",
    "        print ('{}'.format(self.name))\n",
    "        print ('min: {}, avg: {}, median: {}, max: {}'.format(self.min, self.avg, self.median, self.max))\n",
    "        \n",
    "# instances sets\n",
    "iAllSet = set() #allInstances = set()\n",
    "iSet1 = set() #instancesObject = set()\n",
    "iSet2 = set() #instancesObjectNoOverlap = set()\n",
    "#iSet3 = set() #instancesNoun = set()\n",
    "#iSet4 = set() #instancesOrganism = set()\n",
    "#iSet5 = set() #instancesIndividual = set()\n",
    "#iSet6 = set() #instancesProgram = set()\n",
    "#iSet7 = set() #instancesCollection = set()\n",
    "#iSet8 = set() #instancesCity = set()\n",
    "#iSet9 = set() #instancesTempObject = set()\n",
    "#iSet10 = set() #instancesCoorp = set()\n",
    "iSetAll = [iSet1, iSet2]#, iSet3, iSet4] #, iSet5, iSet6, iSet7, iSet8, iSet9, iSet10]\n",
    "\n",
    "# indegree class instantiation  \n",
    "ci1 = ClassInstances('indegree person', '<http://www.wikidata.org/entity/Q215627>')\n",
    "ci2 = ClassInstances('indegree work', '<http://www.wikidata.org/entity/Q386724>')\n",
    "#ci1 = ClassInstances('indegree ', '<http://www.wikidata.org/prop/novalue/P37>')\n",
    "#ci2 = ClassInstances('indegree ', '<http://www.wikidata.org/prop/novalue/P102>')\n",
    "#ci3 = ClassInstances('indegree ', '<http://www.wikidata.org/prop/novalue/P155>')\n",
    "#ci4 = ClassInstances('indegree ', '<http://www.wikidata.org/prop/novalue/P2555>')\n",
    "#ci5 = ClassInstances('indegree ', '')\n",
    "#ci6 = ClassInstances('indegree ', '')\n",
    "#ci7 = ClassInstances('indegree ', '')\n",
    "#ci8 = ClassInstances('indegree ', '')\n",
    "#ci9 = ClassInstances('indegree ', '')\n",
    "#ci10 = ClassInstances('indegree ', '')\n",
    "ciAll = [ci1, ci2]#, ci3, ci4] #, ci5, ci6, ci7, ci8, ci9, ci10]\n",
    "\n",
    "# outdegree class instantiation \n",
    "co1 = ClassInstances('outdegree person', '<http://www.wikidata.org/entity/Q215627>')\n",
    "co2 = ClassInstances('outdegree work', '<http://www.wikidata.org/entity/Q386724>')\n",
    "#co3 = ClassInstances('outdegree ', '')\n",
    "#co4 = ClassInstances('outdegree ', '')\n",
    "#co5 = ClassInstances('outdegree ', '')\n",
    "#co6 = ClassInstances('outdegree ', '')\n",
    "#co7 = ClassInstances('outdegree ', '')\n",
    "#co8 = ClassInstances('outdegree ', '')\n",
    "#co9 = ClassInstances('outdegree ', '')\n",
    "#co10 = ClassInstances('outdegree ', '')\n",
    "coAll = [co1, co2]#, co3, co4] #, co5, co6, co7, co8, co9, co10]\n",
    "\n",
    "def getSPO(splittedLine):\n",
    "    word_position = 0\n",
    "    for word in splittedLine:\n",
    "        if (word_position == 0):\n",
    "            subj = word\n",
    "        elif (word_position == 1):\n",
    "            pred = word\n",
    "        elif (word_position == 2):\n",
    "            obj = word\n",
    "        else:\n",
    "            return subj, pred, obj\n",
    "        word_position += 1\n",
    "    return subj, pred, obj\n",
    "\n",
    "# count all instances of the top10 classes\n",
    "def countInstances(s, o):\n",
    "    #s a o: o=class s=instance\n",
    "    iAllSet.add(s)\n",
    "    if (o == ci1.getURI()):\n",
    "        if (s not in iSet1):\n",
    "            iSet1.add(s)\n",
    "        return\n",
    "    if (o == ci2.getURI()):\n",
    "        if (s not in iSet2):\n",
    "            iSet2.add(s)\n",
    "        return\n",
    "    if (o == ci3.getURI()):\n",
    "        if (s not in iSet3):\n",
    "            iSet3.add(s)\n",
    "        return\n",
    "    if (o == ci4.getURI()):\n",
    "        if (s not in iSet4):\n",
    "            iSet4.add(s)\n",
    "        return\n",
    "    if (o == ci5.getURI()):\n",
    "        if (s not in iSet5):\n",
    "            iSet5.add(s)\n",
    "        return\n",
    "    if (o == ci6.getURI()):\n",
    "        if (s not in iSet6):\n",
    "            iSet6.add(s)\n",
    "        return\n",
    "    if (o == ci7.getURI()):\n",
    "        if (s not in iSet7):\n",
    "            iSet7.add(s)\n",
    "        return\n",
    "    if (o == ci8.getURI()):\n",
    "        if (s not in iSet8):\n",
    "            iSet8.add(s)\n",
    "        return\n",
    "    if (o == ci9.getURI()):\n",
    "        if (s not in iSet9):\n",
    "            iSet9.add(s)\n",
    "        return\n",
    "    if (o == ci10.getURI()):\n",
    "        if (s not in iSet10):\n",
    "            iSet10.add(s)\n",
    "        return\n",
    "\n",
    "# count instance degrees\n",
    "def countInstanceDegrees(s, o):\n",
    "    if (s in iAllSet): # outdegree\n",
    "        if (s in iSet1):\n",
    "            co1.addInstance(s)\n",
    "        elif (s in iSet2):\n",
    "            co2.addInstance(s)\n",
    "        elif (s in iSet3):\n",
    "            co3.addInstance(s)\n",
    "        elif (s in iSet4):\n",
    "            co4.addInstance(s)\n",
    "        elif (s in iSet5):\n",
    "            co5.addInstance(s)\n",
    "        elif (s in iSet6):\n",
    "            co6.addInstance(s)\n",
    "        elif (s in iSet7):\n",
    "            co7.addInstance(s)\n",
    "        elif (s in iSet8):\n",
    "            co8.addInstance(s)\n",
    "        elif (s in iSet9):\n",
    "            co9.addInstance(s)\n",
    "        elif (s in iSet10):\n",
    "            co10.addInstance(s)       \n",
    "    if (o in iAllSet): # indegree\n",
    "        if (o in iSet1):\n",
    "            ci1.addInstance(o)\n",
    "        elif (o in iSet2):\n",
    "            ci2.addInstance(o)\n",
    "        elif (o in iSet3):\n",
    "            ci3.addInstance(o)\n",
    "        elif (o in iSet4):\n",
    "            ci4.addInstance(o)\n",
    "        elif (o in iSet5):\n",
    "            ci5.addInstance(o)\n",
    "        elif (o in iSet6):\n",
    "            ci6.addInstance(o)\n",
    "        elif (o in iSet7):\n",
    "            ci7.addInstance(o)\n",
    "        elif (o in iSet8):\n",
    "            ci8.addInstance(o)\n",
    "        elif (o in iSet9):\n",
    "            ci9.addInstance(o)\n",
    "        elif (o in iSet10):\n",
    "            ci10.addInstance(o)\n",
    "        return\n",
    "\n",
    "def calculateClassDegrees():\n",
    "    for item in ciAll:\n",
    "        item.calculateDegrees()\n",
    "    for item in coAll:\n",
    "        item.calculateDegrees()\n",
    "        \n",
    "def printClassDegreeResults():\n",
    "    for item in ciAll:\n",
    "        item.printResults()\n",
    "    for item in coAll:\n",
    "        item.printResults()\n",
    "        \n",
    "print('START')\n",
    "try:\n",
    "    # get all instances for the top10 classes\n",
    "    f = open(readFile, 'r')\n",
    "    lineCounter = 0\n",
    "    print ('START COUNTING INSTANCES')\n",
    "    for line in f:\n",
    "        splittedLine = line.rstrip('\\n').split()\n",
    "        s, p, o = getSPO(splittedLine)\n",
    "        if (p == rdfType and (o in top10set)):\n",
    "            #top10dic.has_key(o)):\n",
    "            countInstances(s, o)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "        #if (lineCounter > 5000):\n",
    "        #    break\n",
    "    f.close()\n",
    "    print ('DONE COUNTING INSTANCES')\n",
    "    for i, item in enumerate(iSetAll):\n",
    "        print ('iSet{}, #instances: {}'.format(i+1, len(iSetAll[i])))\n",
    "        \n",
    "    # count instance degrees\n",
    "    print ('START COUNTING INSTANCE DEGREES')\n",
    "    f = open(readFile, 'r')\n",
    "    lineCounter = 0\n",
    "    for line in f:\n",
    "        splittedLine = line.rstrip('\\n').split()\n",
    "        s, p, o = getSPO(splittedLine)\n",
    "        countInstanceDegrees(s, o)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "        #if (lineCounter > 5000):\n",
    "        #    break\n",
    "    f.close()\n",
    "    print ('DONE COUNTING INSTANCE DEGREES')\n",
    "    print ('START CALCULATING CLASS DEGREES')\n",
    "    calculateClassDegrees()\n",
    "    print ('DONE CALCULATING CLASS DEGREES')\n",
    "    print ('RESULTS')\n",
    "    printClassDegreeResults()   \n",
    "    print ('EXECUTION TIME: {} s'.format(time.time()-start))\n",
    "except:\n",
    "    print ('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}