NELL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "lineCounter: 2664732\n",
      "DONE\n",
      "EXECUTION TIME: 5.55556488037 s\n"
     ]
    }
   ],
   "source": [
    "# number of triples\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "\n",
    "start = time.time()\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "\n",
    "lineCounter = 0\n",
    "\n",
    "print('START')\n",
    "try:\n",
    "    # read nell\n",
    "    fn = open(nell, 'r')\n",
    "    for line in fn:\n",
    "        lineCounter += 1\n",
    "    print ('lineCounter: {}'.format(lineCounter))\n",
    "    fn.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000000 lines read\n",
      "2000000 lines read\n",
      "len(dbpediaSet): 521\n",
      "len(yagoSet): 489\n",
      "len(wikidataSet): 0\n",
      "len(openCycSet): 110\n",
      "FINISHED\n",
      "EXECUTION TIME: 17.5201160908 s\n"
     ]
    }
   ],
   "source": [
    "# equivalent classes\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "ontology = 'NELL/NELL.08m.995.ontology.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 1000000\n",
    "\n",
    "#equivalentSet = set()\n",
    "#sameSet = set()\n",
    "dbpediaSet = set()\n",
    "yagoSet = set()\n",
    "wikidataSet = set()\n",
    "openCycSet = set()\n",
    "\n",
    "try:\n",
    "    # read ontology\n",
    "    fr = open(nell, 'r')  \n",
    "    for line in fr:\n",
    "        #if \"equivalent\" in line:\n",
    "        #    equivalentSet.add(line)\n",
    "        #if \"same\" in line:\n",
    "        #    sameSet.add(line)\n",
    "        if \"dbpedia\" in line:\n",
    "            dbpediaSet.add(line)\n",
    "        if \"yago\" in line:\n",
    "            yagoSet.add(line)\n",
    "        if \"wikidata\" in line:\n",
    "            wikidataSet.add(line)\n",
    "        if \"opencyc\" in line:\n",
    "            openCycSet.add(line)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    #print ('len(equivalentSet): {}'.format(len(equivalentSet)))\n",
    "    #print ('len(sameSet): {}'.format(len(sameSet)))\n",
    "    print ('len(dbpediaSet): {}'.format(len(dbpediaSet)))\n",
    "    print ('len(yagoSet): {}'.format(len(yagoSet)))\n",
    "    print ('len(wikidataSet): {}'.format(len(wikidataSet)))\n",
    "    print ('len(openCycSet): {}'.format(len(openCycSet)))    \n",
    "    fr.close()\n",
    "    #fw = open('NELL/linesContainingEquivalent.csv', 'w')\n",
    "    #fw = open('NELL/dbpediaLines.csv', 'w')\n",
    "    fw = open('NELL/yagoLines.csv', 'w')\n",
    "    for item in yagoSet:\n",
    "        fw.write(item)\n",
    "    fw.close()\n",
    "    fw = open('NELL/openCycLines.csv', 'w')\n",
    "    for item in openCycSet:\n",
    "        fw.write(item)\n",
    "    fw.close()\n",
    "    print ('FINISHED')\n",
    "except:\n",
    "    print ('ERROR')\n",
    "    \n",
    "end = time.time()        \n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "concept:drug:reserpine\tgeneralizations\tconcept:drug\t340\t0.9999990463256836\tMBL-Iter%3A340-2011%2F07%2F10-08%3A21%3A28-From+ErrorBasedIntegrator+%28SEAL%28reserpine%2Cdrug%29%29\t\"RESERPINE\" \"reserpine\" \"Reserpine\" \t\tRESERPINE\t\t concept:chemical concept:drug\t concept:product\t%5BSEAL-Iter%3A221-2011%2F03%2F18-08%3A29%3A10-%3Ctoken%3Dreserpine%2Cdrug%3E-From%3ACategory%3Adrug-using-KB+http%3A%2F%2Fenc.tfode.com%2FBuspirone+http%3A%2F%2Fen.wikipedia.org%2Fwiki%2FFlupenthixol+http%3A%2F%2Fwww.thefullwiki.org%2FLisdexamfetamine+http%3A%2F%2Fuk.ask.com%2Fwiki%2FCyclobenzaprine+http%3A%2F%2Fwww.digplanet.com%2Fwiki%2FTandamine+http%3A%2F%2Fdictionary.sensagent.com%2FHydroxyzine%2Fen-en%2F+http%3A%2F%2Fuk.ask.com%2Fwiki%2FDosulepin+http%3A%2F%2Fwww.digplanet.com%2Fwiki%2FBlonanserin+http%3A%2F%2Fwww.digplanet.com%2Fwiki%2FOxaprotiline+http%3A%2F%2Fwww.enotes.com%2Ftopic%2FMazindol+http%3A%2F%2Fwww.ualberta.ca%2F%7Egjones%2Faafsdrug.xls+http%3A%2F%2Fwww.territorioscuola.com%2Fwikipedia%2Fen.wikipedia.php%3Ftitle%3DSubstituted_amphetamine+http%3A%2F%2Fuk.ask.com%2Fwiki%2FRisperidone+http%3A%2F%2Fwinx.name%2F%3Fw%3DMedifoxamine+http%3A%2F%2Fwww.google.com%2Fpatents%2FUS20070116729+http%3A%2F%2Fwww.digplanet.com%2Fwiki%2FCiclazindol+http%3A%2F%2Fnethelper.com%2Farticle%2FDOV-216%2C303+http%3A%2F%2Fuk.ask.com%2Fwiki%2FSibutramine+http%3A%2F%2Fwww.digplanet.com%2Fwiki%2FTampramine+http%3A%2F%2Fwww.parmida.ca%2Findex.php%3Fpr%3DDrugs_Neurology%2C+CMC-Iter%3A740-2013%2F06%2F09-10%3A55%3A58-%3Ctoken%3Dreserpine%2Cdrug%3E-LASTSUFFIX%3Dine%093.35690%09LASTSUFFIX%3Dne%092.95632%09SUFFIX%3Dne%092.89387%09SUFFIX%3Dine%092.86995%09SUFFIX%3Dpine%091.75431%09LASTSUFFIX%3Dpine%091.62836%09CHARS%091.52390%09PREFIX%3Dre%09-1.15106%09FULL_POS%3DNN%09-1.52531%09WORDS%09-4.71391%2C+CPL-Iter%3A802-2014%2F01%2F08-15%3A08%3A58-%3Ctoken%3Dreserpine%2Cdrug%3E-_+prevents+the+reuptake%091%09_+is+a+blood+pressure+medication%091%09_+is+an+alkaloid%091%09_+controls+high+blood+pressure%091%09alkaloids+like+_%091%09_+is+a+good+drug%091%09you+are+taking+Generic+_%091%2C+OE-Iter%3A829-2014%2F04%2F08-16%3A11%3A58-%3Ctoken%3Dreserpine%2Cdrug%3E-tell+your+doctor+and+pharmacist+if+you+are+allergic+to+reserpine%2C+aspirin%2C+any+other+medications%2C+tartrazine+%28a+yellow+dye+in+some+processed+foods+and+medications%29%2C+or+any+of+the+ingredients+in+reserpine+tablets.+%09%09drug+derived+from+the+roots+of+certain+species+of+the+tropical+plant+Rauwolfia.+The+powdered+whole+root+of+the+Indian+shrub+Rauwolfia+serpentina+historically+had+been+used+to+treat+snakebites%2C+insomnia%2C+hypertension%28+high+blood+pressure+%29%2C+and+insanity.+Reserpine%2C+isolated+in+1952%2C+was+the+first+of+many+Rauwolfia+alkaloids+found+in+the+crude+drug.+Because+the+drug+produces+a+profound+and+prolonged+tranquilizing+action%2C+it+was+once+used+in+treating+schizophrenia.+Reserpine+is+sometimes+used+in+treating+hypertension%2C+though+newer+antihypertensive+drugs+with+fewer+central+nervous+system+side+effects+are+the+preferred+treatment+%09%09Extreme+caution+should+be+exercised+in+treating+patients+with+a+history+of+mental+depression.+Reserpine+may+cause+mental+depression.+Recognition+of+depression+may+be+difficult+because+this+condition+may+often+be+disguised+by+somatic+complaints+%28Masked+Depression%29.+The+drug+should+be+discontinued+at+first+signs+of+depression+such+as+despondency%2C+early+morning+insomnia%2C+loss+of+appetite%2C+impotence%2C+or+self-deprecation.+Drug-induced+depression+may+persist+for+several+months+after+drug+withdrawal+and+may+be+severe+enough+to+result+in+suicide.+%09%09The+extent+to+which+these+findings+indicate+a+risk+to+humans+is+uncertain.+Tissue+culture+experiments+show+that+about+one+third+of+human+breast+tumors+are+prolactin-dependent+in+vitro%2C+a+factor+of+considerable+importance+if+the+use+of+the+drug+is+contemplated+in+a+patient+with+previously+detected+breast+cancer.+The+possibility+of+an+increased+risk+of+breast+cancer+in+Reserpine+users+has+been+studied+extensively%3B+however%2C+no+firm+conclusion+has+emerged.+Although+a+few+epidemiologic+studies+have+suggested+a+slightly+increased+risk+%28less+than+twofold+in+all+studies+except+one%29+in+women+who+have+used+Reserpine%2C+other+studies+of+generally+similar+design+have+not+confirmed+this.+Epidemiologic+studies+conducted+using+other+drugs+%28neuroleptic+agents%29+that%2C+like+Reserpine%2C+increase+prolactin+levels+and+therefore+would+be+considered+rodent+mammary+carcinogens+have+not+shown+an+association+between+chronic+administration+of+the+drug+and+human+mammary+tumorigenesis.+While+long-term+clinical+observation+has+not+suggested+such+as+association%2C+the+available+evidence+is+considered+too+limited+to+be+conclusive+at+this+time.+An+association+of+Reserpine+intake+with+pheochromocytoma+or+tumors+of+the+seminal+vesicles+has+not+been+explored.+%09%09PREGNANCY+and+BREAST-FEEDING%3A+It+is+unknown+if+reserpine+can+cause+harm+to+the+fetus.+If+you+become+pregnant+while+taking+reserpine%2C+discuss+with+your+doctor+the+benefits+and+risks+of+using+reserpine+during+pregnancy.+Reserpine+is+excreted+in+breast+milk.+Do+not+breast-feed+while+taking+reserpine.+%09%09This+may+not+be+a+complete+list+of+all+interactions+that+may+occur.+Ask+your+health+care+provider+if+reserpine+may+interact+with+other+medicines+that+you+take.+Check+with+your+health+care+provider+before+you+start%2C+stop%2C+or+change+the+dose+of+any+medicine.+%09%09tell+your+doctor+if+you+are+pregnant%2C+plan+to+become+pregnant%2C+or+are+breast-feeding.+If+you+become+pregnant+while+taking+reserpine%2C+call+your+doctor.+%09%09Reserpine+is+extensively+bound+%2895%25%29+to+plasma+proteins.+Reserpine+is+almost+completely+metabolized+in+the+body%2C+and+only+about+1%25+is+excreted+as+unchanged+drug+in+the+urine.+No+definitive+studies+on+the+human+metabolism+of+Reserpine+have+been+made.+After+oral+administration%2C+an+initial+half-life+of+approximately+5+hours+is+followed+by+a+terminal+half-life+of+the+order+of+200+hours.+Plasma+levels+may+be+measurable+14+days+after+a+single+dose.+The+clinical+significance+of+the+long+terminal+half-life+is+unknown.+%09%09There+are+no+adequate+and+well-controlled+studies+of+Reserpine+in+pregnant+women.+Reserpine+should+be+used+during+pregnancy+only+if+the+potential+benefit+justifies+the+potential+risk+to+the+fetus.+%09%09Reserpine+is+used+as+a+long-acting+tranquilizer+in+horses.+It+is+used+to+sedate+excitable+or+difficult+horses+that+are+on+enforced+rest.+It+sometimes+is+used+illicitly+to+sedate+show+horses%2C+sale+horses+or+in+other+circumstances+where+a+%26quot%3Bquieter%26quot%3B+horse+might+be+desired.+Until+relatively+recently%2C+reserpine+was+difficult+to+test+for%2C+but+there+are+now+sensitive+and+accurate+tests.+Blood+testing+for+reserpine+use+can+be+complicated+by+related+herbs+and+plants+found+in+supplements%2C+pastures+and+hay%2C+which+also+can+cause+a+positive+drug+test.+Reserpine+once+was+used+in+pregnant+mares+in+an+attempt+to+treat+fescue+toxicosis.+Domperidone+largely+has+replaced+reserpine+for+this+use.+%09%09Mean+maximum+plasma+levels+of+plasma+concentrations+after+a+single+dose+of+0.5+mg+of+Reserpine%2C+administered+as+two+0.25+mg+tablets+or+as+an+aqueous+solution%2C+peaked+after+2.5+hours.+The+mean+peak+level+was+approximately+1.1+ng%2Fml.+The+two+formulations+were+found+to+be+bioequivalent.+Absolute+bioavailability+of+Reserpine%2C+as+established+by+comparison+to+an+intravenous+dose%2C+has+been+reported+to+be+approximately+50%25.+%09%09Safety+and+effectiveness+in+children+have+not+been+established+by+means+of+controlled+clinical+trials%2C+although+there+is+experience+with+the+use+of+Reserpine+in+children+%28see+DOSAGE+AND+ADMINISTRATION+.%29+Because+of+adverse+effects+such+as+emotional+depression+and+lability%2C+sedation%2C+and+stuffy+nose%2C+Reserpine+is+not+usually+recommended+as+a+step-2+drug+in+the+treatment+of+hypertension+in+children.+%09%09Before+taking+reserpine%2C+tell+your+doctor+or+pharmacist+if+you+are+allergic+to+it%3B+or+if+you+have+any+other+allergies.+This+product+may+contain+inactive+ingredients%2C+which+can+cause+allergic+reactions+or+other+problems.+Talk+to+your+pharmacist+for+more+details.+%09%09Rodent+studies+have+shown+that+Reserpine+is+an+animal+tumorigen%2C+causing+an+increased+incidence+of+mammary+fibroadenomas+in+female+mice%2C+malignant+tumors+of+the+seminal+vesicles+in+male+mice%2C+and+malignant+adrenal+medullary+tumors+in+male+rats.+These+findings+arose+in+2-year+studies+in+which+the+drug+was+administered+in+the+feed+at+concentrations+of+5+to+10+ppm+%E2%80%93+about+100+to+300+times+the+usual+human+dose.+The+breast+neoplasms+are+thought+to+be+related+to+Reserpine%E2%80%99s+prolactin-elevating+effect.+Several+other+prolactin-elevating+drugs+have+also+been+associated+with+an+increased+incidence+of+mammary+neoplasia+in+rodents.+%09%09ask+your+doctor+about+the+safe+use+of+alcohol+while+you+are+taking+reserpine.+Alcohol+can+make+the+side+effects+from+reserpine+worse.+%09%09Reserpine+may+cause+depression.+Contact+your+health+care+provider+right+away+if+any+of+the+following+effects+occur+or+worsen%3A+feelings+of+decreased+self-worth%2C+a+decrease+in+sexual+ability+or+interest%2C+trouble+sleeping+%28especially+early+in+the+morning%29%2C+or+loss+of+appetite.+Drug-induced+depression+may+continue+for+several+months+after+you+stop+taking+reserpine+and+may+%28rarely%29+be+severe+enough+to+result+in+suicide.+%09%09PREGNANCY+and+BREAST-FEEDING%3A+It+is+unknown+if+reserpine+can+cause+harm+to+the+fetus.+If+you+become+pregnant+while+taking+reserpine%2C+discuss+with+your+doctor+the+benefits+and+risks+of+using+reserpine+during+pregnancy.+Reserpine+is+excreted+in+breast+milk.+Do+not+breast-feed+while+taking+reserpine.+%09%09Reserpine+may+be+be+prescribed+for+other+uses%3B+ask+your+doctor+or+pharmacist+for+more+information.+%09%09If+you+have+any+questions+about+reserpine%2C+please+talk+with+your+doctor%2C+pharmacist%2C+or+other+health+care+provider.+%09%09drug+derived+from+the+roots+of+certain+species+of+the+tropical+plant+Rauwolfia.+The+powdered+whole+root+of+the+Indian+shrub+Rauwolfia+serpentina+historically+had+been+used+to+treat+snakebites%2C+insomnia%2C+hypertension%28+high+blood+pressure+%29%2C+and+insanity.+Reserpine%2C+isolated+in+1952%2C+was+the+first+of+many+Rauwolfia+alkaloids+found+in+the+crude+drug.+Because+the+drug+produces+a+profound+and+prolonged+tranquilizing+action%2C+it+was+once+used+in+treating+schizophrenia.+Reserpine+is+sometimes+used+in+treating+hypertension%2C+though+newer+antihypertensive+drugs+with+fewer+central+nervous+system+side+effects+are+the+preferred+treatment+%09%09Safety+and+effectiveness+in+children+have+not+been+established+by+means+of+controlled+clinical+trials%2C+although+there+is+experience+with+the+use+of+Reserpine+in+children+%28see+DOSAGE+AND+ADMINISTRATION+.%29+Because+of+adverse+effects+such+as+emotional+depression+and+lability%2C+sedation%2C+and+stuffy+nose%2C+Reserpine+is+not+usually+recommended+as+a+step-2+drug+in+the+treatment+of+hypertension+in+children.+%09%09This+information+should+not+be+used+to+decide+whether+or+not+to+take+reserpine+or+any+other+medicine.+Only+your+health+care+provider+has+the+knowledge+and+training+to+decide+which+medicines+are+right+for+you.+This+information+does+not+endorse+any+medicine+as+safe%2C+effective%2C+or+approved+for+treating+any+patient+or+health+condition.+This+is+only+a+brief+summary+of+general+information+about+reserpine.+It+does+NOT+include+all+information+about+the+possible+uses%2C+directions%2C+warnings%2C+precautions%2C+interactions%2C+adverse+effects%2C+or+risks+that+may+apply+to+reserpine.+This+information+is+not+specific+medical+advice+and+does+not+replace+information+you+receive+from+your+health+care+provider.+You+must+talk+with+your+healthcare+provider+for+complete+information+about+the+risks+and+benefits+of+using+reserpine.+%09%09Some+medical+conditions+may+interact+with+reserpine.+Tell+your+doctor+or+pharmacist+if+you+have+any+medical+conditions%2C+especially+if+any+of+the+following+apply+to+you%3A+%09%09Some+MEDICINES+MAY+INTERACT+with+reserpine.+Tell+your+health+care+provider+if+you+are+taking+any+other+medicines%2C+especially+any+of+the+following%3A+%09%09At+doses+of+less+than+0.2%C2%A0mg%2Fday%2C+reserpine+has+few+side+effects%2C+the+most+common+of+which+is+nasal+congestion.+%09%09In+some+countries+reserpine+is+still+available+as+part+of+combination+drugs+for+the+treatment+of+hypertension%2C+in+most+cases+they+contain+also+a+diuretic+and%2For+a+vasodilator+like+hydralazine.+These+combinations+are+currently+regarded+as+second+choice+drugs.+The+daily+dose+of+reserpine+in+antihypertensive+treatment+is+as+low+as+0.1+to+0.25%C2%A0mg.+The+use+of+reserpine+as+an+antipsychotic+drug+had+been+nearly+completely+abandoned%2C+but+more+recently+it+made+a+comeback+as+adjunctive+treatment%2C+in+combination+with+other+antipsychotics%2C+so+that+more+refractory+patients+get+dopamine+blockade+from+the+other+antipsychotic%2C+and+dopamine+depletion+from+reserpine.+Doses+for+this+kind+of+adjunctive+goal+can+be+kept+low%2C+resulting+in+better+tolerability.+Originally%2C+doses+of+0.5%C2%A0mg+to+40%C2%A0mg+daily+were+used+to+treat+psychotic+diseases.+Doses+in+excess+of+3%C2%A0mg+daily+often+required+use+of+an+anticholinergic+drug+to+combat+excessive+cholinergic+activity+in+many+parts+of+the+body+as+well+as+parkinsonism.+For+adjunctive+treatment%2C+doses+are+typically+kept+at+or+below+0.25%C2%A0mg+twice+a+day.+Reserpine+may+be+used+as+a+sedative+for+horses.%5D\n",
      "\n",
      "\n",
      "\n",
      "concept:politicsissue:education_training\tgeneralizations\tconcept:politicsissue\t598\t1.0\tMBL-Iter%3A598-2012%2F06%2F21-02%3A17%3A24-From+ErrorBasedIntegrator+%28CMC%28education_training%2Cpoliticsissue%29%2C+SEAL%28education_training%2Cpoliticsissue%29%2C+CPL%28education_training%2Cpoliticsissue%29%2C+OE%28education_training%2Cpoliticsissue%29%29\t\"Education Training\" \"Education/Training\" \"education training\" \"education/training\" \"education-training\" \t\tEducation Training\t\t concept:politicsissue\t concept:abstractthing\t%5BCMC-Iter%3A164-2010%2F11%2F14-04%3A58%3A23-%3Ctoken%3Deducation_training%2Cpoliticsissue%3E-PREFIX%3Dedu%095.26242%09PREFIX%3Deduc%095.21866%09PREFIX%3Deduca%095.19766%09PREFIX%3Ded%093.78332%09SUFFIX%3Dation%092.45108%09SUFFIX%3Ding%092.18110%09SUFFIX%3Dtion%091.76017%09FULL_POS%3DNN_NN%09-1.67289%09POS%3DNN%09-2.41507%09WORDS%09-9.04445%2C+SEAL-Iter%3A177-2010%2F12%2F11-15%3A05%3A52-%3Ctoken%3Deducation_training%2Cpoliticsissue%3E-From%3ACategory%3Apoliticsissue-using-KB+http%3A%2F%2Fwww.b2byellowpages.com%2Fdirectory%2Fb2b-education-a2z-e.shtml%2C+CPL-Iter%3A667-2012%2F12%2F09-07%3A05%3A24-%3Ctoken%3Deducation_training%2Cpoliticsissue%3E-high+quality+continuing+_%091%09training+%2C+continuing+_%094%09people+in+full-time+_%091%09quality+continuing+_%093%09teacher+%2C+special+_%091%09resources+%2C+continuing+_%091%09offering+continuing+_%091%09post+secondary+_%092%09professional+continuing+_%091%09complete+continuing+_%091%09premier+continuing+_%092%09part+of+vocational+_%091%09person+qualified+by+_%093%09archival+continuing+_%091%09adult+basic+_%092%09nursing+continuing+_%091%09meetings+%2C+continuing+_%091%09equivalent+combination+of+_%095%09regular+continuing+_%091%09part+of+elementary+_%091%09provider+of+vocational+_%091%09program+of+physical+_%091%09graduate+medical+_%095%09degree+in+physical+_%091%09post+compulsory+_%091%09prevention+public+_%091%09practice+%2C+continuing+_%091%09staff+continuing+_%092%09adult+education+%2C+vocational+_%091%09minimum+continuing+_%091%09ongoing+continuing+_%093%09research+%2C+public+_%091%09specific+continuing+_%091%09quality+of+postgraduate+_%091%09annual+continuing+_%098%09field+of+international+_%091%09excellence+in+environmental+_%091%09additional+continuing+_%094%09city+board+of+_%091%09extensive+continuing+_%097%09training+%2C+vocational+_%094%09yearly+continuing+_%093%09EMS+continuing+_%091%09insurance+continuing+_%091%09lack+of+formal+_%091%09education+%2C+vocational+_%093%09on-going+continuing+_%091%09experience+%2C+continuing+_%091%09education+teacher+%2C+special+_%091%2C+OE-Iter%3A809-2014%2F02%2F01-16%3A14%3A45-%3Ctoken%3Deducation_training%2Cpoliticsissue%3E-The+Environmental+Education+and+Training+Partnership+%28EETAP%29+delivered+environmental+education+training%2C+services+and+resources+to+education+professionals+across+the+U.S.+It+advanced+environmental+education+with+guidelines%2C+standards%2C+accreditation%2C+certification+and+used+Internet+technology+to+deliver+courses%2C+information+and+resources.+EETAP+made+it+a+priority+to+reach+educators+who+served+ethnically+diverse+and+low-income+communities.+EETAP%27s+third+and+final+phase+started+October+2005+and+ended+February+2011.%5D\n",
      "\n",
      "\n",
      "\n",
      "concept:chemical:kerosene\tconcept:chemicalistypeofchemical\tconcept:chemical:fuels\t655\t0.9998884280841512\tMBL-Iter%3A655-2012%2F11%2F02-16%3A32%3A57-From+ErrorBasedIntegrator+%28CPL%28kerosene%2Cfuels%29%2C+OE%28kerosene%2Cfuels%29%29\t\"KEROSENE\" \"Kerosene\" \"kerosene\" \t\"fuels\" \"Fuels\" \tKEROSENE\tfuels\t concept:chemical\t concept:chemical concept:visualizablething\t%5BCPL-Iter%3A365-2011%2F07%2F28-14%3A36%3A59-%3Ctoken%3Dkerosene%2Cfuels%3E-arg2+like+coal+and+arg1%091%09arg2+like+natural+gas+and+arg1%092%09arg2+such+as+coal+and+arg1%091%09arg2+like+petrol+and+arg1%092%09arg2+such+as+gasoline+and+arg1%097%09arg1+or+other+fossil+arg2%091%09arg1+and+other+fossil+arg2%098%09arg2+such+as+diesel+and+arg1%093%09arg2+like+diesel+and+arg1%092%09arg2+like+gasoline+and+arg1%092%2C+OE-Iter%3A802-2014%2F01%2F09-16%3A38%3A26-%3Ctoken%3Dkerosene%2Cfuels%3E-This+method+is+for+determining+total+cycloparaffins+%28naphthenes%29+and+total+aromatics+in+synthetic+paraffinic+kerosene+fuels+%28SPK%29+or+petroleum+distillates+in+the+boiling+range+of+69%EF%BF%BDC+to+350%EF%BF%BDC+using+comprehensive+two-dimensional+gas+chromatography+%28GCxGC%29+coupled+with+a+flame+ionization+detector+%28FID%29.+Molecular+type+groups+are+identified+by+their+location%2C+as+plotted+on+a+chromatographic+contour+plot+image.+The+precision+of+this+test+method+has+been+established+for+SPK+fuels+containing+from+1.6+to+32.3+mass-%25+naphthenic+hydrocarbons+and+0.03+to+1.9+mass-%25+aromatic+hydrocarbons.+Alkenes+and+compounds+containing+sulfur%2C+nitrogen%2C+and+oxygen+are+possible+interferents.+%09%09Over+2+million+military+and+civilian+personnel+per+year+%28over+1+million+in+the+United+States%29+are+occupationally+exposed%2C+respectively%2C+to+jet+propulsion+fuel-8+%28JP-8%29%2C+JP-8+%2B100+or+JP-5%2C+or+to+the+civil+aviation+equivalents+Jet+A+or+Jet+A-1.+Approximately+60+billion+gallon+of+these+kerosene-based+jet+fuels+are+annually+consumed+worldwide+%2826+billion+gallon+in+the+United+States%29%2C+including+over+5+billion+gallon+of+JP-8+by+the+militaries+of+the+United+States+and+other+NATO+countries.+JP-8%2C+for+example%2C+represents+the+largest+single+chemical+exposure+in+the+U.S.+military+%282.53+billion+gallon+in+2000%29%2C+while+Jet+A+and+A-1+are+among+the+most+common+sources+of+nonmilitary+occupational+chemical+exposure.+Although+more+recent+figures+were+not+available%2C+approximately+4.06+billion+gallon+of+kerosene+per+se+were+consumed+in+the+United+States+in+1990+%28IARC%2C+1992%29.+These+exposures+may+occur+repeatedly+to+raw+fuel%2C+vapor+phase%2C+aerosol+phase%2C+or+fuel+combustion+exhaust+by+dermal+absorption%2C+pulmonary+inhalation%2C+or+oral+ingestion+routes.+Additionally%2C+the+public+may+be+repeatedly+exposed+to+lower+levels+of+jet+fuel+vapor%2Faerosol+or+to+fuel+combustion+products+through+atmospheric+contamination%2C+or+to+raw+fuel+constituents+by+contact+with+contaminated+groundwater+or+soil.+Kerosene-based+hydrocarbon+fuels+are+complex+mixtures+of+up+to+260%2B+aliphatic+and+aromatic+hydrocarbon+compounds+%28C%286%29+-C%2817%2B%29%3B+possibly+2000%2B+isomeric+forms%29%2C+including+varying+concentrations+of+potential+toxicants+such+as+benzene%2C+n-hexane%2C+toluene%2C+xylenes%2C+trimethylpentane%2C+methoxyethanol%2C+naphthalenes+%28including+polycyclic+aromatic+hydrocarbons+%5BPAHs%5D%2C+and+certain+other+C%289%29-C%2812%29+fractions+%28i.e.%2C+n-propylbenzene%2C+trimethylbenzene+isomers%29.+While+hydrocarbon+fuel+exposures+occur+typically+at+concentrations+below+current+permissible+exposure+limits+%28PELs%29+for+the+parent+fuel+or+its+constituent+chemicals%2C+it+is+unknown+whether+additive+or+synergistic+interactions+among+hydrocarbon+constituents%2C+up+to+six+performance+additives%2C+and+other+environmental+exposure+factors+may+result+in+unpredicted+toxicity.+While+there+is+little+epidemiological+evidence+for+fuel-induced+death%2C+cancer%2C+or+other+serious+organic+disease+in+fuel-exposed+workers%2C+large+numbers+of+self-reported+health+complaints+in+this+cohort+appear+to+justify+study+of+more+subtle+health+consequences.+A+number+of+recently+published+studies+reported+acute+or+persisting+biological+or+health+effects+from+acute%2C+subchronic%2C+or+chronic+exposure+of+humans+or+animals+to+kerosene-based+hydrocarbon+fuels%2C+to+constituent+chemicals+of+these+fuels%2C+or+to+fuel+combustion+products.+This+review+provides+an+in-depth+summary+of+human%2C+animal%2C+and+in+vitro+studies+of+biological+or+health+effects+from+exposure+to+JP-8%2C+JP-8+%2B100%2C+JP-5%2C+Jet+A%2C+Jet+A-1%2C+or+kerosene.+%09%09Health+Impacts%3A+Ethanol+burns+very+cleanly%2C+without+the+production+of+harmful+gases+and+fine+particulates+%28soot%29.+Burning+ethanol+produces+significantly+less+carbon+monoxide+%28CO%29+than+kerosene+or+solid+fuels.+Studies+conducted+in+a+number+of+countries%2C+both+in+the+laboratory+and+in+household+field+tests%2C+have+shown+the+benefit+of+alcohol+stoves+in+dramatically+reducing+indoor+air+pollution+as+compared+to+wood%2C+charcoal+and+kerosene+stoves.+%09%09Over+2+million+military+and+civilian+personnel+per+year+%28over+1+million+in+the+United+States%29+are+occupationally+exposed%2C+respectively%2C+to+jet+propulsion+fuel-8+%28JP-8%29%2C+JP-8+%2B100+or+JP-5%2C+or+to+the+civil+aviation+equivalents+Jet+A+or+Jet+A-1.+Approximately+60+billion+gallon+of+these+kerosene-based+jet+fuels+are+annually+consumed+worldwide+%2826+billion+gallon+in+the+United+States%29%2C+including+over+5+billion+gallon+of+JP-8+by+the+militaries+of+the+United+States+and+other+NATO+countries.+JP-8%2C+for+example%2C+represents+the+largest+single+chemical+exposure+in+the+U.S.+military+%282.53+billion+gallon+in+2000%29%2C+while+Jet+A+and+A-1+are+among+the+most+common+sources+of+nonmilitary+occupational+chemical+exposure.+Although+more+recent+figures+were+not+available%2C+approximately+4.06+billion+gallon+of+kerosene+per+se+were+consumed+in+the+United+States+in+1990+%28IARC%2C+1992%29.+These+exposures+may+occur+repeatedly+to+raw+fuel%2C+vapor+phase%2C+aerosol+phase%2C+or+fuel+combustion+exhaust+by+dermal+absorption%2C+pulmonary+inhalation%2C+or+oral+ingestion+routes.+Additionally%2C+the+public+may+be+repeatedly+exposed+to+lower+levels+of+jet+fuel+vapor%2Faerosol+or+to+fuel+combustion+products+through+atmospheric+contamination%2C+or+to+raw+fuel+constituents+by+contact+with+contaminated+groundwater+or+soil.+Kerosene-based+hydrocarbon+fuels+are+complex+mixtures+of+up+to+260%2B+aliphatic+and+aromatic+hydrocarbon+compounds+%28C%286%29+-C%2817%2B%29%3B+possibly+2000%2B+isomeric+forms%29%2C+including+varying+concentrations+of+potential+toxicants+such+as+benzene%2C+n-hexane%2C+toluene%2C+xylenes%2C+trimethylpentane%2C+methoxyethanol%2C+naphthalenes+%28including+polycyclic+aromatic+hydrocarbons+%5BPAHs%5D%2C+and+certain+other+C%289%29-C%2812%29+fractions+%28i.e.%2C+n-propylbenzene%2C+trimethylbenzene+isomers%29.+While+hydrocarbon+fuel+exposures+occur+typically+at+concentrations+below+current+permissible+exposure+limits+%28PELs%29+for+the+parent+fuel+or+its+constituent+chemicals%2C+it+is+unknown+whether+additive+or+synergistic+interactions+among+hydrocarbon+constituents%2C+up+to+six+performance+additives%2C+and+other+environmental+exposure+factors+may+result+in+unpredicted+toxicity.+While+there+is+little+epidemiological+evidence+for+fuel-induced+death%2C+cancer%2C+or+other+serious+organic+disease+in+fuel-exposed+workers%2C+large+numbers+of+self-reported+health+complaints+in+this+cohort+appear+to+justify+study+of+more+subtle+health+consequences.+A+number+of+recently+published+studies+reported+acute+or+persisting+biological+or+health+effects+from+acute%2C+subchronic%2C+or+chronic+exposure+of+humans+or+animals+to+kerosene-based+hydrocarbon+fuels%2C+to+constituent+chemicals+of+these+fuels%2C+or+to+fuel+combustion+products.+This+review+provides+an+in-depth+summary+of+human%2C+animal%2C+and+in+vitro+studies+of+biological+or+health+effects+from+exposure+to+JP-8%2C+JP-8+%2B100%2C+JP-5%2C+Jet+A%2C+Jet+A-1%2C+or+kerosene.+%09%09No%2C+the+E85+price+is+not+a+typo.+Ethanol+based+fuel+has+been+about+a+buck+a+gallon+cheaper+for+the+last+several+years.+I%27ve+been+experimenting+with+varying+percentages+of+E85+in+my+gasoline+powered+5.7-liter+1994+Chevy+Silverado.+A+story+about+my+results+as+well+as+doing+a+full+conversion+to+E85+is+in+the+plans.+Which+brings+me+to+your+question+about+running+a+gasoline+and+SVO+%28straight+vegetable+oil%29+mixture+in+a+diesel.+I+would+steer+clear+for+the+very+reasons+you+mentioned+as+well+as+others+from+horror+stories+I%27ve+heard.+Diesels+are+compression+ignition+engines+designed+for+dense%2C+low+volatility+oil+based+fuels+%28diesel%2C+biodiesel+and+kerosene%29.+Gasoline+style+engines+are+spark+ignited+and+require+light+and+highly+volatile+fuels+%28gasoline%2C+alcohol+and+E85%29.+It+doesn%27t+take+a+stretch+of+the+imagination+to+understand+that+an+improperly-mixed+amount+of+gasoline+and+vegetable+oil+%28too+much+gasoline%29+could+easily+prematurely+ignite+%28detonate%29+and+destroy+a+diesel+engine%27s+pistons+and%2For+valvetrain.+Reciprocally%2C+I%27ve+heard+stories+of+mixtures+with+too+much+veggie+oil+gumming+up+and+clogging+filters%2C+pumps+and+injectors.+%09%09Health+Impacts%3A+In+lab+testing%2C+emissions+from+kerosene+stoves+were+low%2C+though+appreciably+higher+than+alcohol+or+propane.+Kerosene+is+not+as+clean+as+gaseous+fuels+and+produces+more+emissions+than+LPG.+Additionally%2C+kerosene+fuel+use+carries+a+higher+risk+of+injury+than+many+other+household+cooking+fuels.+%09%09Kerosene+wick+lamps+should+only+be+operated+with+kerosene+or+lamp+oil%2C+but+alternative+fuels+are+used+in+an+emergency.+Such+fuels+may+produce+additional+smoke+and+odor+and+may+not+be+usable+indoors.+Tractor+vaporizing+oil+is+made+from+kerosene+with+some+additive+to+make+a+motor+fuel+for+tractors.+No.+1+diesel+fuel+%28also+called+winter+diesel%29+is+about+the+same+as+kerosene+but+with+the+additives+to+make+it+a+motor+fuel.+Jet+A+jet-engine+fuel+is+essentially+kerosene+with+a+few+additives.+%09%09Kerosene+Plattsburgh%2C+NY%28+New+York%29+-+Leroux+Fuels+%09%09Kerosene+in+diesel+engine+question+%E2%80%93+fuels+and+gasoline+in+diesel+engine+%E2%80%93+biodiesel+and+gasoline+in+compression+ignition+engine+%09%09Kerosene+is+usually+called+paraffin+in+the+UK%2C+Ireland%2C+Southeast+Asia+and+South+Africa.+A+more+viscous+paraffin+oil+is+used+as+a+laxative.+A+waxy+solid+extracted+from+petroleum+is+called+paraffin+wax.+Kerosene+is+widely+used+to+power+jet+engines+of+aircraft%28+jet+fuel%29+and+some+rocket+engines%2C+but+is+also+commonly+used+as+a+cooking+and+lighting+fuel+and+for+fire+toys+such+as+poi.+In+parts+of+Asia%2C+where+the+price+of+kerosene+is+subsidized%2C+it+fuels+outboard+motors+on+small+fishing+boats.+%09%09Over+2+million+military+and+civilian+personnel+per+year+%28over+1+million+in+the+United+States%29+are+occupationally+exposed%2C+respectively%2C+to+jet+propulsion+fuel-8+%28JP-8%29%2C+JP-8+%2B100+or+JP-5%2C+or+to+the+civil+aviation+equivalents+Jet+A+or+Jet+A-1.+Approximately+60+billion+gallon+of+these+kerosene-based+jet+fuels+are+annually+consumed+worldwide+%2826+billion+gallon+in+the+United+States%29%2C+including+over+5+billion+gallon+of+JP-8+by+the+militaries+of+the+United+States+and+other+NATO+countries.+JP-8%2C+for+example%2C+represents+the+largest+single+chemical+exposure+in+the+U.S.+military+%282.53+billion+gallon+in+2000%29%2C+while+Jet+A+and+A-1+are+among+the+most+common+sources+of+nonmilitary+occupational+chemical+exposure.+Although+more+recent+figures+were+not+available%2C+approximately+4.06+billion+gallon+of+kerosene+per+se+were+consumed+in+the+United+States+in+1990+%28IARC%2C+1992%29.+These+exposures+may+occur+repeatedly+to+raw+fuel%2C+vapor+phase%2C+aerosol+phase%2C+or+fuel+combustion+exhaust+by+dermal+absorption%2C+pulmonary+inhalation%2C+or+oral+ingestion+routes.+Additionally%2C+the+public+may+be+repeatedly+exposed+to+lower+levels+of+jet+fuel+vapor%2Faerosol+or+to+fuel+combustion+products+through+atmospheric+contamination%2C+or+to+raw+fuel+constituents+by+contact+with+contaminated+groundwater+or+soil.+Kerosene-based+hydrocarbon+fuels+are+complex+mixtures+of+up+to+260%2B+aliphatic+and+aromatic+hydrocarbon+compounds+%28C%286%29+-C%2817%2B%29%3B+possibly+2000%2B+isomeric+forms%29%2C+including+varying+concentrations+of+potential+toxicants+such+as+benzene%2C+n-hexane%2C+toluene%2C+xylenes%2C+trimethylpentane%2C+methoxyethanol%2C+naphthalenes+%28including+polycyclic+aromatic+hydrocarbons+%5BPAHs%5D%2C+and+certain+other+C%289%29-C%2812%29+fractions+%28i.e.%2C+n-propylbenzene%2C+trimethylbenzene+isomers%29.+While+hydrocarbon+fuel+exposures+occur+typically+at+concentrations+below+current+permissible+exposure+limits+%28PELs%29+for+the+parent+fuel+or+its+constituent+chemicals%2C+it+is+unknown+whether+additive+or+synergistic+interactions+among+hydrocarbon+constituents%2C+up+to+six+performance+additives%2C+and+other+environmental+exposure+factors+may+result+in+unpredicted+toxicity.+While+there+is+little+epidemiological+evidence+for+fuel-induced+death%2C+cancer%2C+or+other+serious+organic+disease+in+fuel-exposed+workers%2C+large+numbers+of+self-reported+health+complaints+in+this+cohort+appear+to+justify+study+of+more+subtle+health+consequences.+A+number+of+recently+published+studies+reported+acute+or+persisting+biological+or+health+effects+from+acute%2C+subchronic%2C+or+chronic+exposure+of+humans+or+animals+to+kerosene-based+hydrocarbon+fuels%2C+to+constituent+chemicals+of+these+fuels%2C+or+to+fuel+combustion+products.+This+review+provides+an+in-depth+summary+of+human%2C+animal%2C+and+in+vitro+studies+of+biological+or+health+effects+from+exposure+to+JP-8%2C+JP-8+%2B100%2C+JP-5%2C+Jet+A%2C+Jet+A-1%2C+or+kerosene.%5D\n",
      "\n",
      "\n",
      "\n",
      "concept:insect:scale\tconcept:animalsuchasinsect\tconcept:insect:insects\t724\t0.9999999999967514\tMBL-Iter%3A724-2013%2F04%2F12-16%3A44%3A09-From+ErrorBasedIntegrator+%28CPL%28scale%2Cinsects%29%2C+OE%28scale%2Cinsects%29%2C+PRA%28scale%2Cinsects%29%29\t\"scale\" \"Scale\" \"SCALE\" \t\"Insects\" \"INSECTS\" \"insects\" \tscale\tInsects\t concept:insect\t concept:insect\t%5BCPL-Iter%3A724-2013%2F04%2F10-10%3A36%3A25-%3Ctoken%3Dscale%2Cinsects%3E-arg1+and+some+scale+arg2%091%09arg1+and+other+sucking+arg2%092%09arg1+and+other+soft+bodied+arg2%092%09arg1+and+mealybug+arg2%091%09arg2+that+prey+upon+arg1%091%09arg2+known+as+rose+arg1%092%09arg2+called+rose+arg1%092%09arg2+such+as+aphids+or+arg1%091%09arg2+such+as+mites+and+arg1%092%09arg2+such+as+aphids+and+arg1%092%09arg1+and+other+scale+arg2%091%09arg2+like+caterpillars+and+arg1%091%2C+OE-Iter%3A803-2014%2F01%2F13-16%3A22%3A13-%3Ctoken%3Dscale%2Cinsects%3E-If+inside+plants+are+infested%2C+you+may+be+looking+for+a+way+to+control+scale+insects+on+indoor+plants.+These+tan+or+brown+shell%2C+sap-eating+insects+like+to+live+on+the+stems+or+the+leaves+of+the+plants+in+your+home+or+your+greenhouse.+Scale+insects+can+look+hemispherical%2C+oval%2C+or+flat%2C+and+they+can+ruin+a+plant+and+stunt+its+growth+by+sucking+out+the+sap.+Scales+can+often+be+seen+if+you+flip+a+leaf+upside+down+and+check+out+where+the+leaf+portion+of+the+plant+joins+the+stem.+You+can+also+find+them+at+the+root+of+the+plant+or+in+the+stem+crevices%2C+all+which+make+controlling+them+difficult.+The+secretions+can+also+lead+to+the+growth+of+sooty+mold%2C+a+fungus+that+is+black%2C+in+your+home.+But+you+can+easily+get+rid+of+the+insects+to+keep+your+plants+in+optimal+health.+%09%09Scale+insects+are+among+the+major+pests+in+orchards+and+nurseries%2C+on+trees+in+public+parks+and+forests%2C+and+on+home+garden+ornamentals+and+house+plants.+Because+effective+control+measures+begin+with+accurate+species+identification%2C+Scale+Insects+of+Northeastern+North+America%3A+Identification%2C+Biology%2C+and+Distribution+is+an+essential+tool+for+anyone+involved+in+pest+control.+%09%09Scale+insects+are+among+the+major+pests+in+orchards+and+nurseries%2C+on+trees+in+public+parks+and+forests%2C+and+on+home+garden+ornamentals+and+house+plants.+Because+effective+control+measures+begin+with+accurate+species+identification%2C+Scale+Insects+of+Northeastern+North+America%3A+Identification%2C+Biology%2C+and+Distribution+is+an+essential+tool+for+anyone+involved+in+pest+control.+%09%09O%27Connor%2C+J.P.%2C+Gertsson%2C+C-A.+and+Malumphy%2C+C.+2013.+A+review+of+the+Irish+scale+insects+%28Hemiptera%3A+Coccoidea%29.+Ir.+Nat.+J.+32%3A+32-+44.+%09%09Several+species+of+scale+insects+commonly+infest+plants+in+the+home+or+greenhouse.+These+sap-feeding+insects+have+a+tan+to+brown+shell-like+covering+or+scale+that+protects+the+insect%27s+body.+Scales+may+be+from+1%2F16+to+1%2F4+inch+in+diameter+and+are+usually+found+on+the+stems+and%2For+leaves.+Some+scales+are+hemispherical+in+shape%2C+while+others+are+oval+and+flat.+Scale+insects+feed+by+sucking+plant+sap+and+may+cause+poor%2C+stunted+growth.+Death+of+infested+plants+is+possible+in+severe+cases.+A+large+quantity+of+a+sweet+sticky+liquid+called+honeydew+is+excreted+by+scale+insects.+Honeydew+can+make+a+sticky%2C+shiny+mess+on+the+plant+and+nearby+furniture+and+floors.+A+black+fungus+called+sooty+mold+may+grow+on+the+honeydew.+%09%09Buss+EA%2C+Turner+JC.+2004.+Scale+Insects+and+Mealybugs+on+Ornamental+Plants.+EDIS.+University+of+Florida.+http%3A%2F%2Fufdc.ufl.edu%2Fl%2FIR00002883%2F00001+%2805+July+2013%29+%09%0957.+Pringle+EG%2C+Ak%26ccedil%3Bay+E%2C+Raab+TK%2C+Dirzo+R%2C+Gordon+DM+%282013%29+Water+stress+strengthens+mutualism+among+ants%2C+trees+and+scale+insects.+Dryad+Data+Submission.+Openly+available+via+http%3A%2F%2Fdoi.org%2F10.5061%2Fdryad.nq816+.+%09%09At+the+same+time%2C+some+kinds+of+scale+insects+are+biological+control+agents+for+pest+plants%2C+such+as+various+species+of+Dactylopius%2C+the+cochineal+genus%2C+that+attack+pest+species+of+Opuntia+.+%09%09Citation%3A+Pringle+EG%2C+Ak%26ccedil%3Bay+E%2C+Raab+TK%2C+Dirzo+R%2C+Gordon+DM+%282013%29+Water+Stress+Strengthens+Mutualism+Among+Ants%2C+Trees%2C+and+Scale+Insects.+PLoS+Biol+11%2811%29%3A++e1001705.++doi%3A10.1371%2Fjournal.pbio.1001705+%09%09Scale+insects+feed+on+a+wide+variety+of+plants%2C+though+particular+species+commonly+are+specific+to+particular+host+plants+or+plant+groups.+For+example%2C+various+kinds+of+cochineal+are+restricted+to+cactus+hosts.+Some+scale+insects+species+evolved+symbiotically+with+some+ant+species.+%09%09Ben-Dov+Y%2C+Miller+DR%2C+Gibson+GAP.+2001.+ScaleNet+%28Scale+Insects-General+information%29.+http%3A%2F%2Fwww.sel.barc.usda.gov%2Fscalenet%2Fscalenet.htm+%2805+July+2013%29+%09%09Several+species+of+scale+insects+commonly+infest+plants+in+the+home+or+greenhouse.+These+sap-feeding+insects+have+a+tan+to+brown+shell-like+covering+or+scale+that+protects+the+insect%27s+body.+Scales+may+be+from+1%2F16+to+1%2F4+inch+in+diameter+and+are+usually+found+on+the+stems+and%2For+leaves.+Some+scales+are+hemispherical+in+shape%2C+while+others+are+oval+and+flat.+Scale+insects+feed+by+sucking+plant+sap+and+may+cause+poor%2C+stunted+growth.+Death+of+infested+plants+is+possible+in+severe+cases.+A+large+quantity+of+a+sweet+sticky+liquid+called+honeydew+is+excreted+by+scale+insects.+Honeydew+can+make+a+sticky%2C+shiny+mess+on+the+plant+and+nearby+furniture+and+floors.+A+black+fungus+called+sooty+mold+may+grow+on+the+honeydew.+%09%09The+mutualism+between+trees+and+ants+was+stronger+in+drier+sites+than+in+wetter+sites+across+the+26+surveyed+locations+from+Mexico+to+Costa+Rica%28+Figure+3+%29.+Trees+supported+more+scale+insects%28+Figure+3A+%29%2C+and+ant+colonies+were+correspondingly+larger%28+Figure+3B%29+%5B19%5D+in+drier+sites+than+in+wetter+sites.+Colony+size+increased+faster+than+the+number+of+scale+insects+with+water+stress%2C+such+that+ant+colonies+at+a+drier+site+%28Chamela%29+tended+fewer+scale+insects+per+worker+ant+than+ants+at+a+wetter+site+%28Santa+Rosa%29+%28scale+insects+per+ant%3D+0.25%26plusmn%3B0.02+in+Chamela+and+0.44%26plusmn%3B0.05+in+Santa+Rosa%3B+t+test%2C+p+%26lt%3B0.004%29.+In+contrast+to+the+number+of+worker+ants%2C+colonies+did+not+produce+more+female+reproductives+at+drier+sites%28+Table+S5+%29.+Nitrogen+isotopes+indicated+that+ants+fed+at+the+same+trophic+level+relative+to+host+trees+and+scale+insects+at+a+drier+and+a+wetter+site%28+Figure+S3A+%29%3B+ant+tissues+also+contained+equivalent+percent+carbon+and+nitrogen+at+different+sites%28+Figure+S3B+%26ndash%3BC%29.+Ants+defended+trees+more+effectively+in+drier+sites%2C+responding+more+quickly+to+a+disturbance+to+their+host+tree+than+in+wetter+sites%28+Figures+3C+and+S4+%29.+In+ant-exclusion+experiments+conducted+at+three+sites+encompassing+the+extremes+of+the+precipitation+gradient%2C+herbivory+was+significantly+lower+on+leaves+with+ants+than+on+leaves+without+ants+in+two+of+the+driest+sites+%28%7E774+mm-rain%2Fy%29+but+not+in+a+wetter+site+%28%7E1702+mm-rain%2Fy%29%28+Figure+3D+%29.+Behavioral+assays+of+activity+and+responsiveness+indicated+that+ants+were+more+likely+to+find+herbivores+and+chase+them+from+trees+in+drier+sites+than+in+wetter+sites%28+Figure+S5+%29.+%09%09Several+species+of+scale+insects+commonly+infest+plants+in+the+home+or+greenhouse.+These+sap-feeding+insects+have+a+tan+to+brown+shell-like+covering+or+scale+that+protects+the+insect%27s+body.+Scales+may+be+from+1%2F16+to+1%2F4+inch+in+diameter+and+are+usually+found+on+the+stems+and%2For+leaves.+Some+scales+are+hemispherical+in+shape%2C+while+others+are+oval+and+flat.+Scale+insects+feed+by+sucking+plant+sap+and+may+cause+poor%2C+stunted+growth.+Death+of+infested+plants+is+possible+in+severe+cases.+A+large+quantity+of+a+sweet+sticky+liquid+called+honeydew+is+excreted+by+scale+insects.+Honeydew+can+make+a+sticky%2C+shiny+mess+on+the+plant+and+nearby+furniture+and+floors.+A+black+fungus+called+sooty+mold+may+grow+on+the+honeydew.+%09%09Dekle+GW.+1976.+Florida+Armored+Scale+Insects.+Florida+Department+of+Agricultural+and+Consumer+Services%2C+Division+of+Plant+Industry.+Arthropods+of+Florida+and+neighboring+land+areas+3%3A+1-345.+%09%09Indoor+Plants%3A+Remove+scale+by+rubbing+gently+with+a+facial+quality+sponge+dipped+in+rubbing+alcohol.+The+alcohol+alone+should+kill+the+scale%2C+but+the+dead+insects+will+remain+on+your+plants+and+make+it+difficult+for+you+to+scout+for+new+infestations.+The+small+facial+sponges%2C+found+in+the+cosmetic+aisle%2C+are+abrasive%2C+yet+soft+enough+to+use+without+scraping+the+plant+stems.+%09%09Abiotic+environmental+variables+strongly+affect+the+outcomes+of+species+interactions.+For+example%2C+mutualistic+interactions+between+species+are+often+stronger+when+resources+are+limited.+The+effect+might+be+indirect%3A+water+stress+on+plants+can+lead+to+carbon+stress%2C+which+could+alter+carbon-mediated+plant+mutualisms.+In+mutualistic+ant%26ndash%3Bplant+symbioses%2C+plants+host+ant+colonies+that+defend+them+against+herbivores.+Here+we+show+that+the+partners%27+investments+in+a+widespread+ant%26ndash%3Bplant+symbiosis+increase+with+water+stress+across+26+sites+along+a+Mesoamerican+precipitation+gradient.+At+lower+precipitation+levels%2C+Cordia+alliodora+trees+invest+more+carbon+in+Azteca+ants+via+phloem-feeding+scale+insects+that+provide+the+ants+with+sugars%2C+and+the+ants+provide+better+defense+of+the+carbon-producing+leaves.+Under+water+stress%2C+the+trees+have+smaller+carbon+pools.+A+model+of+the+carbon+trade-offs+for+the+mutualistic+partners+shows+that+the+observed+strategies+can+arise+from+the+carbon+costs+of+rare+but+extreme+events+of+herbivory+in+the+rainy+season.+Thus%2C+water+limitation%2C+together+with+the+risk+of+herbivory%2C+increases+the+strength+of+a+carbon-based+mutualism.+%09%09If+inside+plants+are+infested%2C+you+may+be+looking+for+a+way+to+control+scale+insects+on+indoor+plants.+These+tan+or+brown+shell%2C+sap-eating+insects+like+to+live+on+the+stems+or+the+leaves+of+the+plants+in+your+home+or+your+greenhouse.+Scale+insects+can+look+hemispherical%2C+oval%2C+or+flat%2C+and+they+can+ruin+a+plant+and+stunt+its+growth+by+sucking+out+the+sap.+Scales+can+often+be+seen+if+you+flip+a+leaf+upside+down+and+check+out+where+the+leaf+portion+of+the+plant+joins+the+stem.+You+can+also+find+them+at+the+root+of+the+plant+or+in+the+stem+crevices%2C+all+which+make+controlling+them+difficult.+The+secretions+can+also+lead+to+the+growth+of+sooty+mold%2C+a+fungus+that+is+black%2C+in+your+home.+But+you+can+easily+get+rid+of+the+insects+to+keep+your+plants+in+optimal+health.+%09%09Many+scale+species+are+serious+crop+pests.+The+waxy+covering+of+many+species+of+scale+insects+protects+them+effectively+from+contact+insecticides%2C+which+are+only+effective+against+the+first-+instar+nymph+stage+known+as+the+crawler.+However%2C+scales+often+are+controlled+by+use+of+horticultural+oils%2C+that+suffocate+them%2C+systemic+pesticides+that+poison+the+sap+of+the+host+plants%2C+or+by+biological+control+agents+such+as+tiny+parasitoid+wasps+and+Coccinellid+beetles.+%09%09In+this+study%2C+we+tested+whether+the+strength+of+an+ant%26ndash%3Bplant+mutualism%2C+measured+as+investment+by+trees+in+scale+insects+that+feed+ants+and+by+ants+in+defense+of+leaves%2C+is+related+to+water+availability+across+%7E2%2C300+km+of+Mesoamerica+from+Mexico+to+Costa+Rica.+We+investigated+the+relationships+among+water%2C+carbohydrates%2C+and+mutualistic+traits+across+this+gradient+to+test+whether+water+stress+leads+to+carbon+stress%2C+and+whether+carbon+stress+affects+carbohydrate+allocation+strategies+by+the+mutualists%28+Figure+1+%29.+We+first+asked+whether+precipitation+was+correlated+with+mutualistic+investments+by+trees+and+ants.+We+then+asked+whether+trees+store+more+carbon%2C+and+experience+a+lower+risk+of+carbon+starvation%2C+at+sites+where+precipitation+is+higher+and+the+growing+%28rainy%29+season+is+longer.+Finally%2C+to+investigate+when+the+cost+of+herbivory+during+the+rainy+season+is+high+enough+for+trees+to+invest+carbon+in+ant+defense%2C+we+modeled+carbon-allocation+trade-offs+for+trees+and+ants+in+rainy+seasons+of+different+durations+associated+with+different+precipitation+regimes.%2C+PRA-Iter%3A960-2015%2F11%2F10-12%3A36%3A31-%3Ctoken%3Dscale%2Cinsects%3E-Y%7EX0%7Econcept%3Aanimalthatfeedoninsect%093.20329%09Y%7EX0%7Econcept%3Ainverseofarthropodandotherarthropod%092.02818%09Y%7EX0%7Econcept%3Aarthropodandotherarthropod%091.96327%09Y%7EX0%7Econcept%3Aspecializationof%091.45026%09Y%7EX0%7Econcept%3Ainverseofanimalsuchasinsect%091.33007%5D\n",
      "\n",
      "\n",
      "\n",
      "concept:musicalbum:double_fantasy\tgeneralizations\tconcept:musicalbum\t535\t0.9999459241636904\tMBL-Iter%3A535-2012%2F03%2F21-20%3A24%3A32-From+ErrorBasedIntegrator+%28SEAL%28double_fantasy%2Cmusicalbum%29%2C+OE%28double_fantasy%2Cmusicalbum%29%29\t\"Double Fantasy\" \t\tDouble Fantasy\t\t concept:musicalbum\t concept:creativework\t%5BSEAL-Iter%3A535-2012%2F03%2F20-10%3A45%3A39-%3Ctoken%3Ddouble_fantasy%2Cmusicalbum%3E-From%3ACategory%3Amusicalbum-using-KB+http%3A%2F%2Fwww.worldcat.org%2Ftitle%2Fkill-your-idols-a-new-generation-of-rock-writers-reconsiders-the-classics%2Foclc%2F54694728+http%3A%2F%2Fwww.jimdero.com%2FKillYourIdols%2FKYINewMexico.htm+http%3A%2F%2Fc--noise.blogspot.com%2F2011%2F01%2Fjim-derogatis-kill-your-idols.html+http%3A%2F%2Fwww.anatolianrock.com%2FJohn_Lennon%2Fsarki_sozleri%2Finstant_karma-1116-0.htm+http%3A%2F%2Frateyourmusic.com%2Flist%2Fandland%2Frolling_stones_100_greatest_albums_of_the_80s%2F+http%3A%2F%2Fwww.5starstaff.com%2Fwestbury83_toppop.htm+http%3A%2F%2Fwww.rocklistmusic.co.uk%2Frstone.html+http%3A%2F%2Fwww.rocklistmusic.co.uk%2Fitalian_lists.htm+http%3A%2F%2Fwww.80sxchange.com%2F80s_charts%2Ftop100albums.htm%2C+OE-Iter%3A805-2014%2F01%2F20-11%3A48%3A18-%3Ctoken%3Ddouble_fantasy%2Cmusicalbum%3E-On+5+June+1981%2C+Geffen+re-released+%26quot%3BWoman%26quot%3B+as+a+single+as+part+of+their+%26quot%3BBack+to+Back+Hits%26quot%3B+series%2C+with+the+B-side+%26quot%3B%28Just+Like%29+Starting+Over%26quot%3B.+%26quot%3BWatching+the+Wheels%26quot%3B+was+re-released+as+part+of+the+same+series%2C+with+%26quot%3B+Beautiful+Boy+%28Darling+Boy%29+%26quot%3B+on+the+B-side.+%26quot%3BBeautiful+Boy+%28Darling+Boy%29%26quot%3B+was+re-released+this+time+as+the+B-side+to+a+reissue+of+%26quot%3B+Happy+Xmas+%28War+Is+Over%29+%26quot%3B+by+Geffen%2C+in+a+brand-new+picture+sleeve%2C+in+order+to+promote+The+John+Lennon+Collection+in+November+1982.+It+was+first+released+on+CD+on+13+October+1986+in+the+UK%2C+and+a+nearly+a+year+later+on+15+September+1987+in+the+US.+The+CD+was+issued+again+in+the+US%2C+this+time+by+the+Columbia+Record+Club.+The+album+was+re-released+on+cassette%2C+CD+and+vinyl+in+1989%2C+after+EMI+had+obtained+the+rights+to+the+album.+The+album+was+reissued+again+on+vinyl+this+time+by+Capitol+%2FColumbia+House+in+both+the+US+and+UK+in+1989.+In+1994%2C+Mobile+Fidelity+Sound+Lab+issued+the+album+on+CD.+On+9+October+2000%2C+EMI%2FCapitol+released+a+remastered+version+of+the+album%2C+containing+three+bonus+tracks.+In+2010%2C+a+two-CD+set+called+Double+Fantasy+Stripped+Down+was+released.+It+included+a+newly+remastered+copy+of+the+original+album+along+with+an+alternative+version+of+the+album+featuring+simpler+arrangements%2C+with+cover+artwork+by+Sean+Lennon.+%09%092010+two+CD+release+containing+the+digitally+remastered+version+of+the+1980+Double+Fantasy+album+plus+a+never+before+released+%27Stripped+Down%27+version+produced+by+Yoko+Ono+and+Jack+Douglas.+Includes+%27%28Just+Like%29+Starting+Over%27%2C+%27Woman%27%2C+%27Watching+The+Wheels%27%2C+%27Beautiful+Boy+%28Darling+Boy%29%27+and+more.+%09%09On+24th+February+1982+%26quot%3BDouble+Fantasy%26quot%3B+is+voted+%26quot%3BAlbum+Of+The+Year%26quot%3B+at+the+Grammy+Awards.+Yoko+receives+the+award.+%09%09John+Lennon%27s+comeback+album+after+five+years+out+of+the+public+eye%2C+Double+Fantasy+was+also+the+last+to+be+released+in+his+lifetime.+%09%09In+1982%2C+Douglas%2C+Lennon+and+Ono+won+the+1981+Album+of+the+Year+for+Double+Fantasy+at+the+24th+Annual+Grammy+Awards.+In+1989+the+album+was+ranked+number+29+on+Rolling+Stone+magazine%27s+list+of+the+100+greatest+albums+of+the+1980s.+In+2003%2C+a+copy+of+the+album+that+Lennon+had+signed+for+his+murderer+only+hours+before+his+death+was+put+on+sale+at+a+price+of+%24525%2C000+%28equivalent+to+%24666%2C222+today%29.+%09%09The+most+distinctive+thing+about+Double+Fantasy%2C+the+last+album+John+Lennon+released+during+his+lifetime%2C+is+the+very+thing+that+keeps+it+from+being+a+graceful+return+to+form+from+the+singer%2Fsongwriter%2C+returning+to+active+duty+after+five+years+of+self-imposed+exile.+As+legend+has+it%2C+Lennon+spent+those+years+in+domestic+bliss%2C+being+a+husband%2C+raising+a+baby%2C+and%2C+of+course%2C+baking+bread.+Double+Fantasy+was+designed+as+a+window+into+that+bliss+and%2C+to+that+extent%2C+he+decided+to+make+it+a+joint+album+with+Yoko+Ono%2C+to+illustrate+how+complete+their+union+was.+For+her+part%2C+Ono+decided+to+take+a+stab+at+pop+and+while+these+are+relatively+tuneful+for+her%2C+they+nevertheless+disrupt+the+feel+and+flow+of+Lennon+%27s+material%2C+which+has+a+consistent+tone+and+theme.+He%27s+surprisingly+sentimental%2C+not+just+when+he%27s+expressing+love+for+his+wife+%28%26quot%3BDear+Yoko%2C%26quot%3B+%26quot%3BWoman%26quot%3B%29+and+child+%28%26quot%3BBeautiful+Boy+%5BDarling+Boy%5D%26quot%3B%29%2C+but+when+he%27s+coming+to+terms+with+his+quiet+years+%28%26quot%3BWatching+the+Wheels%2C%26quot%3B+%26quot%3BCleanup+Time%26quot%3B%29+and+his+return+to+creative+life.+These+are+really+nice+tunes%2C+and+what%27s+special+about+them+is+their+niceness+--+it%27s+a+sweet+acceptance+of+middle+age%2C+which%2C+of+course%2C+makes+his+assassination+all+the+sadder.+For+that+alone%2C+Double+Fantasy+is+noteworthy%2C+yet+it%27s+hard+not+to+think+that+it%27s+a+bit+of+a+missed+opportunity+--+primarily+because+its+themes+would+be+stronger+without+the+Ono+songs%2C+but+also+because+the+production+is+just+a+little+bit+too+slick+and+constrained%2C+sounding+very+much+of+its+time.+Ultimately%2C+these+complaints+fall+by+the+wayside+because+Lennon+%27s+best+songs+here+cement+the+last+part+of+his+legend%2C+capturing+him+at+peace+and+in+love.+According+to+some+reports%2C+that+perception+was+a+bit+of+a+fantasy%2C+but+sometimes+the+fantasy+means+more+than+the+reality%2C+and+that%27s+certainly+the+case+here.+%09%09On+a+giant+screen%2C+John+Lennon+and+Yoko+Ono+walk+through+a+chilly+Central+Park%2C+as+the+ex-Beatle+ruminates+over+their+just-released+collaborative+album+Double+Fantasy.+%26quot%3BIt%27s+love+and+a+lot+of+sweat+and+the+life+experience+of+two+people%2C%26quot%3B+he+surmises.+Twelve+days+later%2C+Lennon+was+killed+and+Double+Fantasy+transformed+from+a+poorly+received+comeback+into+a+chart-topping%2C+hit-spawning%2C+Grammy-winning+last+testament.+Now%2C+in+the+last+gig+of+Ono%27s+Meltdown+festival%2C+the+album+is+being+played+live+for+the+first+time+%26ndash%3B+and+as+a+backing+band+and+the+18-strong+Sense+of+Sound+choir+take+to+the+stage%2C+expectation+is+high.+%09%09Moments+in+Time%E2%80%99+s+President%2C+Gary+Zimet%2C+has+announced+that+the+record+album+Double+Fantasy+by+John+Winston+Lennon+autographed+for+Mark+David+Chapman+prior+to+Chapman%E2%80%99s+assassination+of+the+legendary+Beatle+is+being+offered+for+sale+once+again.+The+album%2C+originally+acquired+by+the+gentleman+who+found+it+in+the+front+gate+flower+planter+outside+the+Dakota%2C+home+of+the+late+John+Lennon+in+1980+was+made+famous+when+pictures+of+Lennon+signing+the+album+for+Chapman+appeared+in+the+newspapers+and+on+TV+stations+around+the+world.+A+Beatle+fan+all+of+his+life%2C+this+gentleman+wrestled+for+19+years+before+coming+to+the+decision+to+sell+the+album.+The+album+was+sold+to+a+private+buyer+in+1999+by+Moments+In+Time+Inc+and+is+now+being+offered+once+again+exclusively+here.+The+album+bears+the+signature+of+John+Lennon+and+is+dated+1980.+The+cover+and+dust+jacket+contain+forensically+enhanced+fingerprints+of+Chapman.+This+piece+of+crucial+evidence+against+Chapman+was+turned+into+police+and+then+returned+to+the+owner+with+a+letter+of+extreme+gratitude+from+the+District+Attorney.+%E2%80%9CWe+are+very+excited+to+bring+such+a+historically+significant+piece+to+the+market%E2%80%9D%2C+said+Bob+Zafian%2C+spokesman+for+Moments+in+Time+and+agent+for+the+seller.+%E2%80%9CI+have+never+come+across+a+piece+with+such+provenance%3B+police+reports%2C+fingerprint+documentation%2C+letters+from+the+District+Attorney%2C+it+goes+on+and+on%E2%80%9D.+%09%09Double+Fantasy+is+a+1980+album+released+by+John+Lennon+and+his+wife%2C+Yoko+Ono.+It+is+the+seventh+and+final+studio+album+released+by+Lennon+in+his+lifetime.+Though+initially+poorly+received%2C+the+album+is+notable+for+its+association+with+Lennon%27s+murder+three+weeks+after+its+release%2C+whereupon+it+became+a+worldwide+commercial+success%2C+and+went+on+to+win+the+1981+Album+of+the+Year+at+the+24th+Annual+Grammy+Awards+.+%09%09They+produced+dozens+of+songs%2C+enough+to+fill+Double+Fantasy+and+a+large+part+of+a+projected+second+album%2C+Milk+and+Honey+.+%09%09John+Lennon+wrote+his+last+songs+in+Bermuda+in+1980.+He+named+his+last+album++DOUBLE+FANTASY+after+a+flower+in+the+gardens.+%09%09Double+Fantasy+will+always+hold+a+place+in+music+history+as+it+was+the+last+album+released+during+John+Lennon%27s+lifetime.+It+was+the+first+musical+collaboration+between+John+%26amp%3B+Yoko+since+the+late+60%27s+when+the+released+the+two+avant-garde+Two+Virgins+albums.+The+songs+represent+Mr.+Lennon%27s+first+attempt+at+recording+original+material+since+1974%27s+Walls+%26amp%3B+Bridges+and+they+show+his+typical+autobiographical+songwriting+nature.+%26quot%3B%28Just+Like%29+Starting+Over%26quot%3B+sets+the+tone+of+the+project+and+songs+like+%26quot%3BWoman%26quot%3B%2C+%26quot%3BBeautiful+Boy%26quot%3B%2C+%26quot%3BI%27m+Losing+You%26quot%3B+%26amp%3B+%26quot%3BDear+Yoko%26quot%3B+speak+of+his+family+life+while+%26quot%3BWatching+The+Wheels%26quot%3B+tells+of+his+life+for+the+previous+five+years.+I%27m+not+a+fan+of+the+%26quot%3Bmusic%26quot%3B+Yoko+makes+and+the+songs+here+do+nothing+to+change+that+opinion.+The+real+significance+of+Double+Fantasy+is+that+it+showed+that+John+Lennon+could+still+make+music+that+people+wanted+to+listen+to+and+music+that+still+mattered.+Unfortunately+we+all+know+the+tragedy+that+happened+shortly+after+the+album%27s+release+which+silenced+one+of+the+great+voices+of+a+generation.+Mr.+Lennon+should+be+remember+not+only+as+a+Beatle+or+musician+but+someone+who+took+on+the+problems+of+the+world+head+on+and+wore+his+heart+on+his+sleeve.+The+music+he+made+was+not+always+the+most+artistic%2C+some+of+the+causes+he+took+on+don%27t+seem+worth+the+effort+today+and+the+messages+he+sent+sometimes+seemed+oversimplistic+like+love%2C+peace+and+the+like%2C+but+as+we+approach+the+twenty+anniversary+of+his+passing%2C+we+need+more+artists+who+cared+about+things+as+much+as+he+did+and+actually+tried+to+change+the+world%2C+even+if+it+is+only+in+a+small+way.+%09%09%26quot%3Blove+is+deaf%26quot%3B-+that%27s+the+only+explanantion+for+John+allowing+her+to+sing.+at+least%2C+that%27s+something+you+could+admire.+%5B2%5D+however%2C+double+fantasy+is+the+winner+of+1981+Grammy+Award+for+Album+of+the+Year+%09%09Double+Fantasy-+originally+released+in+1980+and+remastered+in+2010+to+commemorate+what+would+have+been+John+Lennon%27s+70th+birthday%2C+overseen+by+Yoko+Ono.+%09%09To+close+the+festival%2C+Meltdown+presents+a+unique+performance+of+Double+Fantasy-+the+last+album+to+be+written+by+John+Lennon+and+Yoko+Ono+before+John%27s+death%2C+three+weeks+after+the+album%27s+release+in+1980.+%09%09Double+Fantasy+will+always+hold+a+place+in+music+history+as+it+was+the+last+album+released+during+John+Lennon%27s+lifetime.+It+was+the+first+musical+collaboration+between+John+%26amp%3B+Yoko+since+the+late+60%27s+when+the+released+the+two+avant-garde+Two+Virgins+albums.+The+songs+represent+Mr.+Lennon%27s+first+attempt+at+recording+original+material+since+1974%27s+Walls+%26amp%3B+Bridges+and+they+show+his+typical+autobiographical+songwriting+nature.+%26quot%3B%28Just+Like%29+Starting+Over%26quot%3B+sets+the+tone+of+the+project+and+songs+like+%26quot%3BWoman%26quot%3B%2C+%26quot%3BBeautiful+Boy%26quot%3B%2C+%26quot%3BI%27m+Losing+You%26quot%3B+%26amp%3B+%26quot%3BDear+Yoko%26quot%3B+speak+of+his+family+life+while+%26quot%3BWatching+The+Wheels%26quot%3B+tells+of+his+life+for+the+previous+five+years.+I%27m+not+a+fan+of+the+%26quot%3Bmusic%26quot%3B+Yoko+makes+and+the+songs+here+do+nothing+to+change+that+opinion.+The+real+significance+of+Double+Fantasy+is+that+it+showed+that+John+Lennon+could+still+make+music+that+people+wanted+to+listen+to+and+music+that+still+mattered.+Unfortunately+we+all+know+the+tragedy+that+happened+shortly+after+the+album%27s+release+which+silenced+one+of+the+great+voices+of+a+generation.+Mr.+Lennon+should+be+remember+not+only+as+a+Beatle+or+musician+but+someone+who+took+on+the+problems+of+the+world+head+on+and+wore+his+heart+on+his+sleeve.+The+music+he+made+was+not+always+the+most+artistic%2C+some+of+the+causes+he+took+on+don%27t+seem+worth+the+effort+today+and+the+messages+he+sent+sometimes+seemed+oversimplistic+like+love%2C+peace+and+the+like%2C+but+as+we+approach+the+twenty+anniversary+of+his+passing%2C+we+need+more+artists+who+cared+about+things+as+much+as+he+did+and+actually+tried+to+change+the+world%2C+even+if+it+is+only+in+a+small+way.%5D\n",
      "\n",
      "\n",
      "\n",
      "concept:stateorprovince:ust_ordyn_buryat\tgeneralizations\tconcept:stateorprovince\t538\t0.9999999999990905\tMBL-Iter%3A538-2012%2F03%2F25-11%3A33%3A58-From+ErrorBasedIntegrator+%28SEAL%28ust_ordyn_buryat%2Cstateorprovince%29%29\t\"Ust-Ordyn-Buryat\" \t\tUst-Ordyn-Buryat\t\t concept:stateorprovince\t concept:geopoliticallocation concept:geopoliticalorganization concept:geolocatablething\t%5BSEAL-Iter%3A538-2012%2F03%2F24-10%3A09%3A44-%3Ctoken%3Dust_ordyn_buryat%2Cstateorprovince%3E-From%3ACategory%3Astateorprovince-using-KB+http%3A%2F%2Fwww.uglypeopledate.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.britishsinglesonline.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.couplesalike.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.datehouse.co.uk%2F+http%3A%2F%2Fwww.pakchatting.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fintoswing.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.datesofasia.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fswirlsociety.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fthroughafriend.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.inyourcommand.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.meetmysoulmates.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.singleblackmuslim.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.anglomatch.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.bbwaustralia.com%2Fem%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.ojapon.fr%2Findex.php%3Fdll%3Dsearch%26sub%3Dadvanced+http%3A%2F%2Fwww.myecupid.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.uniquelovers.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.singlesonmaui.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Ftruelovejapan.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.arab7ob.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.mapenziplanet.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.foot-worshipers.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.hungerford-singles.co.uk%2Findex1.php%3Fdll%3Dregister+http%3A%2F%2Fwww.hitchhikers.co%2Findex.php+http%3A%2F%2Fwww.guys4now.net%2Fregister.php+http%3A%2F%2Fwww.filipinoloveonline.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.crewdating.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.keyholderfinder.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.shaadionline.pk%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.keyholderfinder.com%2Findex.php%3Fdll%3Dsearch%26sub%3Dadvanced+http%3A%2F%2Fwww.datingromanticfriends.com%2Findex.php%3Fdll%3Dsearch%26sub%3Dadvanced+http%3A%2F%2Fwww.singleswithdreadlocks.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.ulnsf.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fanrdating.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.hindupunjabimatch.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2F2befriends.net%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.africanmatch.net%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Finteractive.com%2Ficomee-regs%2Flogin%2Fnewfreetrial.jsp+http%3A%2F%2Fwww.muslimknot.com%2Findex.php%3Fdll%3Dregister+http%3A%2F%2Fwww.afrobbwdating.com%2Findex.php%3Fdll%3Dregister%2C+OE-Iter%3A811-2014%2F02%2F08-11%3A37%3A04-%3Ctoken%3Dust_ordyn_buryat%2Cstateorprovince%3E-Irkutsk+%2836%25%29%2C+Krasnoyarsk+%2829%25%29%2C+Tuva+%2819%25%29%2C+Buryatia+%287%25%29%2C+Khakassia+%285%25%29%2C+Ust-Ordyn-Buryat+%281%25%29+%09%09Administratively%2C+the+Federation+has+generally+relied+on+regional+divisions+inherited+from+the+Stalin+and+Brezhnev+constitutions+of+1936+and+1977.+Each+area+with+a+predominantly+Russian+population+is+constituted+as+a+territory+%28kray%29+or+region+%28oblast%29%3B+some+non-Russian+nationalities+are+constituted%2C+in+descending+order+of+importance%2C+as+republics%2C+autonomous+regions+%28oblasts%29%2C+and+autonomous+areas+%28okrugs%29.+Russia+has+21+republics%3A+Adygey%2C+Altai%2C+Bashkortostan%2C+Buryat%2C+Chechnya%2C+Chuvash%2C+Dagestan%2C+Ingushetia%2C+Kabardino-Balkar%2C+Kalmykia%2C+Karachay-Cherkess%2C+Karelia%2C+Khakass%2C+Komi%2C+Mari+El%2C+Mordovia%2C+North+Ossetia-Alania%2C+Sakha%2C+Tatarstan%2C+Tuva%2C+and+Udmurt%3B+one+autonomous+region+%28or+oblast%29%3A+Jewish+%28Birobidzhan%29%3B+six+autonomous+national+areas+%28okrugs%29%3A+Agin-Buryat%2C+Chukotka%2C+Khanty-Mansi%2C+Nenets%2C+Ust-Ordyn-Buryat%2C+and+Yamalo-Nenets%3B+47+Russian+regions+%28oblasts%3B+the+cities+of+Moscow+and+St.+Petersburg+also+have+oblast+status%29%3B+and+eight+Russian+territories+%28krays%29.+Oblasts+and+krays+are+roughly+equivalent+to+provinces.+In+addition+to+Moscow%2C+other+major+urban+areas+in+Russia+include+Saint+Petersburg+%28formerly+Leningrad%29%2C+Nizhny+Novgorod+%28formerly+Gorky%29%2C+Rostov-na-Donu%2C+Volgograd%2C+Kazan%2C+Samara+%28formerly+Kuybyshev%29%2C+Ufa%2C+Perm%2C+Yekaterinburg+%28formerly+Sverdlovsk%29%2C+Omsk%2C+Chelyabinsk%2C+Novosibirsk%2C+and+Vladivostok.+%09%0985-+Ust-Ordyn-Buryat+autonomous+area%5D\n",
      "\n",
      "len(equivalentSet): 1969\n",
      "FINISHED\n",
      "EXECUTION TIME: 0.00585699081421 s\n"
     ]
    }
   ],
   "source": [
    "# read equivalent classes\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 1000\n",
    "\n",
    "\n",
    "try:\n",
    "    fr = open('NELL/linesContainingEquivalent.txt', 'r')  \n",
    "    for line in fr:\n",
    "        print line\n",
    "        lineCounter += 1\n",
    "        if lineCounter > 10:\n",
    "            break\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    fr.close()\n",
    "\n",
    "    print ('FINISHED')\n",
    "except:\n",
    "    print ('ERROR')\n",
    "    \n",
    "end = time.time()        \n",
    "print ('EXECUTION TIME: {} s'.format(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000 lines read\n",
      "200000 lines read\n",
      "300000 lines read\n",
      "400000 lines read\n",
      "500000 lines read\n",
      "600000 lines read\n",
      "700000 lines read\n",
      "1184 classes found\n",
      "FINISHED\n",
      "EXECUTION TIME: 8.07979989052 s\n"
     ]
    }
   ],
   "source": [
    "# get all classes (\"concept:\")\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "ontologySample = 'NELL/NELL_ontology_sample.csv'\n",
    "ontology = 'NELL/NELL.08m.995.ontology.csv'\n",
    "\n",
    "# CHANGE HERE FOR FULL OR SAMPLE ONTOLOGY\n",
    "readOntology = ontology\n",
    "\n",
    "conceptArray = []\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 100000\n",
    "#maxLines = 1000000\n",
    "\n",
    "try:\n",
    "    # read ontology\n",
    "    fr = open(readOntology, 'r')\n",
    "    \n",
    "    for line in fr:\n",
    "        splitted_line = line.rstrip('\\n').split('\\t') #remove line break in the end, split line on tab\n",
    "        #position_counter = 0\n",
    "        #print splitted_line\n",
    "        for word in splitted_line:\n",
    "#### concept:book\tmemberofsets\tconcept:rtwcategory: check for rtwcategory?            \n",
    "            if word.startswith('concept:'): # check for concepts\n",
    "                if not word in conceptArray:\n",
    "                    conceptArray.append(word)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    #print conceptArray\n",
    "    print ('{} classes found'.format(len(conceptArray)))\n",
    "    fr.close()\n",
    "    print ('FINISHED')\n",
    "except:\n",
    "    print ('ERROR')\n",
    "    \n",
    "end = time.time()        \n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "500000 lines read\n",
      "1000000 lines read\n",
      "1500000 lines read\n",
      "2000000 lines read\n",
      "2500000 lines read\n",
      "{'hasCat': 1974586, 'noCat': 167399}\n",
      "len(hasCat): 1974586\n",
      "DONE\n",
      "EXECUTION TIME: 21.2120959759 s\n"
     ]
    }
   ],
   "source": [
    "# check how many entities have the \"categories for entity\" attribute\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "readNell = nell\n",
    "\n",
    "\n",
    "countCat = {} #key: value, value: count\n",
    "countCat['hasCat'] = 0\n",
    "countCat['noCat'] = 0\n",
    "\n",
    "hasCat = set()\n",
    "seenEntities = set()\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 500000\n",
    "\n",
    "def checkWords(splittedLine):\n",
    "    word_position = 0\n",
    "    #newEntity = False\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity\n",
    "            entity = word\n",
    "        if (word_position==10): # categories for entity\n",
    "            categories = word\n",
    "            if entity in seenEntities:\n",
    "                return\n",
    "            else:\n",
    "                seenEntities.add(entity)\n",
    "                if (categories == ''):\n",
    "                    countCat['noCat'] += 1\n",
    "                    return\n",
    "                else:\n",
    "                    countCat['hasCat'] += 1\n",
    "                    hasCat.add(entity)\n",
    "                    #print ('empty cat')\n",
    "                    return # break function\n",
    "        word_position += 1\n",
    "    \n",
    "\n",
    "print('START')\n",
    "try:\n",
    "    # read nell\n",
    "    fn = open(readNell, 'r')\n",
    "    for line in fn:\n",
    "        if (lineCounter != 0):\n",
    "            splitted_line = line.rstrip('\\n').split('\\t')\n",
    "            checkWords(splitted_line)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    print countCat\n",
    "    print ('len(hasCat): {}'.format(len(hasCat)))\n",
    "    fn.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "500000 lines read\n",
      "1000000 lines read\n",
      "1500000 lines read\n",
      "2000000 lines read\n",
      "2500000 lines read\n",
      "len(allEntities): 2141985\n",
      "len(hasGeneralization): 1974586\n",
      "len(hasCat): 1974586\n",
      "# of different items in hasGeneralization and hasCat: 0\n",
      "DONE\n",
      "EXECUTION TIME: 21.9871029854 s\n"
     ]
    }
   ],
   "source": [
    "# check how many entities have the \"generalizations\" relation & \n",
    "# check if they are the same that have the categories for entity attribute\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "readNell = nell\n",
    "\n",
    "\n",
    "countCat = {} #key: value, value: count\n",
    "countCat['hasGenRel'] = 0\n",
    "countCat['noGenRel'] = 0\n",
    "\n",
    "allEntities = set()\n",
    "hasGeneralization = set()\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 500000\n",
    "\n",
    "def checkWords(splittedLine):\n",
    "    word_position = 0\n",
    "    #newEntity = False\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity\n",
    "            entity = word\n",
    "            if entity not in allEntities:\n",
    "                # add to list of all entities\n",
    "                allEntities.add(entity)                \n",
    "        if (word_position==1): #relation\n",
    "            relation = word\n",
    "            if (relation == \"generalizations\"):\n",
    "                hasGeneralization.add(entity)\n",
    "            else:\n",
    "                return\n",
    "        word_position += 1\n",
    "    \n",
    "\n",
    "print('START')\n",
    "try:\n",
    "    # read nell\n",
    "    fn = open(readNell, 'r')\n",
    "    for line in fn:\n",
    "        if (lineCounter != 0):\n",
    "            splitted_line = line.rstrip('\\n').split('\\t')\n",
    "            checkWords(splitted_line)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    print ('len(allEntities): {}'.format(len(allEntities)))\n",
    "    print ('len(hasGeneralization): {}'.format(len(hasGeneralization)))\n",
    "    print ('len(hasCat): {}'.format(len(hasCat)))\n",
    "    #find different items in both sets\n",
    "    print('# of different items in hasGeneralization and hasCat: {}'.format(len(hasGeneralization-hasCat)))\n",
    "    fn.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "1000000 lines read\n",
      "2000000 lines read\n",
      "[('concept:beach', 34501), ('concept:actor', 29293), ('concept:governmentorganization', 27738), ('concept:director', 27569), ('concept:skiarea', 27356), ('concept:street', 25287), ('concept:river', 23763), ('concept:airport', 23719), ('concept:celebrity', 23424), ('concept:male', 23277)]\n",
      "DONE\n",
      "EXECUTION TIME: 19.7327260971 s\n"
     ]
    }
   ],
   "source": [
    "# \"generalizations\"-relation attribute\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "readNell = nell\n",
    "\n",
    "#header of files: Entity\tRelation\tValue\tIteration of Promotion\tProbability\tSource\tEntity literalStrings\tValue literalStrings\tBest Entity literalString\tBest Value literalString\tCategories for Entity\tCategories for Value\tCandidate Source\n",
    "\n",
    "genValueCount = {} #key: value, value: count\n",
    "#seenEntities = set()\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 1000000\n",
    "\n",
    "print('START')\n",
    "try:\n",
    "    # read nell\n",
    "    fn = open(readNell, 'r')\n",
    "    for line in fn:\n",
    "        genFound = False\n",
    "        lineWords = []\n",
    "        splitted_line = line.rstrip('\\n').split('\\t')\n",
    "        word_position = 0\n",
    "        catCandidate = \"\"\n",
    "        for word in splitted_line:\n",
    "            #if (word_position==0): #Entity\n",
    "            #    if word not in seenEntities:\n",
    "            #        seenEntities.add(word)\n",
    "            if (word_position==1): #relation\n",
    "                # get all values when the relation is 'generalizations'\n",
    "                if (word == \"generalizations\"):\n",
    "                    genFound = True\n",
    "            if (word_position==2 and genFound): #value\n",
    "                catCandidate = word\n",
    "###\n",
    "# WITHOUT PROBABILITY\n",
    "                if catCandidate not in genValueCount:\n",
    "                    genValueCount[catCandidate] = 0\n",
    "                else:\n",
    "                    genValueCount[catCandidate] += 1\n",
    "# END WITHOUT PROBABILITY\n",
    "###\n",
    "###\n",
    "# WITH PROBABILITY                \n",
    "#            if (word_position==4 and genFound): #Probability\n",
    "#                prob = float(word)\n",
    "#                if prob==1:\n",
    "#                    if catCandidate not in genValueCount:\n",
    "#                        genValueCount[catCandidate] = 0\n",
    "#                    else:\n",
    "#                        genValueCount[catCandidate] += 1\n",
    "# END WITH PROBABILITY\n",
    "###\n",
    "            word_position += 1\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    # sorted list for generalization count\n",
    "    top10dict = dict(sorted(genValueCount.items(), key=operator.itemgetter(1), reverse=True)[:10])\n",
    "    print (sorted(top10dict.items(), key=operator.itemgetter(1), reverse=True))\n",
    "    fn.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "1000000 lines read\n",
      "2000000 lines read\n",
      "[('concept:beach', 34501), ('concept:actor', 29293), ('concept:governmentorganization', 27738), ('concept:director', 27569), ('concept:skiarea', 27356), ('concept:street', 25287), ('concept:river', 23763), ('concept:airport', 23719), ('concept:celebrity', 23424), ('concept:male', 23277)]\n",
      "DONE\n",
      "EXECUTION TIME: 19.2660820484 s\n"
     ]
    }
   ],
   "source": [
    "# categories for entity\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "readNell = nell\n",
    "\n",
    "#header of files: Entity\tRelation\tValue\tIteration of Promotion\tProbability\tSource\tEntity literalStrings\tValue literalStrings\tBest Entity literalString\tBest Value literalString\tCategories for Entity\tCategories for Value\tCandidate Source\n",
    "\n",
    "\n",
    "categoryCount = {} #key: category, value: count\n",
    "genValueCount = {} #key: value, value: count\n",
    "seenEntities = set()\n",
    "\n",
    "lineCounter = 0\n",
    "lineProgress = 1000000\n",
    "\n",
    "def checkWords(splittedLine):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity\n",
    "            entity = word\n",
    "        # WITH PROBABILITY    \n",
    "        #if (word_position==4): #Probability\n",
    "        #    prob = float(word)\n",
    "        #    if (prob < 1):\n",
    "        #        return\n",
    "        if (word_position==10): # categories for entity\n",
    "            categories = word\n",
    "            if (categories == ''):\n",
    "                return # break function\n",
    "            else:\n",
    "                if entity in seenEntities:\n",
    "                    return #break function\n",
    "                else:\n",
    "                    seenEntities.add(entity)\n",
    "                    for category in categories.split(): #split on whitespace for multiple categories\n",
    "                        if category not in categoryCount:\n",
    "                            categoryCount[category] = 0\n",
    "                        else:\n",
    "                            categoryCount[category] += 1\n",
    "        word_position += 1\n",
    "    \n",
    "print('START')\n",
    "try:\n",
    "    # read nell\n",
    "    fn = open(readNell, 'r')\n",
    "    for line in fn:\n",
    "        if (lineCounter != 0):\n",
    "            splitted_line = line.rstrip('\\n').split('\\t')\n",
    "            checkWords(splitted_line)\n",
    "        lineCounter += 1\n",
    "        if (lineCounter % lineProgress == 0):\n",
    "            print ('{} lines read'.format(lineCounter))\n",
    "    top10dictCat = dict(sorted(categoryCount.items(), key=operator.itemgetter(1), reverse=True)[:10])\n",
    "    print (sorted(top10dictCat.items(), key=operator.itemgetter(1), reverse=True))\n",
    "    fn.close()\n",
    "    print ('DONE')\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TOP 10 classes without probability\n",
    "\n",
    "#[('concept:beach', 34501), ('concept:actor', 29293),\n",
    "#('concept:governmentorganization', 27738), ('concept:director', 27569),\n",
    "#('concept:skiarea', 27356), ('concept:street', 25287),\n",
    "#('concept:river', 23763), ('concept:airport', 23719),\n",
    "#('concept:celebrity', 23424), ('concept:male', 23277)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "read ontology\n",
      "500000 lines read\n",
      "read NELL\n",
      "500000 lines read\n",
      "1000000 lines read\n",
      "1500000 lines read\n",
      "2000000 lines read\n",
      "2500000 lines read\n",
      "DONE\n",
      "INDEGREE RESULT:\n",
      "[('concept:beach', 34548), ('concept:actor', 29319), ('concept:governmentorganization', 27742), ('concept:director', 27593), ('concept:skiarea', 27369), ('concept:street', 25352), ('concept:river', 23823), ('concept:airport', 23742), ('concept:celebrity', 23436), ('concept:male', 23291), ('concept:chemical', 22440), ('concept:athlete', 19708), ('concept:city', 18789), ('concept:musicsong', 12830), ('concept:politician', 9172), ('concept:militaryconflict', 9038), ('concept:politicalparty', 8888), ('concept:book', 8532), ('concept:movie', 6646), ('concept:ethnicgroup', 6430), ('concept:company', 6190), ('concept:sport', 6154), ('concept:vehicle', 6149), ('concept:country', 6026), ('concept:software', 5740), ('concept:geopoliticallocation', 5707), ('concept:person', 5593), ('concept:visualizablething', 5444), ('concept:musicalbum', 5379), ('concept:sportsevent', 2845), ('concept:time', 2752), ('concept:visualizableobject', 2499), ('concept:building', 1334), ('concept:event', 324), ('concept:location', 317), ('concept:creativework', 294), ('concept:planet', 263), ('concept:agent', 216), ('concept:geopoliticalentity', 205), ('concept:militaryeventtype', 170), ('concept:species', 145), ('concept:abstractthing', 104), ('concept:humanagent', 30)]\n",
      "OUTDEGREE RESULT\n",
      "[('concept:city', 101), ('concept:country', 100), ('concept:agent', 100), ('concept:location', 96), ('concept:airport', 96), ('concept:river', 87), ('concept:company', 87), ('concept:building', 86), ('concept:street', 86), ('concept:humanagent', 85), ('concept:vehicle', 83), ('concept:chemical', 83), ('concept:director', 82), ('concept:athlete', 81), ('concept:skiarea', 80), ('concept:software', 80), ('concept:beach', 80), ('concept:politician', 80), ('concept:militaryconflict', 79), ('concept:sportsevent', 79), ('concept:militaryeventtype', 79), ('concept:actor', 78), ('concept:geopoliticallocation', 77), ('concept:planet', 77), ('concept:governmentorganization', 73), ('concept:politicalparty', 72), ('concept:musicalbum', 71), ('concept:movie', 71), ('concept:book', 70), ('concept:musicsong', 69), ('concept:celebrity', 68), ('concept:male', 64), ('concept:event', 63), ('concept:sport', 63), ('concept:ethnicgroup', 63), ('concept:person', 63), ('concept:time', 62), ('concept:creativework', 62), ('concept:visualizableobject', 62), ('concept:geopoliticalentity', 61), ('concept:species', 60), ('concept:visualizablething', 16), ('concept:abstractthing', 7)]\n",
      "EXECUTION TIME: 22.3544209003 s\n"
     ]
    }
   ],
   "source": [
    "# class indegree & outdegree\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "ontology = 'NELL/NELL.08m.995.ontology.csv'\n",
    "lineProgress = 500000\n",
    "\n",
    "indegreeDic = {} #key:entity, value:indegree\n",
    "outdegreeDic = {} #key:entity, value:outdegree\n",
    "topClasses = set()\n",
    "#topClasses.update(['concept:beach','concept:actor','concept:governmentorganization','concept:director','concept:skiarea','concept:street','concept:river','concept:airport', 'concept:celebrity','concept:male'])\n",
    "topClasses.update(['concept:visualizableobject',\n",
    "                   'concept:visualizablething',\n",
    "                   'concept:species',\n",
    "                   'concept:humanagent',\n",
    "                   'concept:agent',\n",
    "                   'concept:person',\n",
    "                   'concept:athlete',\n",
    "                   'concept:actor',\n",
    "                   'concept:director',\n",
    "                   'concept:celebrity',\n",
    "                   'concept:male',\n",
    "                   'concept:ethnicgroup',\n",
    "                   'concept:governmentorganization',\n",
    "                   'concept:company',\n",
    "                   'concept:location',\n",
    "                   'concept:geopoliticalentity',\n",
    "                   'concept:geopoliticallocation',\n",
    "                   'concept:city',\n",
    "                   'concept:street',\n",
    "                   'concept:beach',\n",
    "                   'concept:skiarea',\n",
    "                   'concept:river',\n",
    "                   'concept:airport',\n",
    "                   'concept:abstractthing',\n",
    "                   'concept:creativework',\n",
    "                   'concept:time',\n",
    "                   'concept:politician',\n",
    "                   'concept:politicalparty',\n",
    "                   'concept:musicalbum',\n",
    "                   'concept:musicsong',\n",
    "                   'concept:movie',\n",
    "                   'concept:book',\n",
    "                   'concept:software',\n",
    "                   'concept:sport',\n",
    "                   'concept:event',\n",
    "                   'concept:militaryeventtype',\n",
    "                   'concept:militaryconflict',\n",
    "                   'concept:sportsevent',\n",
    "                   'concept:country',\n",
    "                   'concept:building',\n",
    "                   'concept:chemical',\n",
    "                   'concept:planet',\n",
    "                   'concept:vehicle',\n",
    "                  ])\n",
    "\n",
    "def calcDegree(splitted_line):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity OUTDEGREE\n",
    "            entity = word\n",
    "            if entity in topClasses:\n",
    "                if entity in outdegreeDic:\n",
    "                    outdegreeDic[entity] += 1\n",
    "                else:\n",
    "                    outdegreeDic[entity] = 1\n",
    "                    #return\n",
    "            #else: #entity not in top classes\n",
    "                #return\n",
    "        elif (word_position==2): #Value INDEGREE\n",
    "            value = word\n",
    "            if value in topClasses:\n",
    "                if value in indegreeDic:\n",
    "                    indegreeDic[value] += 1\n",
    "                else:\n",
    "                    indegreeDic[value] = 1\n",
    "                    return\n",
    "            else: #entity not in top classes\n",
    "                return\n",
    "        elif(word_position>2):\n",
    "            break\n",
    "        word_position +=1\n",
    "\n",
    "print('START')\n",
    "try:\n",
    "    # read nell & ontology\n",
    "    for i in xrange(2):\n",
    "        if (i==0):\n",
    "            print ('read ontology')\n",
    "            readFile = ontology\n",
    "        elif (i==1):\n",
    "            print ('read NELL')\n",
    "            readFile = nell\n",
    "        else:\n",
    "            print ('ERROR for xrange')\n",
    "        fn = open(readFile, 'r')\n",
    "        lineCounter = 0\n",
    "        for line in fn:\n",
    "            if (lineCounter != 0):\n",
    "                splitted_line = line.rstrip('\\n').split('\\t')\n",
    "                calcDegree(splitted_line)\n",
    "            lineCounter += 1\n",
    "            if (lineCounter % lineProgress == 0):\n",
    "                print ('{} lines read'.format(lineCounter))\n",
    "        fn.close()\n",
    "    print ('DONE')\n",
    "    print ('INDEGREE RESULT:')\n",
    "    print sorted(indegreeDic.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    print ('OUTDEGREE RESULT')\n",
    "    print sorted(outdegreeDic.items(), key=operator.itemgetter(1), reverse=True)\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "1. run: get class instances\n",
      "read ontology\n",
      "read NELL\n",
      "1000000 lines read\n",
      "2000000 lines read\n",
      "2. run: count instance degrees\n",
      "read ontology\n",
      "read NELL\n",
      "1000000 lines read\n",
      "2000000 lines read\n",
      "calculate class degrees\n",
      "DONE\n",
      "RESULTS\n",
      "class instances indegree for beach\n",
      "min: 1, avg: 1.58333333333, median: 1.0, max: 4\n",
      "class instances indegree for actor\n",
      "min: 1, avg: 1.3, median: 1.0, max: 7\n",
      "class instances indegree for governmentorganization\n",
      "min: 1, avg: 1.8835978836, median: 1.0, max: 49\n",
      "class instances indegree for director\n",
      "min: 1, avg: 1.81481481481, median: 1.0, max: 8\n",
      "class instances indegree for skiarea\n",
      "min: 1, avg: 1.89928057554, median: 1.0, max: 15\n",
      "class instances indegree for street\n",
      "min: 1, avg: 1.66666666667, median: 1.0, max: 3\n",
      "class instances indegree for river\n",
      "min: 1, avg: 4.30039525692, median: 2.0, max: 93\n",
      "class instances indegree for airport\n",
      "min: 1, avg: 1.69230769231, median: 1.0, max: 7\n",
      "class instances indegree for celebrity\n",
      "min: 1, avg: 2.35862068966, median: 1.0, max: 31\n",
      "class instances indegree for male\n",
      "min: 1, avg: 2.43521594684, median: 1.0, max: 50\n",
      "class instances outdegree for beach\n",
      "min: 1, avg: 1.02625934728, median: 1.0, max: 6\n",
      "class instances outdegree for actor\n",
      "min: 1, avg: 2.87106574725, median: 3.0, max: 20\n",
      "class instances outdegree for governmentorganization\n",
      "min: 1, avg: 1.09812898807, median: 1.0, max: 64\n",
      "class instances outdegree for director\n",
      "min: 1, avg: 1.33289453889, median: 1.0, max: 12\n",
      "class instances outdegree for skiarea\n",
      "min: 1, avg: 1.8337304351, median: 2.0, max: 11\n",
      "class instances outdegree for street\n",
      "min: 1, avg: 1.5068411895, median: 1.0, max: 4\n",
      "class instances outdegree for river\n",
      "min: 1, avg: 1.13555063708, median: 1.0, max: 36\n",
      "class instances outdegree for airport\n",
      "min: 1, avg: 1.42299422404, median: 1.0, max: 9\n",
      "class instances outdegree for celebrity\n",
      "min: 1, avg: 2.28725937686, median: 2.0, max: 64\n",
      "class instances outdegree for male\n",
      "min: 1, avg: 2.39131253531, median: 2.0, max: 56\n",
      "EXECUTION TIME: 30.768807888 s\n"
     ]
    }
   ],
   "source": [
    "# class instances indegree & outdegree\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "import numpy\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "ontology = 'NELL/NELL.08m.995.ontology.csv'\n",
    "\n",
    "lineProgress = 1000000\n",
    "\n",
    "#topClasses = set()\n",
    "#topClasses.update(['concept:beach','concept:actor','concept:governmentorganization','concept:director','concept:skiarea','concept:street','concept:river','concept:airport', 'concept:celebrity','concept:male'])\n",
    "\n",
    "# sets for each class to hold all instances\n",
    "allInstances = set()\n",
    "instancesBeach = set()\n",
    "instancesActor = set()\n",
    "instancesGovOrg = set()\n",
    "instancesDirector = set()\n",
    "instancesSkiArea = set()\n",
    "instancesStreet = set()\n",
    "instancesRiver = set()\n",
    "instancesAirport = set()\n",
    "instancesCelebrity = set()\n",
    "instancesMale = set()\n",
    "\n",
    "class ClassInstances:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.allClassInstances = set()\n",
    "        self.countDict = dict()\n",
    "        self.min = 9999999\n",
    "        self.max = 0\n",
    "        self.avg = 0.0\n",
    "        self.median = 0.0\n",
    "    def addInstance(self, instance):\n",
    "        if instance in self.allClassInstances:\n",
    "            self.countDict[instance] += 1\n",
    "        else:\n",
    "            self.countDict[instance] = 1\n",
    "            self.allClassInstances.add(instance)\n",
    "    def calculateDegrees(self):\n",
    "        allValueList = []\n",
    "        for k,v in self.countDict.iteritems():\n",
    "            allValueList.append(v)\n",
    "            if (v < self.min):\n",
    "                self.min = v\n",
    "            if (v > self.max):\n",
    "                self.max = v\n",
    "            self.avg += v\n",
    "        self.avg = self.avg / len(self.countDict)\n",
    "        self.median = numpy.median(numpy.array(allValueList))\n",
    "    def printResults(self):\n",
    "        print ('{}'.format(self.name))\n",
    "        print ('min: {}, avg: {}, median: {}, max: {}'.format(self.min, self.avg, self.median, self.max))\n",
    "        \n",
    "\n",
    "# instantiation of all class instances for indegree          \n",
    "beachCi = ClassInstances('class instances indegree for beach')\n",
    "actorCi = ClassInstances('class instances indegree for actor')\n",
    "govOrgCi = ClassInstances('class instances indegree for governmentorganization')\n",
    "directorCi = ClassInstances('class instances indegree for director')\n",
    "skiAreaCi = ClassInstances('class instances indegree for skiarea')\n",
    "streetCi = ClassInstances('class instances indegree for street')\n",
    "riverCi = ClassInstances('class instances indegree for river')\n",
    "airportCi = ClassInstances('class instances indegree for airport')\n",
    "celebrityCi = ClassInstances('class instances indegree for celebrity')\n",
    "maleCi = ClassInstances('class instances indegree for male')\n",
    "# list with references to all indegree classes\n",
    "classInstancesIndegree = [beachCi, actorCi, govOrgCi, directorCi, skiAreaCi, streetCi, riverCi, airportCi, celebrityCi, maleCi]\n",
    "\n",
    "# instantiation of all class instances for outdegree          \n",
    "beachCo = ClassInstances('class instances outdegree for beach')\n",
    "actorCo = ClassInstances('class instances outdegree for actor')\n",
    "govOrgCo = ClassInstances('class instances outdegree for governmentorganization')\n",
    "directorCo = ClassInstances('class instances outdegree for director')\n",
    "skiAreaCo = ClassInstances('class instances outdegree for skiarea')\n",
    "streetCo = ClassInstances('class instances outdegree for street')\n",
    "riverCo = ClassInstances('class instances outdegree for river')\n",
    "airportCo = ClassInstances('class instances outdegree for airport')\n",
    "celebrityCo = ClassInstances('class instances outdegree for celebrity')\n",
    "maleCo = ClassInstances('class instances outdegree for male')\n",
    "# list with references to all outdegree classes\n",
    "classInstancesOutdegree = [beachCo, actorCo, govOrgCo, directorCo, skiAreaCo, streetCo, riverCo, airportCo, celebrityCo, maleCo]\n",
    "\n",
    "def getInstances(splitted_line):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity \n",
    "            entity = word\n",
    "        elif (word_position==1): #Relation\n",
    "            relation = word\n",
    "        elif (word_position==2): #Value\n",
    "            value = word\n",
    "            if (relation == 'generalizations' and value in topClasses):\n",
    "                # check for specific class and add instance to the class set\n",
    "                if (value == 'concept:beach'):\n",
    "                    if entity not in instancesBeach:\n",
    "                        instancesBeach.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:actor'):\n",
    "                    if entity not in instancesActor:\n",
    "                        instancesActor.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:governmentorganization'):\n",
    "                    if entity not in instancesGovOrg:\n",
    "                        instancesGovOrg.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:director'):\n",
    "                    if entity not in instancesDirector:\n",
    "                        instancesDirector.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:skiarea'):\n",
    "                    if entity not in instancesSkiArea:\n",
    "                        instancesSkiArea.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:street'):\n",
    "                    if entity not in instancesStreet:\n",
    "                        instancesStreet.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:river'):\n",
    "                    if entity not in instancesRiver:\n",
    "                        instancesRiver.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:airport'):\n",
    "                    if entity not in instancesAirport:\n",
    "                        instancesAirport.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:celebrity'):\n",
    "                    if entity not in instancesCelebrity:\n",
    "                        instancesCelebrity.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:male'):\n",
    "                    if entity not in instancesMale:\n",
    "                        instancesMale.add(entity)\n",
    "                    return\n",
    "                return\n",
    "            else: # NOT relation == 'generalizations' and value in topClasses\n",
    "                return                \n",
    "        elif(word_position>2):\n",
    "            return\n",
    "        word_position +=1\n",
    "\n",
    "def countInstanceDegrees(splitted_line):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity: OUTDEGREE\n",
    "            entity = word\n",
    "            if entity in allInstances:\n",
    "                if entity in instancesBeach:\n",
    "                    beachCo.addInstance(entity)\n",
    "                elif entity in instancesActor:\n",
    "                    actorCo.addInstance(entity)\n",
    "                elif entity in instancesGovOrg:\n",
    "                    govOrgCo.addInstance(entity)\n",
    "                elif entity in instancesDirector:\n",
    "                    directorCo.addInstance(entity)\n",
    "                elif entity in instancesSkiArea:\n",
    "                    skiAreaCo.addInstance(entity)\n",
    "                elif entity in instancesStreet:\n",
    "                    streetCo.addInstance(entity)\n",
    "                elif entity in instancesRiver:\n",
    "                    riverCo.addInstance(entity)\n",
    "                elif entity in instancesAirport:\n",
    "                    airportCo.addInstance(entity)\n",
    "                elif entity in instancesCelebrity:\n",
    "                    celebrityCo.addInstance(entity)\n",
    "                elif entity in instancesMale:\n",
    "                    maleCo.addInstance(entity)\n",
    "        \n",
    "        elif (word_position==2): #Value: INDEGREE\n",
    "            value = word\n",
    "            if value in allInstances:\n",
    "                if value in instancesBeach:\n",
    "                    beachCi.addInstance(value)\n",
    "                elif value in instancesActor:\n",
    "                    actorCi.addInstance(value)\n",
    "                elif value in instancesGovOrg:\n",
    "                    govOrgCi.addInstance(value)\n",
    "                elif value in instancesDirector:\n",
    "                    directorCi.addInstance(value)\n",
    "                elif value in instancesSkiArea:\n",
    "                    skiAreaCi.addInstance(value)\n",
    "                elif value in instancesStreet:\n",
    "                    streetCi.addInstance(value)\n",
    "                elif value in instancesRiver:\n",
    "                    riverCi.addInstance(value)\n",
    "                elif value in instancesAirport:\n",
    "                    airportCi.addInstance(value)\n",
    "                elif value in instancesCelebrity:\n",
    "                    celebrityCi.addInstance(value)\n",
    "                elif value in instancesMale:\n",
    "                    maleCi.addInstance(value)                \n",
    "            return # done processing entity and value\n",
    "        elif(word_position>2):\n",
    "            return\n",
    "        word_position +=1\n",
    "        \n",
    "def calculateClassDegrees():\n",
    "    for item in classInstancesIndegree:\n",
    "        item.calculateDegrees()\n",
    "    for item in classInstancesOutdegree:\n",
    "        item.calculateDegrees()\n",
    "        \n",
    "def printClassDegreeResults():\n",
    "    for item in classInstancesIndegree:\n",
    "        item.printResults()\n",
    "    for item in classInstancesOutdegree:\n",
    "        item.printResults()\n",
    "    \n",
    "print('START')\n",
    "try:\n",
    "    # get Instances and calculate indegree&outdegree\n",
    "    for i in xrange(2):\n",
    "        if (i==0):\n",
    "            print ('1. run: get class instances')\n",
    "        elif (i==1):\n",
    "            print ('2. run: count instance degrees')\n",
    "        # read nell & ontology\n",
    "        for j in xrange(2):\n",
    "            if (j==0):\n",
    "                print ('read ontology')\n",
    "                readFile = ontology\n",
    "            elif (j==1):\n",
    "                print ('read NELL')\n",
    "                readFile = nell\n",
    "            fn = open(readFile, 'r')\n",
    "            lineCounter = 0\n",
    "            for line in fn:\n",
    "                if (lineCounter != 0):\n",
    "                    splitted_line = line.rstrip('\\n').split('\\t')\n",
    "                    if (i==0):\n",
    "                        getInstances(splitted_line)\n",
    "                    elif (i==1):\n",
    "                        countInstanceDegrees(splitted_line)\n",
    "                    else:\n",
    "                        calculateClassDegrees(splitted_line)\n",
    "                lineCounter += 1\n",
    "                if (lineCounter % lineProgress == 0):\n",
    "                    print ('{} lines read'.format(lineCounter))\n",
    "            fn.close()\n",
    "        if (i==0):\n",
    "            # create set of all instances\n",
    "            allInstances = instancesBeach|instancesActor|instancesGovOrg|instancesDirector|instancesSkiArea|instancesStreet|instancesRiver|instancesAirport|instancesCelebrity|instancesMale\n",
    "    # calculate min, avg, max class degrees\n",
    "    print ('calculate class degrees')\n",
    "    calculateClassDegrees()\n",
    "    \n",
    "    print ('DONE')\n",
    "    print('RESULTS')\n",
    "    printClassDegreeResults()\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n",
      "1. run: get class instances\n",
      "read ontology\n",
      "read NELL\n",
      "1000000 lines read\n",
      "2000000 lines read\n",
      "2. run: count instance degrees\n",
      "read ontology\n",
      "ERROR\n",
      "EXECUTION TIME: 23.5431351662 s\n"
     ]
    }
   ],
   "source": [
    "# class instances indegree & outdegree for consolidated list\n",
    "\n",
    "import os\n",
    "import time\n",
    "import string\n",
    "import itertools\n",
    "import re\n",
    "import operator\n",
    "import numpy\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "ontology = 'NELL/NELL.08m.995.ontology.csv'\n",
    "\n",
    "lineProgress = 1000000\n",
    "\n",
    "topClasses = set()\n",
    "topClasses.update(['concept:visualizableobject',\n",
    "                   'concept:visualizablething',\n",
    "                   'concept:species',\n",
    "                   'concept:humanagent',\n",
    "                   'concept:agent',\n",
    "                   'concept:person',\n",
    "                   'concept:athlete',\n",
    "                   'concept:actor',\n",
    "                   'concept:director',\n",
    "                   'concept:celebrity',\n",
    "                   'concept:male',\n",
    "                   'concept:ethnicgroup',\n",
    "                   'concept:governmentorganization',\n",
    "                   'concept:company',\n",
    "                   'concept:location',\n",
    "                   'concept:geopoliticalentity',\n",
    "                   'concept:geopoliticallocation',\n",
    "                   'concept:city',\n",
    "                   'concept:street',\n",
    "                   'concept:beach',\n",
    "                   'concept:skiarea',\n",
    "                   'concept:river',\n",
    "                   'concept:airport',\n",
    "                   'concept:abstractthing',\n",
    "                   'concept:creativework',\n",
    "                   'concept:time',\n",
    "                   'concept:politician',\n",
    "                   'concept:politicalparty',\n",
    "                   'concept:musicalbum',\n",
    "                   'concept:musicsong',\n",
    "                   'concept:movie',\n",
    "                   'concept:book',\n",
    "                   'concept:software',\n",
    "                   'concept:sport',\n",
    "                   'concept:event',\n",
    "                   'concept:militaryeventtype',\n",
    "                   'concept:militaryconflict',\n",
    "                   'concept:sportsevent',\n",
    "                   'concept:country',\n",
    "                   'concept:building',\n",
    "                   'concept:chemical',\n",
    "                   'concept:planet',\n",
    "                   'concept:vehicle'\n",
    "                  ])\n",
    "\n",
    "# sets for each class to hold all instances\n",
    "allInstances = set()\n",
    "instancesVisualizableObject = set()\n",
    "instancesVisualizableThing = set()\n",
    "instancesSpecies = set()\n",
    "instancesHumanAgent = set()\n",
    "instancesAgent = set()\n",
    "instancesPerson = set()\n",
    "instancesAthlete = set()\n",
    "instancesActor = set()\n",
    "instancesDirector = set()\n",
    "instancesCelebrity = set()\n",
    "instancesMale = set()\n",
    "instancesEthnicGroup = set()\n",
    "instancesGovOrg = set()\n",
    "instancesCompany = set()\n",
    "instancesLocation = set()\n",
    "instancesGeopoliticalEntity = set()\n",
    "instancesGeopoliticalLocation = set()\n",
    "instancesCity = set()\n",
    "instancesStreet = set()\n",
    "instancesBeach = set()\n",
    "instancesSkiArea = set()\n",
    "instancesRiver = set()\n",
    "instancesAirport = set()\n",
    "instancesAbstractThing = set()\n",
    "instancesCreativeWork = set()\n",
    "instancesTime = set()\n",
    "instancesPolitician = set()\n",
    "instancesPoliticalParty = set()\n",
    "instancesMusicAlbum = set()\n",
    "instancesMusicSong = set()\n",
    "instancesMovie = set()\n",
    "instancesBook = set()\n",
    "instancesSoftware = set()\n",
    "instancesSport = set()\n",
    "instancesEvent = set()\n",
    "instancesMilitaryEventType = set()\n",
    "instancesMilitaryConflict = set()\n",
    "instancesSportsEvent = set()\n",
    "instancesCountry = set()\n",
    "instancesBuilding = set()\n",
    "instancesChemical = set()\n",
    "instancesPlanet = set()\n",
    "instancesVehicle = set()\n",
    "\n",
    "\n",
    "class ClassInstances:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.allClassInstances = set()\n",
    "        self.countDict = dict()\n",
    "        self.min = 9999999\n",
    "        self.max = 0\n",
    "        self.avg = 0.0\n",
    "        self.median = 0.0\n",
    "    def addInstance(self, instance):\n",
    "        if instance in self.allClassInstances:\n",
    "            self.countDict[instance] += 1\n",
    "        else:\n",
    "            self.countDict[instance] = 1\n",
    "            self.allClassInstances.add(instance)\n",
    "    def calculateDegrees(self):\n",
    "        if (len(self.countDict) != 0):\n",
    "            allValueList = []\n",
    "            for k,v in self.countDict.iteritems():\n",
    "                allValueList.append(v)\n",
    "                if (v < self.min):\n",
    "                    self.min = v\n",
    "                if (v > self.max):\n",
    "                    self.max = v\n",
    "                self.avg += v\n",
    "            self.avg = self.avg / len(self.countDict)\n",
    "            self.median = numpy.median(numpy.array(allValueList))\n",
    "    def printResults(self):\n",
    "        print ('{}'.format(self.name))\n",
    "        print ('min: {}, avg: {}, median: {}, max: {}'.format(self.min, self.avg, self.median, self.max))\n",
    "        \n",
    "\n",
    "# instantiation of all class instances for indegree          \n",
    "visualizableObjectCi = ClassInstances('class instances indegree for VisualizableObject')\n",
    "visualizableThingCi = ClassInstances('class instances indegree for VisualizableThing')\n",
    "speciesCi = ClassInstances('class instances indegree for species')\n",
    "humanAgentCi = ClassInstances('class instances indegree for humanagent')\n",
    "agentCi = ClassInstances('class instances indegree for agent')\n",
    "personCi = ClassInstances('class instances indegree for person')\n",
    "athleteCi = ClassInstances('class instances indegree for athlete')\n",
    "actorCi = ClassInstances('class instances indegree for actor')\n",
    "directorCi = ClassInstances('class instances indegree for director')\n",
    "celebrityCi = ClassInstances('class instances indegree for celebrity')\n",
    "maleCi = ClassInstances('class instances indegree for male')\n",
    "ethnicGroupCi = ClassInstances('class instances indegree for ethnicGroup')\n",
    "govOrgCi = ClassInstances('class instances indegree for governmentOrganization')\n",
    "companyCi = ClassInstances('class instances indegree for company')\n",
    "locationCi = ClassInstances('class instances indegree for location')\n",
    "geopoliticalEntityCi = ClassInstances('class instances indegree for geopoloticalEntity')\n",
    "geopoliticalLocationCi = ClassInstances('class instances indegree for geopoliticalLocation')\n",
    "cityCi = ClassInstances('class instances indegree for city')\n",
    "streetCi = ClassInstances('class instances indegree for street')\n",
    "beachCi = ClassInstances('class instances indegree for beach')\n",
    "skiAreaCi = ClassInstances('class instances indegree for skiarea')\n",
    "riverCi = ClassInstances('class instances indegree for river')\n",
    "airportCi = ClassInstances('class instances indegree for airport')\n",
    "abstractThingCi = ClassInstances('class instances indegree for abstractThing')\n",
    "creativeWorkCi = ClassInstances('class instances indegree for creativeWork')\n",
    "timeCi = ClassInstances('class instances indegree for time')\n",
    "politicianCi = ClassInstances('class instances indegree for politician')\n",
    "politicalPartyCi = ClassInstances('class instances indegree for politicalParty')\n",
    "musicAlbumCi = ClassInstances('class instances indegree for musicAlbum')\n",
    "musicSongCi = ClassInstances('class instances indegree for musicSong')\n",
    "movieCi = ClassInstances('class instances indegree for movie')\n",
    "bookCi = ClassInstances('class instances indegree for book')\n",
    "softwareCi = ClassInstances('class instances indegree for software')\n",
    "sportCi = ClassInstances('class instances indegree for sport')\n",
    "eventCi = ClassInstances('class instances indegree for event')\n",
    "militaryEventTypeCi = ClassInstances('class instances indegree for militaryEventType')\n",
    "militaryConflictCi = ClassInstances('class instances indegree for militaryConflict')\n",
    "sportsEventCi = ClassInstances('class instances indegree for sportsEvent')\n",
    "countryCi = ClassInstances('class instances indegree for country')\n",
    "buildingCi = ClassInstances('class instances indegree for building')\n",
    "chemicalCi = ClassInstances('class instances indegree for chemical')\n",
    "planetCi = ClassInstances('class instances indegree for planet')\n",
    "vehicleCi = ClassInstances('class instances indegree for vehicle')\n",
    "\n",
    "# list with references to all indegree classes\n",
    "classInstancesIndegree = [visualizableObjectCi, visualizableThingCi, speciesCi, humanAgentCi, agentCi, personCi, athleteCi, actorCi, directorCi, celebrityCi, maleCi, ethnicGroupCi, govOrgCi, companyCi, locationCi, geopoliticalEntityCi, geopoliticalLocationCi, cityCi, streetCi, beachCi, skiAreaCi, riverCi, airportCi, abstractThingCi, creativeWorkCi, timeCi, politicianCi, politicalPartyCi, musicAlbumCi, musicSongCi, movieCi, bookCi, softwareCi, sportCi, eventCi, militaryEventTypeCi, militaryConflictCi, sportsEventCi, countryCi, buildingCi, chemicalCi, planetCi, vehicleCi]\n",
    "\n",
    "# instantiation of all class instances for outdegree          \n",
    "visualizableObjectCo = ClassInstances('class instances outdegree for VisualizableObject')\n",
    "visualizableThingCo = ClassInstances('class instances outdegree for VisualizableThing')\n",
    "speciesCo = ClassInstances('class instances outdegree for species')\n",
    "humanAgentCo = ClassInstances('class instances outdegree for humanagent')\n",
    "agentCo = ClassInstances('class instances outdegree for agent')\n",
    "personCo = ClassInstances('class instances outdegree for person')\n",
    "athleteCo = ClassInstances('class instances outdegree for athlete')\n",
    "actorCo = ClassInstances('class instances outdegree for actor')\n",
    "directorCo = ClassInstances('class instances outdegree for director')\n",
    "celebrityCo = ClassInstances('class instances outdegree for celebrity')\n",
    "maleCo = ClassInstances('class instances outdegree for male')\n",
    "ethnicGroupCo = ClassInstances('class instances outdegree for ethnicGroup')\n",
    "govOrgCo = ClassInstances('class instances outdegree for governmentOrganization')\n",
    "companyCo = ClassInstances('class instances outdegree for company')\n",
    "locationCo = ClassInstances('class instances outdegree for location')\n",
    "geopoliticalEntityCo = ClassInstances('class instances outdegree for geopoloticalEntity')\n",
    "geopoliticalLocationCo = ClassInstances('class instances outdegree for geopoliticalLocation')\n",
    "cityCo = ClassInstances('class instances outdegree for city')\n",
    "streetCo = ClassInstances('class instances outdegree for street')\n",
    "beachCo = ClassInstances('class instances outdegree for beach')\n",
    "skiAreaCo = ClassInstances('class instances outdegree for skiarea')\n",
    "riverCo = ClassInstances('class instances outdegree for river')\n",
    "airportCo = ClassInstances('class instances outdegree for airport')\n",
    "abstractThingCo = ClassInstances('class instances outdegree for abstractThing')\n",
    "creativeWorkCo = ClassInstances('class instances outdegree for creativeWork')\n",
    "timeCo = ClassInstances('class instances outdegree for time')\n",
    "politicianCo = ClassInstances('class instances outdegree for politician')\n",
    "politicalPartyCo = ClassInstances('class instances outdegree for politicalParty')\n",
    "musicAlbumCo = ClassInstances('class instances outdegree for musicAlbum')\n",
    "musicSongCo = ClassInstances('class instances outdegree for musicSong')\n",
    "movieCo = ClassInstances('class instances outdegree for movie')\n",
    "bookCo = ClassInstances('class instances outdegree for book')\n",
    "softwareCo = ClassInstances('class instances outdegree for software')\n",
    "sportCo = ClassInstances('class instances outdegree for sport')\n",
    "eventCo = ClassInstances('class instances outdegree for event')\n",
    "militaryEventTypeCo = ClassInstances('class instances outdegree for militaryEventType')\n",
    "militaryConflictCo = ClassInstances('class instances outdegree for militaryConflict')\n",
    "sportsEventCo = ClassInstances('class instances outdegree for sportsEvent')\n",
    "countryCo = ClassInstances('class instances outdegree for country')\n",
    "buildingCo = ClassInstances('class instances outdegree for building')\n",
    "chemicalCo = ClassInstances('class instances outdegree for chemical')\n",
    "planetCo = ClassInstances('class instances outdegree for planet')\n",
    "vehicleCo = ClassInstances('class instances outdegree for vehicle')\n",
    "\n",
    "# list with references to all outdegree classes\n",
    "classInstancesOutdegree = [visualizableObjectCo, visualizableThingCo, speciesCo, humanAgentCo, agentCo, personCo, athleteCo, actorCo, directorCo, celebrityCo, maleCo, ethnicGroupCo, govOrgCo, companyCo, locationCo, geopoliticalEntityCo, geopoliticalLocationCo, cityCo, streetCo, beachCo, skiAreaCo, riverCo, airportCo, abstractThingCo, creativeWorkCo, timeCo, politicianCo, politicalPartyCo, musicAlbumCo, musicSongCo, movieCo, bookCo, softwareCo, sportCo, eventCo, militaryEventTypeCo, militaryConflictCo, sportsEventCo, countryCo, buildingCo, chemicalCo, planetCo, vehicleCi]\n",
    "\n",
    "def getInstances(splitted_line):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity \n",
    "            entity = word\n",
    "        elif (word_position==1): #Relation\n",
    "            relation = word\n",
    "        elif (word_position==2): #Value\n",
    "            value = word\n",
    "            if (relation == 'generalizations' and value in topClasses):\n",
    "                # check for specific class and add instance to the class set\n",
    "                if (value == 'concept:visualizableobject'):\n",
    "                    if entity not in instancesVisualizableObject:\n",
    "                        instancesVisualizableObject.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:visualizablething'):\n",
    "                    if entity not in instancesVisualizableThing:\n",
    "                        instancesVisualizableThing.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:species'):\n",
    "                    if entity not in instancesSpecies:\n",
    "                        instancesSpecies.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:humanagent'):\n",
    "                    if entity not in instancesHumanAgent:\n",
    "                        instancesHumanAgent.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:agent'):\n",
    "                    if entity not in instancesAgent:\n",
    "                        instancesAgent.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:person'):\n",
    "                    if entity not in instancesPerson:\n",
    "                        instancesPerson.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:athlete'):\n",
    "                    if entity not in instancesAthlete:\n",
    "                        instancesAthlete.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:actor'):\n",
    "                    if entity not in instancesActor:\n",
    "                        instancesActor.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:director'):\n",
    "                    if entity not in instancesDirector:\n",
    "                        instancesDirector.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:celebrity'):\n",
    "                    if entity not in instancesCelebrity:\n",
    "                        instancesCelebrity.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:male'):\n",
    "                    if entity not in instancesMale:\n",
    "                        instancesMale.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:ethnicgroup'):\n",
    "                    if entity not in instancesEthnicGroup:\n",
    "                        instancesEthnicGroup.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:governmentorganization'):\n",
    "                    if entity not in instancesGovOrg:\n",
    "                        instancesGovOrg.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:company'):\n",
    "                    if entity not in instancesCompany:\n",
    "                        instancesCompany.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:location'):\n",
    "                    if entity not in instancesLocation:\n",
    "                        instancesLocation.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:geopoliticalentity'):\n",
    "                    if entity not in instancesGeopoliticalEntity:\n",
    "                        instancesGeopoliticalEntity.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:geopoliticallocation'):\n",
    "                    if entity not in instancesGeopoliticalLocation:\n",
    "                        instancesGeopoliticalLocation.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:city'):\n",
    "                    if entity not in instancesCity:\n",
    "                        instancesCity.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:street'):\n",
    "                    if entity not in instancesStreet:\n",
    "                        instancesStreet.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:beach'):\n",
    "                    if entity not in instancesBeach:\n",
    "                        instancesBeach.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:skiarea'):\n",
    "                    if entity not in instancesSkiArea:\n",
    "                        instancesSkiArea.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:river'):\n",
    "                    if entity not in instancesRiver:\n",
    "                        instancesRiver.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:airport'):\n",
    "                    if entity not in instancesAirport:\n",
    "                        instancesAirport.add(entity)\n",
    "                    return           \n",
    "                elif (value == 'concept:abstractthing'):\n",
    "                    if entity not in instancesAbstractThing:\n",
    "                        instancesAbstractThing.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:creativework'):\n",
    "                    if entity not in instancesCreativeWork:\n",
    "                        instancesCreativeWork.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:time'):\n",
    "                    if entity not in instancesTime:\n",
    "                        instancesTime.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:politician'):\n",
    "                    if entity not in instancesPolitician:\n",
    "                        instancesPolitician.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:politicalparty'):\n",
    "                    if entity not in instancesPoliticalParty:\n",
    "                        instancesPoliticalParty.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:musicalbum'):\n",
    "                    if entity not in instancesMusicAlbum:\n",
    "                        instancesMusicAlbum.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:musicsong'):\n",
    "                    if entity not in instancesMusicSong:\n",
    "                        instancesMusicSong.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:movie'):\n",
    "                    if entity not in instancesMovie:\n",
    "                        instancesMovie.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:book'):\n",
    "                    if entity not in instancesBook:\n",
    "                        instancesBook.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:software'):\n",
    "                    if entity not in instancesSoftware:\n",
    "                        instancesSoftware .add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:sport'):\n",
    "                    if entity not in instancesSport:\n",
    "                        instancesSport.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:event'):\n",
    "                    if entity not in instancesEvent:\n",
    "                        instancesEvent.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:militaryeventtype'):\n",
    "                    if entity not in instancesMilitaryEventType:\n",
    "                        instancesMilitaryEventType.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:militaryconflict'):\n",
    "                    if entity not in instancesMilitaryConflict:\n",
    "                        instancesMilitaryConflict.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:sportsevent'):\n",
    "                    if entity not in instancesSportsEvent:\n",
    "                        instancesSportsEvent.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:country'):\n",
    "                    if entity not in instancesCountry:\n",
    "                        instancesCountry.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:building'):\n",
    "                    if entity not in instancesBuilding:\n",
    "                        instancesBuilding.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:chemical'):\n",
    "                    if entity not in instancesChemical:\n",
    "                        instancesChemical.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:planet'):\n",
    "                    if entity not in instancesPlanet:\n",
    "                        instancesPlanet.add(entity)\n",
    "                    return\n",
    "                elif (value == 'concept:vehicle'):\n",
    "                    if entity not in instancesVehicle:\n",
    "                        instancesVehicle.add(entity)\n",
    "                    return\n",
    "                return\n",
    "            else: # NOT relation == 'generalizations' and value in topClasses\n",
    "                return                \n",
    "        elif(word_position>2):\n",
    "            return\n",
    "        word_position +=1\n",
    "\n",
    "def countInstanceDegrees(splitted_line):\n",
    "    word_position = 0\n",
    "    for word in splitted_line:\n",
    "        if (word_position==0): #Entity: OUTDEGREE\n",
    "            entity = word\n",
    "            if entity in allInstances:\n",
    "                if entity in instancesVisualizableObject:\n",
    "                    visualizableObjectCo.addInstance(entity)\n",
    "                elif entity in instancesVisualizableThing:\n",
    "                    visualizableThingCo.addInstance(entity)\n",
    "                elif entity in instancesSpecies:\n",
    "                    speciesCo.addInstance(entity)\n",
    "                elif entity in instancesHumanAgent:\n",
    "                    humanAgentCo.addInstance(entity)\n",
    "                elif entity in instancesAgent:\n",
    "                    agentCo.addInstance(entity)\n",
    "                elif entity in instancesPerson:\n",
    "                    personCo.addInstance(entity)\n",
    "                elif entity in instancesAthlete:\n",
    "                    athleteCo.addInstance(entity)\n",
    "                elif entity in instancesActor:\n",
    "                    actorCo.addInstance(entity)\n",
    "                elif entity in instancesDirector:\n",
    "                    directorCo.addInstance(entity)\n",
    "                elif entity in instancesCelebrity:\n",
    "                    celebrityCo.addInstance(entity)\n",
    "                elif entity in instancesMale:\n",
    "                    maleCo.addInstance(entity)\n",
    "                elif entity in instancesEthnicGroup:\n",
    "                    ethnicGroupCo.addInstance(entity)\n",
    "                elif entity in instancesGovOrg:\n",
    "                    govOrgCo.addInstance(entity)\n",
    "                elif entity in instancesCompany:\n",
    "                    companyCo.addInstance(entity)\n",
    "                elif entity in instancesLocation:\n",
    "                    locationCo.addInstance(entity)\n",
    "                elif entity in instancesGeopoliticalEntity:\n",
    "                    geopoliticalEntityCo.addInstance(entity)\n",
    "                elif entity in instancesGeopoliticalLocation:\n",
    "                    geopoliticalLocationCo.addInstance(entity)\n",
    "                elif entity in instancesCity:\n",
    "                    cityCo.addInstance(entity)\n",
    "                elif entity in instancesStreet:\n",
    "                    streetCo.addInstance(entity)\n",
    "                elif entity in instancesBeach:\n",
    "                    beachCo.addInstance(entity)\n",
    "                elif entity in instancesSkiArea:\n",
    "                    skiAreaCo.addInstance(entity)\n",
    "                elif entity in instancesRiver:\n",
    "                    riverCo.addInstance(entity)\n",
    "                elif entity in instancesAirport:\n",
    "                    airportCo.addInstance(entity)\n",
    "                elif entity in instancesAbstractThing:\n",
    "                    abstractThingCo.addInstance(entity)\n",
    "                elif entity in instancesCreativeWork:\n",
    "                    creativeWorkCo.addInstance(entity)\n",
    "                elif entity in instancesTime:\n",
    "                    timeCo.addInstance(entity)\n",
    "                elif entity in instancesPolitician:\n",
    "                    politicianCo.addInstance(entity)\n",
    "                elif entity in instancesPoliticalParty:\n",
    "                    politicalPartyCo.addInstance(entity)\n",
    "                elif entity in instancesMusicAlbum:\n",
    "                    musicAlbumCo.addInstance(entity)\n",
    "                elif entity in instancesMusicSong:\n",
    "                    musicSongCo.addInstance(entity)\n",
    "                elif entity in instancesMovie:\n",
    "                    movieCo.addInstance(entity)\n",
    "                elif entity in instancesBook:\n",
    "                    bookCo.addInstance(entity)\n",
    "                elif entity in instancesSoftware:\n",
    "                    softwareCo.addInstance(entity)\n",
    "                elif entity in instancesSport:\n",
    "                    sportCo.addInstance(entity)\n",
    "                elif entity in instancesEvent:\n",
    "                    eventCo.addInstance(entity)\n",
    "                elif entity in instancesMilitaryEventType:\n",
    "                    militaryEventTypeCo.addInstance(entity)\n",
    "                elif entity in instancesMilitaryConflict:\n",
    "                    militaryConflictCo.addInstance(entity)\n",
    "                elif entity in instancesSportsEvent:\n",
    "                    sportsEventCo.addInstance(entity)\n",
    "                elif entity in instancesCountry:\n",
    "                    countryCo.addInstance(entity)\n",
    "                elif entity in instancesBuilding:\n",
    "                    buildingCo.addInstance(entity)\n",
    "                elif entity in instancesChemical:\n",
    "                    chemicalCo.addInstance(entity)\n",
    "                elif entity in instancesPlanet:\n",
    "                    planetCo.addInstance(entity)\n",
    "                elif entity in instancesVehicle:\n",
    "                    vehicleCo.addInstance(entity)\n",
    "        elif (word_position==2): #Value: INDEGREE\n",
    "            value = word\n",
    "            if value in allInstances:\n",
    "                if value in instancesVisualizableObject:\n",
    "                    visualizableObjectCi.addInstance(value)\n",
    "                elif value in instancesVisualizableThing:\n",
    "                    visualizableThingCi.addInstance(value)\n",
    "                elif value in instancesSpecies:\n",
    "                    speciesCi.addInstance(value)\n",
    "                elif value in instancesHumanAgent:\n",
    "                    humanAgentCi.addInstance(value)\n",
    "                elif value in instancesAgent:\n",
    "                    agentCi.addInstance(value)\n",
    "                elif value in instancesPerson:\n",
    "                    personCi.addInstance(value)\n",
    "                elif value in instancesAthlete:\n",
    "                    athleteCi.addInstance(value)\n",
    "                elif value in instancesActor:\n",
    "                    actorCi.addInstance(value)\n",
    "                elif value in instancesDirector:\n",
    "                    directorCi.addInstance(value)\n",
    "                elif value in instancesCelebrity:\n",
    "                    celebrityCi.addInstance(value)\n",
    "                elif value in instancesMale:\n",
    "                    maleCi.addInstance(value)\n",
    "                elif value in instancesEthnicGroup:\n",
    "                    ethnicGroupCi.addInstance(value)\n",
    "                elif value in instancesGovOrg:\n",
    "                    govOrgCi.addInstance(value)\n",
    "                elif value in instancesCompany:\n",
    "                    companyCi.addInstance(value)\n",
    "                elif value in instancesLocation:\n",
    "                    locationCi.addInstance(value)\n",
    "                elif value in instancesGeopoliticalEntity:\n",
    "                    geopoliticalEntityCi.addInstance(value)\n",
    "                elif value in instancesGeopoliticalLocation:\n",
    "                    geopoliticalLocationCi.addInstance(value)\n",
    "                elif value in instancesCity:\n",
    "                    cityCi.addInstance(value)\n",
    "                elif value in instancesStreet:\n",
    "                    streetCi.addInstance(value)\n",
    "                elif value in instancesBeach:\n",
    "                    beachCi.addInstance(value)\n",
    "                elif value in instancesSkiArea:\n",
    "                    skiAreaCi.addInstance(value)\n",
    "                elif value in instancesRiver:\n",
    "                    riverCi.addInstance(value)\n",
    "                elif value in instancesAirport:\n",
    "                    airportCi.addInstance(value)\n",
    "                elif value in instancesAbstractThing:\n",
    "                    abstractThingCi.addInstance(value)\n",
    "                elif value in instancesCreativeWork:\n",
    "                    creativeWorkCi.addInstance(value)\n",
    "                elif value in instancesTime:\n",
    "                    timeCi.addInstance(value)\n",
    "                elif value in instancesPolitician:\n",
    "                    politicianCi.addInstance(value)\n",
    "                elif value in instancesPoliticalParty:\n",
    "                    politicalPartyCi.addInstance(value)\n",
    "                elif value in instancesMusicAlbum:\n",
    "                    musicAlbumCi.addInstance(value)\n",
    "                elif value in instancesMusicSong:\n",
    "                    musicSongCi.addInstance(value)\n",
    "                elif value in instancesMovie:\n",
    "                    movieCi.addInstance(value)\n",
    "                elif value in instancesBook:\n",
    "                    bookCi.addInstance(value)\n",
    "                elif value in instancesSoftware:\n",
    "                    softwareCi.addInstance(value)\n",
    "                elif value in instancesSport:\n",
    "                    sportCi.addInstance(value)\n",
    "                elif value in instancesEvent:\n",
    "                    eventCi.addInstance(value)\n",
    "                elif value in instancesMilitaryEventType:\n",
    "                    militaryEventTypeCi.addInstance(value)\n",
    "                elif value in instancesMilitaryConflict:\n",
    "                    militaryConflictCi.addInstance(value)\n",
    "                elif value in instancesSportsEvent:\n",
    "                    sportsEventCi.addInstance(value)\n",
    "                elif value in instancesCountry:\n",
    "                    countryCi.addInstance(value)\n",
    "                elif value in instancesBuilding:\n",
    "                    buildingCi.addInstance(value)\n",
    "                elif value in instancesChemical:\n",
    "                    chemicalCi.addInstance(value)\n",
    "                elif value in instancesPlanet:\n",
    "                    planetCi.addInstance(value)\n",
    "                elif value in instancesVehicle:\n",
    "                    vehicleCi.addInstance(value)               \n",
    "            return # done processing entity and value\n",
    "        elif(word_position>2):\n",
    "            return\n",
    "        word_position +=1\n",
    "        \n",
    "def calculateClassDegrees():\n",
    "    for item in classInstancesIndegree:\n",
    "        item.calculateDegrees()\n",
    "    for item in classInstancesOutdegree:\n",
    "        item.calculateDegrees()\n",
    "        \n",
    "def printClassDegreeResults():\n",
    "    for item in classInstancesIndegree:\n",
    "        item.printResults()\n",
    "    for item in classInstancesOutdegree:\n",
    "        item.printResults()\n",
    "    \n",
    "print('START')\n",
    "try:\n",
    "    # get Instances and calculate indegree&outdegree\n",
    "    for i in xrange(2):\n",
    "        if (i==0):\n",
    "            print ('1. run: get class instances')\n",
    "        elif (i==1):\n",
    "            print ('2. run: count instance degrees')\n",
    "        # read nell & ontology\n",
    "        for j in xrange(2):\n",
    "            if (j==0):\n",
    "                print ('read ontology')\n",
    "                readFile = ontology\n",
    "            elif (j==1):\n",
    "                print ('read NELL')\n",
    "                readFile = nell\n",
    "            fn = open(readFile, 'r')\n",
    "            lineCounter = 0\n",
    "            for line in fn:\n",
    "                if (lineCounter != 0):\n",
    "                    splitted_line = line.rstrip('\\n').split('\\t')\n",
    "                    if (i==0):\n",
    "                        getInstances(splitted_line)\n",
    "                    elif (i==1):\n",
    "                        countInstanceDegrees(splitted_line)\n",
    "                    else:\n",
    "                        calculateClassDegrees(splitted_line)\n",
    "                lineCounter += 1\n",
    "                if (lineCounter % lineProgress == 0):\n",
    "                    print ('{} lines read'.format(lineCounter))\n",
    "            fn.close()\n",
    "        if (i==0):\n",
    "            # create set of all instances\n",
    "            #allInstances = instancesBeach|instancesActor|instancesGovOrg|instancesDirector|instancesSkiArea|instancesStreet|instancesRiver|instancesAirport|instancesCelebrity|instancesMale\n",
    "            allInstances = instancesVisualizableObject|instancesVisualizableThing|instancesSpecies|instancesHumanAgent|instancesAgent|instancesPerson|instancesAthlete|instancesActor|instancesDirector|instancesCelebrity|instancesMale|instancesEthnicGroup|instancesGovOrg|instancesCompany|instancesLocation|instancesGeopoliticalEntity|instancesGeopoliticalLocation|instancesCity|instancesStreet|instancesBeach|instancesSkiArea|instancesRiver|instancesAirport|instancesAbstractThing|instancesCreativeWork|instancesTime|instancesPolitician|instancesPoliticalParty|instancesMusicAlbum|instancesMusicSong|instancesMovie|instancesBook|instancesSoftware|instancesSport|instancesEvent|instancesMilitaryEventType|instancesMilitaryConflict|instancesSportsEvent|instancesCountry|instancesBuilding|instancesChemical|instancesPlanet|instancesVehicle\n",
    "    # calculate min, avg, max class degrees\n",
    "    print ('calculate class degrees')\n",
    "    calculateClassDegrees()\n",
    "    \n",
    "    print ('DONE')\n",
    "    print('RESULTS')\n",
    "    printClassDegreeResults()\n",
    "except:\n",
    "    print('ERROR')\n",
    "\n",
    "end = time.time()\n",
    "print ('EXECUTION TIME: {} s'.format(end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START\n"
     ]
    }
   ],
   "source": [
    "# check concept:rtwcategory property\n",
    "\n",
    "nell = 'NELL/NELL.08m.995.esv.csv'\n",
    "nellSample = 'NELL/NELL_sample.csv'\n",
    "readFile = nellSample\n",
    "\n",
    "classInstanceDict = {}\n",
    "\n",
    "def getSPO(splittedLine):\n",
    "    word_position = 0\n",
    "    for word in splittedLine:\n",
    "        if (word_position == 0):\n",
    "            subj = word\n",
    "        elif (word_position == 1):\n",
    "            pred = word\n",
    "        elif (word_position == 2):\n",
    "            obj = word\n",
    "        else:\n",
    "            return subj, pred, obj\n",
    "        word_position += 1\n",
    "    return subj, pred, obj\n",
    "print ('START')\n",
    "fn = open(readFile, 'r')\n",
    "for line in fn:\n",
    "    splitted_line = line.rstrip('\\n').split('\\t')\n",
    "    s, p, o = getSPO (splitted_line)\n",
    "    if (p==\"concept:rtwcategory\"):\n",
    "        if classInstanceDict.hasKey(s):\n",
    "            classInstanceDict[s] += 1\n",
    "        else:\n",
    "            classInstanceDict[s] = 1\n",
    "fn.close()\n",
    "\n",
    "for k, v in classInstanceDict.values():\n",
    "    print ('{}: {}'.format(k,v))\n",
    "        \n",
    "        \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}