-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathgetTimeAndIncidentInformation.py
90 lines (86 loc) · 3.55 KB
/
getTimeAndIncidentInformation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import datetime
import pymongo
import pytz
import time
from pymongo import MongoClient
import datetime
from collections import Counter
import json
import requests
import wikipedia
from pygoogle import pygoogle
import duckduckgo
from googlesearch import GoogleSearch
client = MongoClient('localhost',27017)
utc=pytz.UTC
db = client['disaster']
threeHourlyAlert = db['threeHourlyAlert']
Tweets = db['Tweets']
incrementBy2Hour= datetime.timedelta(hours=2)
allDisasters = list(threeHourlyAlert.find())
disasterList = []
for disaster in allDisasters:
for nameOfDisaster in disaster['count']:
disasterList.append( (disaster['date'] , nameOfDisaster) )
markers = []
infoWindowContent = []
for date,disasterKeyWord in disasterList:
date = date.replace(tzinfo=utc)
dateFinal = date + incrementBy2Hour
dateFinal = dateFinal.replace(tzinfo=utc)
disasterTweets = Tweets.find({"created_at" : { '$gt' : date , '$lt': dateFinal } , '$text': {'$search': disasterKeyWord}})#.count()
d = Counter()
for content in disasterTweets:
for countContent in content['text'].split():
d[countContent] += 1
useToGetLocation = ""
for k,v in d.most_common(100):
useToGetLocation += k + " "
payload = {'text':useToGetLocation,'demonyms': 'false'}
r = requests.post("http://cliff.mediameter.org/process", data=payload)
if (json.loads(r.text)['results']['places']['focus'] != {}):
if (json.loads(r.text)['results']['places']['focus']['cities'] != []):
location = json.loads(r.text)['results']['places']['focus']['cities']
elif (json.loads(r.text)['results']['places']['focus']['states'] != []):
location = json.loads(r.text)['results']['places']['focus']['states']
elif (json.loads(r.text)['results']['places']['focus']['countries'] != []):
location = json.loads(r.text)['results']['places']['focus']['countries']
else:
continue
else:
continue
name=location[0]['name']
lat=location[0]['lat']
lon=location[0]['lon']
searchText= disasterKeyWord + " " + name + " " + date.strftime("%d %B %Y")
#print searchText
search=GoogleSearch(searchText);
time.sleep(300)
wikiTitle = search.top_results()[0]['title']
wikiURL = search.top_results()[0]['url']
wikiContent = search.top_results()[0]['content']
#wikiSearchList = wikipedia.search(searchText);
#print wikiSearchList
# for wikiSearch in wikiSearchList:
# if "2016" in wikiSearch :
# wikiSearchWord = wikiSearch
# break
# else:
# wikiSearchWord = wikiSearchList[0]
# if ( wikiSearchWord != "" ):
# wikiContent = wikipedia.page(wikiSearch)
# wikiTitle = date.strftime("%d %B %Y") + " " + wikiContent.title
# wikiPostConent = wikiContent.content.split('.')[0]
# wikiURL = wikiContent.url
# else:
# #googleSearch = googlesearch(searchText)
# wikiTitle = date.strftime("%d %B %Y") + " " + disasterKeyWord + " at " + name
# #wikiURL = googleSearch.get_urls()[0]
# #wikiPostConent = ""
# wikiTitle = date.strftime("%d %B %Y") + " at " + name + " " + " Incident: " ;
markers.append([ wikiTitle.encode('utf8') + " " + disasterKeyWord.encode('utf8') , lat , lon ])
infoWindowContent.append([ '<div class="info_content">' + '<h3>' + wikiTitle.encode('utf8') +'</h3>' + '<br><h4>' + wikiContent.encode('utf8') + '</h4>' + '<br><p>' + wikiURL.encode('utf8') + '</p>' +'</div>' ])
print markers
print infoWindowContent
print markers
print infoWindowContent