Skip to content

Commit 1f4e2ad

Browse files
nherbautnherbaut
committed
bootstrap and usability improvments
1 parent 9b1a99f commit 1f4e2ad

File tree

8 files changed

+261
-66
lines changed

8 files changed

+261
-66
lines changed

‎Dockerfile

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
#FROM tiangolo/meinheld-gunicorn-flask:python3.8
2-
FROM tiangolo/uwsgi-nginx-flask:python3.8
1+
FROM python:3.8
32

43
COPY ./requirements.txt /tmp
5-
RUN pip install -r /tmp/requirements.txt
6-
COPY ./app /app
4+
RUN pip3 install -r /tmp/requirements.txt
5+
COPY ./app/app /app
6+
WORKDIR /app
7+
ENV FLASK_APP=app.main
8+
CMD ["python","main.py"]
79

810

‎Makefile

+1-3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ build:
33
push:
44
docker push nherbaut/scpushack
55
run:
6-
docker run -d -p 8106:80 -e API_KEY=${API_KEY} --name "scpushack" nherbaut/scpushack
7-
sleep 2
8-
echo Go to localhost:8106 to visit the website
6+
docker run -d -p 8106:5000 -e API_KEY=${API_KEY} --name "scpushack" nherbaut/scpushack
97
stop:
108
docker rm -f scpushack
119

‎app/app/.idea/vcs.xml

-6
This file was deleted.

‎app/app/main.py

+68-15
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,14 @@
33
import itertools
44
import os
55
import urllib
6+
from flask_socketio import SocketIO, emit, send
7+
68
app = Flask(__name__)
9+
socketio = SocketIO(app)
10+
socketio.init_app(app, cors_allowed_origins="*")
11+
API_KEY = os.environ["API_KEY"]
12+
13+
SCPUS_BACKEND = f'https://api.elsevier.com/content/search/scopus?start=%d&count=%d&query=%s&apiKey={API_KEY}'
714

815

916
@app.route('/')
@@ -13,28 +20,74 @@ def hello_world():
1320
return render_template('index.html')
1421

1522

16-
1723
@app.route('/query', methods=["POST"])
1824
def query():
19-
query = request.form.get("query")
20-
21-
is_count = request.form.get("count")
25+
the_query = request.form.get("query")
2226

23-
p = f'https://api.elsevier.com/content/search/scopus?start=%d&count=%d&query=%s&apiKey={os.environ("API_KEY")}'
24-
count = int(requests.get(p % (0, 1, query.replace(" ", "+").replace("\\", "%%22"))).json()["search-results"][
25-
"opensearch:totalResults"])
27+
count, is_count, p = count_results_for_query(the_query)
2628

2729
print(f"fetching {count} results")
2830
if is_count:
29-
return render_template('index.html', query=query, count=count)
31+
return render_template('index.html', query=the_query, count=count)
3032
else:
31-
refs = list(itertools.chain(*[aa["search-results"]["entry"] for aa in
32-
[requests.get(p % (i, 25, query.replace(" ", "+").replace("\\", "%%22"))).json() for i
33-
in
34-
range(0, min(1000,count), 25)]]))
35-
dois = [ r.get("prism:doi") for r in refs if "prism:doi" in r]
36-
return render_template('dois.html',dois=dois)
33+
refs = get_results_for_query(count, the_query)
34+
dois = [r.get("prism:doi") for r in refs if "prism:doi" in r]
35+
return render_template('dois.html', dois=dois)
36+
37+
38+
def get_results_for_query(count, query):
39+
dois = []
40+
41+
failed = 0
42+
success = 0
43+
for i in range(0, min(1000, count), 25):
44+
bucket = []
45+
partial_results = requests.get(
46+
SCPUS_BACKEND % (i, 25, query.replace(" ", "+").replace("\\", "%%22"))).json()
47+
for aa in partial_results["search-results"]["entry"]:
48+
if "prism:doi" in aa:
49+
success += 1
50+
else:
51+
failed += 1
52+
bucket.append({"doi": aa.get("prism:doi", ""), "title": aa.get("dc:title", "???")});
53+
emit('doi_update', {"total": count, "done": success, "failed": failed})
54+
emit('doi_results', bucket)
55+
dois = dois + bucket
56+
emit('doi_export_done', dois)
57+
return dois
58+
59+
60+
def count_results_for_query(query):
61+
print(f"query with {API_KEY} API_KEY")
62+
is_count = request.form.get("count")
63+
64+
count = int(
65+
requests.get(SCPUS_BACKEND % (0, 1, query.replace(" ", "+").replace("\\", "%%22"))).json()["search-results"][
66+
"opensearch:totalResults"])
67+
return count
68+
69+
70+
@socketio.on('my event')
71+
def handle_message(data):
72+
for i in range(0, 10000):
73+
emit('news', i)
74+
75+
76+
@socketio.on('count')
77+
def handle_count(json_data):
78+
emit("count", count_results_for_query(json_data["query"]))
79+
80+
81+
@socketio.on('get_dois')
82+
def handle_get_dois(json_data):
83+
the_query = json_data["query"]
84+
count = count_results_for_query(the_query)
85+
dois = get_results_for_query(count, the_query)
86+
87+
emit("dois", {"dois": dois})
3788

3889

3990
if __name__ == '__main__':
40-
app.run()
91+
app.config['SECRET_KEY'] = 'secret!'
92+
print(f"Using API_KEY={API_KEY}")
93+
socketio.run(app,host="0.0.0.0")

‎app/app/static/js/socket.io.min.js

+7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/app/templates/index.html

+174-38
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,212 @@
11
<!DOCTYPE html>
22
<html lang="en">
33
<head>
4+
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css"
5+
integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
46
<meta charset="UTF-8">
57
<title>Scpus Hack</title>
8+
<script src="https://cdn.socket.io/socket.io-3.0.1.min.js"></script>
69

710
<script>
811

9-
function onLoaded(evt){
1012

11-
document.getElementById("querybox").value = window.localStorage.getItem("query");
12-
}
13+
var bag_of_doi = "";
14+
var item_info = new Array();
1315

16+
function download(filename, text) {
17+
var element = document.createElement('a');
18+
element.setAttribute('href', 'data:application/x-bibtex;charset=utf-8,' + encodeURIComponent(text));
19+
element.setAttribute('download', filename);
1420

15-
function textAreaChange(){
16-
updateQueryStorage(document.getElementById("querybox").value);
17-
}
21+
element.style.display = 'none';
22+
document.body.appendChild(element);
1823

19-
function updateQueryStorage(str){
20-
window.localStorage.setItem("query", str);
21-
}
24+
element.click();
2225

26+
document.body.removeChild(element);
27+
}
2328

24-
window.onload = onLoaded;
29+
function onLoaded(evt) {
2530

31+
document.getElementById("querybox").value = window.localStorage.getItem("query");
32+
}
2633

27-
</script>
34+
35+
function textAreaChange() {
36+
updateQueryStorage(document.getElementById("querybox").value);
37+
}
38+
39+
function updateQueryStorage(str) {
40+
window.localStorage.setItem("query", str);
41+
}
42+
43+
44+
window.onload = onLoaded;
45+
46+
const socket = io();
47+
48+
socket.on('connect', function () {
49+
socket.emit('my event', {data: 'I\'m connected!'});
50+
});
51+
52+
socket.on('count', (count) => {
53+
console.log(count);
54+
document.getElementById("main_button").value = "fetch " + count + " results ";
55+
handle_submit = sendDoisRequest;
56+
});
57+
58+
socket.on('dois', (data) => {
59+
console.log(data);
60+
61+
});
62+
63+
socket.on('doi_export_done', (data) => {
64+
document.getElementById("dlresults").removeAttribute("hidden");
65+
})
66+
67+
socket.on('doi_update', (data) => {
68+
69+
document.getElementById("pb_success").setAttribute("max", data["total"]);
70+
document.getElementById("pb_success").setAttribute("value", data["done"]);
71+
document.getElementById("pb_success_count").innerHTML = data["done"] + "/" + data["total"];
72+
document.getElementById("pb_failure").setAttribute("max", data["total"]);
73+
document.getElementById("pb_failure").setAttribute("value", data["failed"]);
74+
document.getElementById("pb_failure_count").innerHTML = data["failed"] + "/" + data["total"];
75+
76+
77+
});
78+
79+
socket.on('doi_results', (data) => {
80+
for (let doi_item of data) {
81+
doi = doi_item["doi"];
82+
item_info.push(doi_item);
83+
add_item_to_table(doi_item);
84+
bag_of_doi += "@article{article1, \n\tdoi = { " + doi + "}\n },\n"
85+
}
86+
87+
88+
});
89+
90+
function add_item_to_table(doi_item) {
91+
var row = document.getElementById("doi_table").insertRow()
92+
if (doi_item["doi"] != "") {
93+
row.innerHTML = "<th scope=\"row\"><a href=\"https://www.doi.org/" + doi_item["doi"] + "\">" + doi_item["doi"] + "</a></th><td>" + doi_item["title"] + "</td><td><a href=\"https://sci-hub.se/" + doi_item["doi"] + "\">Download</a></td>";
94+
}
95+
96+
97+
}
98+
99+
100+
function sendCountRequest() {
101+
socket.emit('count', {query: document.getElementById("querybox").value});
102+
}
103+
104+
function sendDoisRequest() {
105+
socket.emit('get_dois', {query: document.getElementById("querybox").value});
106+
document.getElementById("main_button").disabled = true;
107+
}
108+
109+
110+
handle_submit = sendCountRequest;
111+
112+
function resetAll() {
113+
bag_of_doi = "";
114+
handle_submit = sendCountRequest;
115+
document.getElementById("main_button").value = "Count Results";
116+
document.getElementById("dlresults").setAttribute("hidden", "true");
117+
document.getElementById("querybox").value = "";
118+
document.getElementById("pb_success").setAttribute("max", 0);
119+
document.getElementById("pb_success").setAttribute("value", 0);
120+
document.getElementById("pb_failure").setAttribute("max", 0);
121+
document.getElementById("pb_failure").setAttribute("value", 0);
122+
document.getElementById("pb_failure_count").innerHTML = "";
123+
document.getElementById("pb_success_count").innerHTML = "";
124+
document.getElementById("doi_table").innerHTML = "";
125+
document.getElementById("main_button").disabled = false;
126+
}
127+
128+
function downloadResults() {
129+
download("results.bib", bag_of_doi)
130+
}
131+
132+
</script>
28133
</head>
29134
<body>
30-
<h1>Scpus Hack: Get a list of DOIs that match your search queries</h1>
31-
<h2>How to</h2>
32-
<ol>
33-
<li>Type your query by following the <a
34-
href="http://schema.elsevier.com/dtds/document/bkapi/search/SCOPUSSearchTips.htm">doc</a> e.g. <i>TITLE-ABS-KEY(blockchain) AND TITLE-ABS-KEY(INDUSTRY 4.0) AND (TITLE-ABS-KEY(Security) OR TITLE-ABS-KEY(Privacy))</i></li>
35-
<li>Click "count results" to get the number of results</li>
36-
<li>Once you have a number, click Fetch to show the dois as bibtex</li>
37-
<li>save the content in a .bib file</li>
38-
<li>import it in mendeley Desktop</li>
39-
<li>Select all the (untitled) entries and right click > update details</li>
40-
<li>Wait for mendeley to populate each entry</li>
41-
<li>In mendeley, select all entires and export as a bib file</li>
42-
<li>Inport your results in <a href="parsif.al">parsif.al</a></li>
43-
</ol>
135+
<div class="jumbotron">
136+
<h1 class="display-4">Scpus Hack</h1>
137+
<p class="lead">Get a list of DOIs that match your search queries</p>
138+
<hr class="my-4">
139+
<ol>
140+
<li>Type your query by following the <a
141+
href="http://schema.elsevier.com/dtds/document/bkapi/search/SCOPUSSearchTips.htm">doc</a> e.g. <i>TITLE-ABS-KEY(blockchain)
142+
AND TITLE-ABS-KEY(INDUSTRY 4.0) AND (TITLE-ABS-KEY(Security) OR TITLE-ABS-KEY(Privacy))</i></li>
143+
<li>Click "count results" to get the number of results</li>
144+
<li>Once you have a number, click Fetch to resolve DOIs</li>
145+
<li>Once the resolving is down, a button allows you to download a bibtex file</li>
146+
<li>import it in <a href="mendeley.com/">mendeley Desktop</a></li>
147+
<li>Select all the (yet untitled) entries and right click > update details</li>
148+
<li>Wait for mendeley to populate each entry</li>
149+
<li>In mendeley, select all entires and export as a bib file</li>
150+
<li>Inport your results in <a href="https://parsif.al/">parsif.al</a></li>
151+
</ol>
152+
153+
</div>
44154

45155

46156
<form action="query" method="POST">
47157
<p>Query:</p>
48158
<fieldset>
49159

50160

161+
{% if count %}
162+
<textarea id="querybox" name="query" readonly>{{ query }}</textarea>
51163

52164

53-
{% if count %}
54-
<textarea id="querybox" name="query" readonly>{{ query }}</textarea>
165+
</fieldset>
166+
<input type="submit" value="Fetch {{ count }} results (up to 1000)">
167+
{% else %}
168+
<textarea id="querybox" name="query" onkeyup="textAreaChange()" style="width:100%"></textarea>
55169

56170

57-
</fieldset>
58-
<input type="submit" value="Fetch {{ count }} results (up to 1000)">
59-
{% else %}
60-
<textarea id="querybox" name="query" onkeyup="textAreaChange()"></textarea>
61-
<input type="hidden" name="count" value="true">
62171

63172
</fieldset>
64-
<input type="submit" value="Count Results">
65-
{% endif %}
173+
<input class="btn btn-primary" id="main_button" type="button" value="Count Results"
174+
onclick="handle_submit()">
175+
{% endif %}
66176

67177

178+
</form>
68179

69180

70-
</form>
71-
<form action="/home" method="GET">
72-
<input type="submit" value="Reset" />
73-
</form>
181+
<input class="btn btn-secondary" type="button" value="Reset" onclick="resetAll()"/>
182+
<input class="btn btn-success" id="dlresults" type="button" value="Download dois as bibtex" onclick="downloadResults()"
183+
hidden/>
184+
<div>
185+
<div>
186+
<label for="file">Resolved DOIs:</label>
187+
<progress id="pb_success" max="0" value="0"></progress>
188+
<span id="pb_success_count"></span>
189+
</div>
190+
<div>
191+
<label for="file">Entry without DOIs:</label>
192+
<progress id="pb_failure" max="0" value="0"></progress>
193+
<span id="pb_failure_count"></span>
194+
</div>
195+
</div>
196+
</div>
197+
<table class="table">
198+
<thead>
199+
<td>DOI</td>
200+
<td>Title</td>
201+
<td>Sci-Hb link</td>
202+
203+
</thead>
204+
<tbody id="doi_table">
205+
206+
207+
</tbody>
208+
209+
</table>
74210

75211
</body>
76212

‎app/uwsgi.ini

+4
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@
22
module = app.main
33
callable = app
44
buffer-size = 8192
5+
http-websockets = true
6+
gevent = 1
7+
async = 20
8+
logto = /var/log/syslog

0 commit comments

Comments
 (0)