Skip to content
This repository was archived by the owner on Jan 25, 2018. It is now read-only.

Commit 9c630bd

Browse files
author
Lee Kamentsky
committed
Largely working version of BatchProfiler
1 parent 1639676 commit 9c630bd

16 files changed

+1257
-747
lines changed

‎BuildCellProfiler.py

+16-11
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,17 @@
2121
import os
2222

2323
from bputilities import build_cellprofiler, get_version_and_githash, is_built
24+
from bputilities import CellProfilerContext
2425
from bpformdata import RM_GET, RM_PUT, REQUEST_METHOD, REVISION, GIT_HASH, \
25-
DATETIME_VERSION, IS_BUILT, QUEUE, PROJECT, EMAIL, BATCHPROFILER_DEFAULTS
26+
DATETIME_VERSION, IS_BUILT, QUEUE, PROJECT, EMAIL, \
27+
BATCHPROFILER_DEFAULTS, BATCHPROFILER_VARIABLES
2628

27-
query_revision = BATCHPROFILER_DEFAULTS[REVISION]
29+
query_revision = BATCHPROFILER_VARIABLES[REVISION]
2830
def do_get():
2931
if query_revision is not None:
30-
datetime_version, git_hash = get_version_and_githash(query_revision)
31-
buildstatus = is_built(version=datetime_version, git_hash=git_hash)
32+
with CellProfilerContext():
33+
datetime_version, git_hash = get_version_and_githash(query_revision)
34+
buildstatus = is_built(version=datetime_version, git_hash=git_hash)
3235
print "Content-Type: application/json\r"
3336
print "\r"
3437
print json.dumps(
@@ -75,17 +78,19 @@ def do_get():
7578
def do_put():
7679
datetime_version, git_hash = get_version_and_githash(query_revision)
7780
buildstatus = is_built(version=datetime_version, git_hash=git_hash)
81+
options = {}
7882
if not buildstatus:
7983
try:
80-
options = json.load(data)
84+
options = json.load(sys.stdin)
8185
assert isinstance(options, dict)
8286
except:
83-
options = {}
84-
build_cellprofiler(version = datetime_version,
85-
git_hash = git_hash,
86-
queue_name = options.get(QUEUE, None),
87-
group_name = options.get(PROJECT, None),
88-
email_address = options.get(EMAIL, None))
87+
pass
88+
with CellProfilerContext() :
89+
build_cellprofiler(version = datetime_version,
90+
git_hash = git_hash,
91+
queue_name = options.get(QUEUE, None),
92+
group_name = options.get(PROJECT, None),
93+
email_address = options.get(EMAIL, None))
8994

9095
print "Content-Type: application/json\r"
9196
print "\r"

‎DeleteFile.py

100644100755
+4-6
Original file line numberDiff line numberDiff line change
@@ -29,20 +29,18 @@ def remove_if_exists(path):
2929
def delete_run(my_batch, my_run):
3030
if delete_action in (A_DELETE_ALL, A_DELETE_TEXT):
3131
remove_if_exists(RunBatch.run_text_file_path(my_batch, my_run))
32+
remove_if_exists(RunBatch.run_err_file_path(my_batch, my_run))
3233

3334
if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT):
34-
remove_if_exists(RunBatch.RunOutFilePath(my_batch, my_run))
35+
remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
3536

36-
form = cgi.FieldStorage()
3737
delete_action = BATCHPROFILER_DEFAULTS[K_DELETE_ACTION]
38-
if delete_action is not None:
39-
delete_action = delete_action.upper()
4038
run_id = BATCHPROFILER_DEFAULTS[RUN_ID]
4139
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
4240
if run_id is not None and delete_action is not None:
4341
my_run = RunBatch.BPRun.select(run_id)
4442
my_batch = RunBatch.BPBatch()
45-
batch.select(run.batch_id)
43+
my_batch.select(my_run.batch_id)
4644
delete_run(my_batch, my_run)
4745
elif batch_id is not None:
4846
my_batch = RunBatch.BPBatch()
@@ -51,7 +49,7 @@ def delete_run(my_batch, my_run):
5149
delete_run(my_batch, my_run)
5250

5351

54-
url = "ViewBatch.py?batch_id=%(batch_id)d"%(my_batch)
52+
url = "ViewBatch.py?batch_id=%d"%(my_batch.batch_id)
5553
print "Content-Type: text/html"
5654
print
5755
print "<html><head>"

‎FixPermissions.py

+13-8
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ def maybe_chmod(path, mode, d):
3030
d - add the key/value of path:mode to the dictionary if the mode was changed
3131
'''
3232
if os.path.exists(path) and \
33-
stat.S_IMODE(os.stat(directory).st_mode) != mode:
34-
os.chmod(directory, mode)
35-
what_i_did[directory] = mode
33+
stat.S_IMODE(os.stat(path).st_mode) != mode:
34+
os.chmod(path, mode)
35+
d[path] = mode
3636

3737
def handle_post():
3838
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
@@ -76,7 +76,7 @@ def handle_post():
7676
var count = Object.keys(result).length
7777
alert("Changed permissions for "+count+" file(s) / folder(s)");
7878
} else {
79-
alert("Failed to change permissions.\n" + xmlhttp.responseText);
79+
alert("Failed to change permissions.\\n" + xmlhttp.responseText);
8080
}
8181
button.innerText = oldInnerText;
8282
button.disabled = false;
@@ -96,8 +96,7 @@ def handle_get():
9696
'''Display a form for fixing the permissions'''
9797
batch_id_id = "input_%s" % BATCH_ID
9898
button_id = "button_%s" % BATCH_ID
99-
fix_permissions_action = \
100-
"fix_permissions(document.getElementById('%s'));" % button_id
99+
fix_permissions_action = "on_click_%s()" % button_id
101100
doc, tag, text = yattag.Doc().tagtext()
102101
assert isinstance(doc, yattag.Doc)
103102
with tag("html"):
@@ -106,10 +105,14 @@ def handle_get():
106105
text(TITLE)
107106
with tag("script", language="JavaScript"):
108107
doc.asis(FIX_PERMISSIONS_AJAX_JAVASCRIPT)
108+
doc.asis("""
109+
function on_click_%s() {
110+
fix_permissions(document.getElementById('%s'));
111+
}""" % (button_id, button_id))
109112
with tag("body"):
110113
with tag("h1"):
111114
text(TITLE)
112-
with tag("div"):
115+
with tag("div", style="width: 6in; margin-bottom: 24pt"):
113116
text("""
114117
This webpage fixes permission problems for the files and folders in your batch
115118
that were created outside BatchProfiler's control. It will grant read
@@ -129,7 +132,9 @@ def handle_get():
129132
'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
130133
print yattag.indent(doc.getvalue())
131134

132-
if __name__ == "__main__":
135+
script_filename = os.environ.get(SCRIPT_FILENAME_KEY, None)
136+
if script_filename is not None and \
137+
script_filename.endswith("FixPermissions.py"):
133138
import cgitb
134139
cgitb.enable()
135140

‎GetPipeline.py

100644100755
+19-16
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/usr/bin/env ./python-2.6.sh
1+
#!/usr/bin/env ./batchprofiler.sh
22
"""
33
CellProfiler is distributed under the GNU General Public License.
44
See the accompanying file LICENSE for details.
@@ -19,32 +19,35 @@
1919
import cgi
2020

2121
import os, sys
22+
import bputilities
23+
from bpformdata import *
2224
import RunBatch
2325
from cellprofiler.utilities.hdf5_dict import HDF5Dict
2426
M_USER_PIPELINE = "Pipeline_UserPipeline"
2527
M_PIPELINE = "Pipeline_Pipeline"
2628
F_BATCH_DATA_H5 = "Batch_data.h5"
2729
EXPERIMENT = "Experiment"
2830

29-
form = cgi.FieldStorage()
30-
batch_id = int(form["batch_id"].value)
31-
my_batch = RunBatch.LoadBatch(batch_id)
32-
path = os.path.join(my_batch["data_dir"], F_BATCH_DATA_H5)
33-
h = HDF5Dict(path, mode="r")
34-
if M_USER_PIPELINE in h.second_level_names(EXPERIMENT):
35-
feature = M_USER_PIPELINE
36-
else:
37-
feature = M_PIPELINE
38-
pipeline_text = h[EXPERIMENT, feature, 0][0].decode("unicode_escape")
39-
h.close()
40-
del h
41-
if isinstance(pipeline_text, unicode):
42-
pipeline_text = pipeline_text.encode("us-ascii")
31+
with bputilities.CellProfilerContext():
32+
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
33+
my_batch = RunBatch.BPBatch()
34+
my_batch.select(batch_id)
35+
path = RunBatch.batch_data_file_path(my_batch)
36+
h = HDF5Dict(path, mode="r")
37+
if M_USER_PIPELINE in h.second_level_names(EXPERIMENT):
38+
feature = M_USER_PIPELINE
39+
else:
40+
feature = M_PIPELINE
41+
pipeline_text = h[EXPERIMENT, feature, 0][0].decode("unicode_escape")
42+
h.close()
43+
del h
44+
if isinstance(pipeline_text, unicode):
45+
pipeline_text = pipeline_text.encode("us-ascii")
4346
print "Content-Type: text/plain"
4447
print "Content-Length: %d" % len(pipeline_text)
4548
print "Content-Disposition: attachment; filename=\"%d.cppipe\"" % batch_id
4649
print
4750
sys.stdout.write(pipeline_text)
4851
sys.stdout.flush()
49-
os._exit(0)
52+
5053

‎JobStatus.py

+9-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import json
2525
import os
2626
import sys
27-
from RunBatch import BPJob
27+
from RunBatch import BPJob, JS_SUBMITTED
2828
from bpformdata import REQUEST_METHOD, RM_PUT
2929

3030
K_ACTION = "action"
@@ -42,12 +42,18 @@
4242
action = data[K_ACTION]
4343
job_id = int(data[K_JOB_ID])
4444
run_id = int(data[K_RUN_ID])
45+
status = data.get(K_STATUS, JS_SUBMITTED)
4546
job = BPJob(run_id, job_id)
4647
if action == A_CREATE:
47-
job.create()
48+
job.create(status)
49+
print "Content-Type: text/plain"
50+
print
51+
print "OK"
4852
elif action == A_UPDATE:
49-
status = data[K_STATUS]
5053
job.update_status(status)
54+
print "Content-Type: text/plain"
55+
print
56+
print "OK"
5157
else:
5258
raise NotImplementedError("Unsupported action: %s" % action)
5359
else:

‎KillJobs.py

+13-10
Original file line numberDiff line numberDiff line change
@@ -20,25 +20,28 @@
2020
cgitb.enable()
2121
import RunBatch
2222
import bputilities
23+
from bpformdata import *
2324
import cgi
2425
import subprocess
2526
import sys
2627

27-
form = cgi.FieldStorage()
28-
if form.has_key("job_id"):
29-
import subprocess
30-
job_id = int(form["job_id"].value)
31-
run = {"job_id":job_id}
32-
RunBatch.KillOne(run)
28+
job_id = BATCHPROFILER_VARIABLES[JOB_ID]
29+
batch_id = BATCHPROFILER_VARIABLES[BATCH_ID]
30+
if job_id is not None:
31+
job = RunBatch.BPJob.select(job_id)
32+
if job is not None:
33+
run = RunBatch.BPRun.select(job.run_id)
34+
RunBatch.kill_one(run)
35+
else:
36+
bputilities.kill_job(job_id)
3337
print"""
3438
<html><head><title>Job %(job_id)d killed</title></head>
3539
<body>Job %(job_id)d killed
3640
</body>
3741
</html>
3842
"""%(globals())
39-
elif form.has_key("batch_id"):
40-
batch_id = int(form["batch_id"].value)
41-
RunBatch.KillBatch(batch_id)
43+
elif batch_id is not None:
44+
RunBatch.kill_batch(batch_id)
4245

4346
url = "ViewBatch.py?batch_id=%d"%(batch_id)
4447
print "<html><head>"
@@ -58,7 +61,7 @@
5861
p = subprocess.Popen(["bash"],stdin = subprocess.PIPE,
5962
stdout=subprocess.PIPE)
6063
listing = p.communicate(
61-
". /broad/software/scripts/useuse;reuse GridEngine8;qjobs\n")[0]
64+
". /broad/software/scripts/useuse;reuse GridEngine8;qstat\n")[0]
6265
listing_lines = listing.split('\n')
6366
header = listing_lines[0]
6467
columns = [header.find(x) for x in header.split(' ') if len(x)]

0 commit comments

Comments
 (0)