update fit curve again

Author: sgosline

build/beatAML/requirements.txt

@@ -5,3 +5,5 @@ synapseclient
 argparse
 numpy
 openpyxl
+matplotlib
+scikit-learn

build/broad_sanger/04-drug_dosage_and_curves.py

@@ -13,6 +13,7 @@
 import os
 import argparse
 import pandas as pd
+import subprocess
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--curSampleFile',dest='samplefile',default=None,help='DepMap sample file')
@@ -24,18 +25,18 @@
 samplefile = opts.samplefile
 drugfile = opts.dfile
 
-cmd = '/opt/venv/bin/python 04b-nci60-updated.py --sampleFile '+samplefile+' --drugFile '+drugfile
+cmd = ['/opt/venv/bin/python 04b-nci60-updated.py','--sampleFile',samplefile,'--drugFile',drugfile]
 print(cmd)
-os.system(cmd)
+subprocess.run(cmd)
 
 ####step 4a - get dose response data
-cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' CTRPv2,FIMM,GDSC'
+cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC']
 print(cmd)
-os.system(cmd)
+subprocess.run(cmd)
 
-cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' gCSI,PRISM,CCLE'
+cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE']
 print(cmd)
-os.system(cmd)
+subprocess.run(cmd)
 
 
 #cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' NCI60'
@@ -46,7 +47,7 @@
 allfiles=[a for a in os.listdir('./') if 'DoseResponse' in a]
 print(allfiles)
 for a in allfiles:
-    os.system('/opt/venv/bin/python fit_curve.py --input '+a+' --output '+a)
+    subprocess.run(['/opt/venv/bin/python','fit_curve.py','--input='+a+,'--output='+a])
 
 ###step 4c concatenate all files
 outfiles = [a for a in os.listdir("./") if ".0" in a]

build/utils/fit_curve.py

@@ -162,8 +162,10 @@ def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None):
     groups = df.groupby(cols)
     count = count or (4484081 - start)
     groups = islice(groups, start, start+count)
-    
-    with multiprocessing.Pool(processes=multiprocessing.cpu_count()/4) as pool:
+    cores = multiprocessing.cpu_count()
+    poolsize = round(cores/2)
+    print('we have '+str(cores)+' cores and '+str(poolsize)+' threads')
+    with multiprocessing.Pool(processes=poolsize) as pool:
         results = pool.map(process_single_drug, groups)
 
     with open(f'{fname}.{start}', 'w') as f: