AbArray And CL EDA

Model/bin/enrichmentAnalysis

@@ -123,7 +123,7 @@ def main(argv = None):
         results = results[results.pValue <= pValueCutoff]
 
         # sort on p-value asc
-        results.sort(columns = 'pValue', ascending = True, inplace = True)
+        results.sort_values(by = 'pValue', ascending = True, inplace = True)
 
         # print results.shape
 

Model/lib/dst/categories.dst

@@ -25,9 +25,10 @@ anchorFile=ApiCommonModel/Model/lib/wdk/ontology/individuals.txt
 prop=presenterId
 prop=datasetName
 prop=profileSamplesHelp
+prop=cleanDatasetName
 >templateTextStart<
 normalizedSampleValues${datasetName}	DatasetRecordClasses.DatasetRecordClass.${presenterId}				normalizedSampleValuesInternal${datasetName}	normalized sample values		${profileSamplesHelp}			internal		
-TranscriptAttributes.MetaProfileSamples${datasetName}	normalizedSampleValues${datasetName}		TranscriptRecordClasses.TranscriptRecordClass	attributeMetaQuery	MetaProfileSamples${datasetName}			${profileSamplesHelp}	gene		results	download	
+TranscriptAttributes.MetaProfileSamples${cleanDatasetName}	normalizedSampleValues${datasetName}		TranscriptRecordClasses.TranscriptRecordClass	attributeMetaQuery	MetaProfileSamples${cleanDatasetName}			${profileSamplesHelp}	gene		results	download	
 >templateTextEnd<
 
 

Model/lib/dst/cellularLocalization.dst

@@ -18,8 +18,9 @@ prop=cellularLocalizationWdkAttributes
         <paramRef ref="geneParams.eda_dataset_id" default="${presenterId}" visible="false"/>
         <paramRef ref="geneParams.eda_analysis_spec" allowEmpty="true" prompt="Filter genes based on cellular localization data"/>
 
+<!-- TODO - add in attributesList, when available: ${cellularLocalizationWdkAttributes} -->
          <attributesList
-              summary="organism,gene_location_text,gene_product,${cellularLocalizationWdkAttributes}"
+              summary="organism,gene_location_text,gene_product"
         />
 
 	<summary><![CDATA[

Model/lib/dst/expression.dst

@@ -12,32 +12,22 @@ prop=projectName
              <column name="term"/>
           <sql>
            <![CDATA[
-select s.study_id as internal
+select distinct s.node_set_id as internal
      , s.name as term
-     , s.name as display
-from study.study s
-where name = '${datasetName}'
-
-/**
-select distinct s.study_id as internal
-     , s.name as term
-     , REGEXP_REPLACE (s.name, '\[(.)+\]', '')  AS display 
+     , REGEXP_REPLACE (s.name, '\[(.)+\]', '')  AS display
      , d.name
-from study.study s
-   , study.studylink sl
+from study.nodeset s
+   , study.nodenodeset sl
    , RESULTS.NAFEATUREDIFFRESULT dr
    , SRES.EXTERNALDATABASERELEASE r
    , sres.externaldatabase d
-where 
+where
 s.EXTERNAL_DATABASE_RELEASE_ID = r.EXTERNAL_DATABASE_RELEASE_ID
 and r.EXTERNAL_DATABASE_ID = d.EXTERNAL_DATABASE_ID
-and s.study_id = sl.study_id
+and s.node_set_id = sl.node_set_id
 and sl.PROTOCOL_APP_NODE_ID = dr.PROTOCOL_APP_NODE_ID
-and s.INVESTIGATION_ID is null
 and d.name = '${datasetName}'
 ORDER BY s.name desc
-**/
-
             ]]>
           </sql>
         </sqlQuery>

Model/lib/dst/metaCycle.dst

@@ -68,11 +68,12 @@ prop=includeProjectsExcludeEuPathDB
              <column name="term"/>
           <sql>
            <![CDATA[
-
 	   SELECT s.name as internal, REPLACE (s.name, ' MetaCycle', '') as term
-	   FROM STUDY.STUDY s, STUDY.STUDY i
-	   WHERE i.name = '${datasetName}'
-	   AND i.study_id = s.INVESTIGATION_ID and s.name like '%MetaCycle%'
+     FROM STUDY.nodeset s, sres.externaldatabase ed, sres.externaldatabaserelease edr
+	   WHERE ed.name = '${datasetName}'
+     AND ed.external_database_id = edr.external_database_id
+     AND edr.external_database_release_id = s.external_database_release_id
+     AND node_type IN ('RNASeq','microarray')
             ]]>
           </sql>
         </sqlQuery>

Model/lib/dst/phenotype.dst

@@ -54,7 +54,7 @@ prop=phenotypeWdkAttributes
         <paramRef ref="geneParams.eda_analysis_spec" allowEmpty="true" prompt="Filter genes based on phenotype data"/>
 
          <attributesList 
-              summary="organism,gene_location_text,gene_product,${phenotypeWdkAttributes}"
+              summary="organism,gene_location_text,gene_product"
         />
 
 	<summary><![CDATA[

Model/lib/dst/profileGraphs.dst

@@ -162,10 +162,11 @@ prop=isGraphCustom
 name=profileAttributeQueries
 anchorFile=ApiCommonModel/Model/lib/wdk/model/records/transcriptAttributeQueries.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=projectName
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
-        <sqlQuery name="ProfileSamples${datasetName}" doNotTest="true" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${datasetName}"  includeProjects="${includeProjectsExcludeEuPathDB}">
+        <sqlQuery name="ProfileSamples${cleanDatasetName}" doNotTest="true" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${cleanDatasetName}"  includeProjects="${includeProjectsExcludeEuPathDB}">
             <column name="source_id"/>
             <column name="gene_source_id"/>
             <column name="project_id"/>
@@ -192,7 +193,7 @@ prop=includeProjectsExcludeEuPathDB
             </sql>
         </sqlQuery>
 
-       <sqlQuery name="MetaProfileSamples${datasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
+       <sqlQuery name="MetaProfileSamples${cleanDatasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
           <column name="name" />
           <column name="display_name" />
           <column name="help" />
@@ -259,8 +260,9 @@ prop=projectName
 prop=sense
 prop=antisense
 prop=includeProjectsExcludeEuPathDB
+prop=cleanDatasetName
 >templateTextStart<
-        <sqlQuery name="ProfileSamples${datasetName}" doNotTest="true" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${datasetName}"  includeProjects="${includeProjectsExcludeEuPathDB}">
+        <sqlQuery name="ProfileSamples${cleanDatasetName}" doNotTest="true" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${cleanDatasetName}"  includeProjects="${includeProjectsExcludeEuPathDB}">
             <column name="source_id"/>
             <column name="gene_source_id"/>
             <column name="project_id"/>
@@ -287,7 +289,7 @@ prop=includeProjectsExcludeEuPathDB
             </sql>
         </sqlQuery>
 
-       <sqlQuery name="MetaProfileSamples${datasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
+       <sqlQuery name="MetaProfileSamples${cleanDatasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
           <column name="name" />
           <column name="display_name" />
           <column name="help" />
@@ -358,8 +360,9 @@ prop=datasetName
 prop=presenterId
 prop=includeProjectsExcludeEuPathDB
 prop=projectName
+prop=cleanDatasetName
 >templateTextStart<
-              <attributeQueryRef ref="TranscriptAttributes.ProfileSamples${datasetName}" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${datasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
+              <attributeQueryRef ref="TranscriptAttributes.ProfileSamples${cleanDatasetName}" attributeMetaQueryRef="TranscriptAttributes.MetaProfileSamples${cleanDatasetName}" includeProjects="${includeProjectsExcludeEuPathDB}">
               </attributeQueryRef>
 >templateTextEnd<
 

Model/lib/dst/quantProteomics.dst

@@ -83,17 +83,17 @@ prop=dataType
              <column name="internal"/>
              <column name="term"/>
           <sql>
-select distinct s.study_id as internal, s.name as term, s.name as display
-from study.study s, sres.externaldatabase ed, sres.externaldatabaserelease edr 
-  ,results.nafeaturediffresult ndr, study.protocolappnode pan, study.studylink sl
+select distinct s.node_set_id as internal, s.name as term, s.name as display
+from study.nodeset s, sres.externaldatabase ed, sres.externaldatabaserelease edr
+  ,results.nafeaturediffresult ndr, study.protocolappnode pan, study.nodenodeset sl
 where ed.external_database_id=edr.external_database_id
 and s.external_database_release_id = edr.external_database_release_id
 and ed.name='${datasetName}'
-and s.study_id = sl.study_id
+and s.node_set_id = sl.node_set_id
 and sl.protocol_app_node_id =  pan.protocol_app_node_id
 and ndr.protocol_app_node_id = pan.protocol_app_node_id
 and s.name NOT like 'Quantitative Mass Spec%'
-order by s.study_id
+order by s.node_set_id
           </sql>
         </sqlQuery>
 

Model/lib/dst/rnaSeqTemplates.dst

@@ -173,6 +173,7 @@ prop=organismAbbrevDisplay
 name=rnaSeqFoldChangeQuestion
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/geneQuestions.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName 
 prop=datasetShortDisplayName
 prop=projectName
@@ -190,8 +191,8 @@ prop=includeProjectsExcludeEuPathDB
          searchCategory="Transcript Expression"
          queryRef="GeneId.GenesByRnaSeqFoldChange"
          recordClassRef="TranscriptRecordClasses.TranscriptRecordClass">
-        <paramRef ref="geneParams.profileset_generic" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${datasetName}RnaSeqProfiles"/>
-        <paramRef ref="geneParams.dataset_url" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${datasetName}DatasetUrl"/>
+        <paramRef ref="geneParams.profileset_generic" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${cleanDatasetName}RnaSeqProfiles"/>
+        <paramRef ref="geneParams.dataset_url" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${cleanDatasetName}DatasetUrl"/>
 	<paramRef ref="geneParams.hard_floor" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.hardFloorRnaSeq" visible="true"/>
 	<sqlParamValue name="isLogged">0</sqlParamValue>
 	<attributesList includeProjects="${includeProjectsExcludeEuPathDB}"
@@ -313,6 +314,7 @@ ${optionalQuestionDescription}
 name=rnaSeqPercentileQuestion
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/geneQuestions.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName 
 prop=datasetShortDisplayName
 prop=projectName
@@ -324,14 +326,14 @@ prop=optionalQuestionDescription
 prop=organismAbbrevDisplay
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
-    <question name="GenesByRNASeq${datasetName}Percentile" includeProjects="${includeProjects}" newBuild="${buildNumberIntroduced}"
+    <question name="GenesByRNASeq${cleanDatasetName}Percentile" includeProjects="${includeProjects}" newBuild="${buildNumberIntroduced}"
          displayName="${organismAbbrevDisplay} ${datasetDisplayName} RNA-Seq (percentile)"
          shortDisplayName="${datasetShortDisplayName} (%ile)"
          searchCategory="Transcript Expression"
          queryRef="GeneId.GenesByGenericPercentile" 
          recordClassRef="TranscriptRecordClasses.TranscriptRecordClass">
 
-        <paramRef ref="geneParams.profileset_generic" queryRef="GeneVQ.${datasetName}RnaSeqPctProfiles"/>
+        <paramRef ref="geneParams.profileset_generic" queryRef="GeneVQ.${cleanDatasetName}RnaSeqPctProfiles"/>
 
         <attributesList includeProjects="${includeProjectsExcludeEuPathDB}"
               summary="gene_product,min_percentile_chosen,max_percentile_chosen,${pctGraphAttr}"
@@ -385,6 +387,7 @@ ${optionalQuestionDescription}
 name=rnaSeqDESeqQuestion
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/geneQuestions.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName
 prop=datasetShortDisplayName
 prop=projectName
@@ -395,13 +398,13 @@ prop=optionalQuestionDescription
 prop=organismAbbrevDisplay
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
- <question name="GenesByRNASeq${datasetName}DESeq" newBuild="29"
+ <question name="GenesByRNASeq${cleanDatasetName}DESeq" newBuild="29"
          displayName="${organismAbbrevDisplay} ${datasetDisplayName} RNA-Seq (Differential Expression)" includeProjects="${includeProjects}"
          shortDisplayName="${datasetShortDisplayName} (de)"
          searchCategory="Transcript Expression"
          queryRef="GeneId.GenesByDESeqAnalysis"
          recordClassRef="TranscriptRecordClasses.TranscriptRecordClass">
-        <paramRef ref="geneParams.profileset_generic" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${datasetName}DESeqProfiles"/>
+        <paramRef ref="geneParams.profileset_generic" groupRef="paramGroups.dynamicParams" queryRef="GeneVQ.${cleanDatasetName}DESeqProfiles"/>
         <attributesList includeProjects="${includeProjectsExcludeEuPathDB}"
            summary="organism,gene_product, fold_change, adj_p_value"
            sorting="fold_change desc" />
@@ -596,13 +599,14 @@ prop=includeProjectsExcludeEuPathDB
 name=rnaSeqProfileSetParamQuery
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/params/geneParams.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName 
 prop=projectName
 prop=sense
 prop=antisense
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
-      <sqlQuery name="${datasetName}RnaSeqProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
+      <sqlQuery name="${cleanDatasetName}RnaSeqProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
              <column name="display"/>
              <column name="internal"/>
              <column name="term"/>
@@ -630,7 +634,7 @@ ORDER BY term desc
             ]]>
           </sql>
         </sqlQuery>
-        <processQuery name="${datasetName}RnaSeqProfiles" processName="org.apidb.apicomplexa.wsfplugin.apifed.ApiFedPlugin" includeProjects="EuPathDB"> 
+        <processQuery name="${cleanDatasetName}RnaSeqProfiles" processName="org.apidb.apicomplexa.wsfplugin.apifed.ApiFedPlugin" includeProjects="EuPathDB"> 
             <wsColumn name="display" width="300"/>
             <wsColumn name="internal" width="300"/>
             <wsColumn name="term" width="300"/>
@@ -642,22 +646,23 @@ ORDER BY term desc
 name=datasetUrlParamQuery
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/params/geneParams.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=projectName
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
-      <sqlQuery name="${datasetName}DatasetUrl" includeProjects="${includeProjectsExcludeEuPathDB},Huh">
+      <sqlQuery name="${cleanDatasetName}DatasetUrl" includeProjects="${includeProjectsExcludeEuPathDB},Huh">
              <column name="internal"/>
              <column name="term"/>
           <sql>
            <![CDATA[
 SELECT 'https://${projectName}.org/a/app/record/dataset/' || dataset_presenter_id as internal,
         'https://${projectName}.org/a/app/record/dataset/' || dataset_presenter_id as term
-FROM ApidbTuning.DatasetPresenter
+FROM ApidbTuning.DatasetDatasource
 WHERE name = '${datasetName}'
             ]]>
           </sql>
         </sqlQuery>
-        <processQuery name="${datasetName}DatasetUrl" processName="org.apidb.apicomplexa.wsfplugin.apifed.ApiFedPlugin" includeProjects="EuPathDB"> 
+        <processQuery name="${cleanDatasetName}DatasetUrl" processName="org.apidb.apicomplexa.wsfplugin.apifed.ApiFedPlugin" includeProjects="EuPathDB"> 
             <wsColumn name="internal" width="300"/>
             <wsColumn name="term" width="300"/>
         </processQuery>
@@ -668,13 +673,14 @@ WHERE name = '${datasetName}'
 name=rnaSeqPctProfileSetParamQuery
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/params/geneParams.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName 
 prop=projectName
 prop=sense
 prop=includeProjectsExcludeEuPathDB
 prop=antisense
 >templateTextStart<
-      <sqlQuery name="${datasetName}RnaSeqPctProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
+      <sqlQuery name="${cleanDatasetName}RnaSeqPctProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
              <column name="display"/>
              <column name="internal"/>
              <column name="term"/>
@@ -715,13 +721,14 @@ ORDER BY term desc
 name=DESeqProfileSetParamQuery
 anchorFile=ApiCommonModel/Model/lib/wdk/model/questions/params/geneParams.xml
 prop=datasetName
+prop=cleanDatasetName
 prop=datasetDisplayName 
 prop=projectName
 prop=sense
 prop=antisense
 prop=includeProjectsExcludeEuPathDB
 >templateTextStart<
-      <sqlQuery name="${datasetName}DESeqProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
+      <sqlQuery name="${cleanDatasetName}DESeqProfiles" includeProjects="${includeProjectsExcludeEuPathDB}">
              <column name="display"/>
              <column name="internal"/>
              <column name="term"/>
@@ -734,14 +741,14 @@ SELECT study_id AS internal,
     ELSE  ps_name || sense END AS display
 FROM (
 
-    SELECT DISTINCT s.study_id as study_id
+    SELECT DISTINCT s.node_set_id as study_id
      , s.name as name
      , REGEXP_REPLACE (s.name, '\[(.)+\]', '')  AS ps_name
      ,REPLACE (REPLACE ( REPLACE ( REPLACE (  REGEXP_SUBSTR (s.name, ' \[\S+ ') , 'DESeq2Analysis', '')  , 'unique' , '' ), '[', '') , '-', ' ' )  AS anal
      ,CASE WHEN REGEXP_LIKE  (s.name, '${sense}')  THEN  '- Sense'
       WHEN REGEXP_LIKE  (s.name, ' ${antisense}')  THEN  '- Antisense'
        ELSE 'unstranded' END AS sense
-from study.study s
+from study.nodeset s
    , SRES.EXTERNALDATABASERELEASE r
    , sres.externaldatabase d
 where lower(s.name) like '%deseq%'
@@ -807,13 +814,8 @@ prop=includeProjectsExcludeEuPathDB
            <![CDATA[
 	    SELECT num AS internal, num ||' hours' AS term, num ||' hours' AS display
 	    FROM
-	    (SELECT Rownum num
- 	     CONNECT BY Rownum <= '${profileTimeShift}'
-	     UNION
-	     SELECT 0 num
-	     UNION
-	     SELECT Rownum * (-1) num
-	     CONNECT BY Rownum <= '${profileTimeShift}'
+	    (SELECT generate_series(-${profileTimeShift}.0::integer, ${profileTimeShift}.0::integer) 
+             AS num ORDER BY num
 	    ) t
 	    ORDER BY t.num ASC
             ]]>
@@ -1154,11 +1156,12 @@ prop=includeProjectsExcludeEuPathDB
              <column name="term"/>
           <sql>
            <![CDATA[
-
 	   SELECT s.name as internal, REPLACE (s.name, ' MetaCycle', '') as term
-	   FROM STUDY.STUDY s, STUDY.STUDY i
-	   WHERE i.name = '${datasetName}'
-	   AND i.study_id = s.INVESTIGATION_ID and s.name like '%MetaCycle%'
+     FROM STUDY.nodeset s, sres.externaldatabase ed, sres.externaldatabaserelease edr
+	   WHERE ed.name = '${datasetName}'
+     AND ed.external_database_id = edr.external_database_id
+     AND edr.external_database_release_id = s.external_database_release_id
+     AND node_type IN ('RNASeq')
             ]]>
           </sql>
         </sqlQuery>

Model/lib/perl/JBrowseUtil.pm

@@ -129,7 +129,7 @@ my @tester = %{$args};
   if($buildNumber && $webservicesDir && $nameForFileNames ) {
     #todo:  construct the path
     #/var/www/Common/apiSiteFilesMirror/webServices/ToxoDB/build-68/TgondiiME49
-    my $webServicePropertiesFile = "$webservicesDir/$projectName/build-$buildNumber/$nameForFileNames/genomeAndProteome/config/jbrowse.conf";
+    my $webServicePropertiesFile = "$webservicesDir/$projectName/build-$buildNumber/$nameForFileNames/genomeBrowser/config/jbrowse.conf";
     $webServicePropsHash = $self->makeProperties($webServicePropertiesFile);
   }