[Proposition] Results only competitions (with copy to predictions dir)

compute_worker/compute_worker.py

@@ -219,6 +219,7 @@ def __init__(self, run_args):
 
         # Details for submission
         self.is_scoring = run_args["is_scoring"]
+        self.submission_is_result_only = run_args["submission_is_result_only"]  # true when the submission phase only accepts results submissions
         self.user_pk = run_args["user_pk"]
         self.submission_id = run_args["id"]
         self.submissions_api_url = run_args["submissions_api_url"]
@@ -585,10 +586,19 @@ def _get_host_path(self, *paths):
 
     async def _run_program_directory(self, program_dir, kind):
         """
-        Function responsible for running program directory
+        This function is executed 2 times during the ingestion and scoring.
+        Ingestion:
+            - During ingestion this function is called for program(submission) and ingestion
+            - For ingestion, a metadata file is expected and if not provided, an error is raised
+            - For program(submission), a metadata file is optional
+
+        Scoring:
+            - During scoring this function is called for program(scoring) and ingestion
+            - Because there is no ingestion during scoring, the function returns without executing ingestion
+            - For program(scoring), a metadata file is optional
 
         Args:
-            - program_dir : can be either ingestion program or program/submission
+            - program_dir : can be either ingestion program or program(submission or scoring)
             - kind : either `program` or `ingestion`
         """
         # If the directory doesn't even exist, move on
@@ -599,6 +609,9 @@ async def _run_program_directory(self, program_dir, kind):
             self.completed_program_counter += 1
             return
 
+        # metadata or metadata.yaml file is optional for program(submission/scoring)
+        # result submission is not supposed to have metadata file because it has no program to run just some predictions
+        # if a code submission has no metadata file, it means the ingestion knows how to run the code submission without a metadata file
         if os.path.exists(os.path.join(program_dir, "metadata.yaml")):
             metadata_path = 'metadata.yaml'
         elif os.path.exists(os.path.join(program_dir, "metadata")):
@@ -609,10 +622,19 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.info(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
+                # Copy only if is result only phase i.e. self.submission_is_result_only is True
+                # Copy program dir content to output directory because in case of only result submission,
+                # we need to copy the result submission files because the scoring program will use these as predictions
+                print(f"---IHSAN---{self.submission_is_result_only}")
+                if self.submission_is_result_only:
+                    shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")
 
+        # If metadata file is found, then we check for command in the metadata
+        # If command is not there for ingestion, an error is raised
+        # If command is not there for submission/scoring, a warning is displayed
         logger.info(f"Metadata path is {os.path.join(program_dir, metadata_path)}")
         with open(os.path.join(program_dir, metadata_path), 'r') as metadata_file:
             try:  # try to find a command in the metadata, in other cases set metadata to None

src/apps/api/serializers/competitions.py

@@ -45,6 +45,7 @@ class Meta:
             'hide_output',
             'hide_prediction_output',
             'hide_score_output',
+            'accepts_only_result_submissions',
             'leaderboard',
             'public_data',
             'starting_kit',
@@ -128,6 +129,7 @@ class Meta:
             'hide_output',
             'hide_prediction_output',
             'hide_score_output',
+            'accepts_only_result_submissions',
             # no leaderboard
             'public_data',
             'starting_kit',

src/apps/api/views/competitions.py

@@ -277,6 +277,7 @@ def update(self, request, *args, **kwargs):
                             hide_output=phase["hide_output"],
                             hide_prediction_output=phase["hide_prediction_output"],
                             hide_score_output=phase["hide_score_output"],
+                            accepts_only_result_submissions=phase["accepts_only_result_submissions"],
                             competition=Competition.objects.get(id=data['id'])
                         )
                         # Get phase id

src/apps/competitions/migrations/0059_phase_accepts_only_result_submissions.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2025-06-19 07:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0058_phase_hide_prediction_output'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='phase',
+            name='accepts_only_result_submissions',
+            field=models.BooleanField(default=False),
+        ),
+    ]

src/apps/competitions/models.py

@@ -357,6 +357,8 @@ class Phase(ChaHubSaveMixin, models.Model):
     public_data = models.ForeignKey('datasets.Data', on_delete=models.SET_NULL, null=True, blank=True, related_name="phase_public_data")
     starting_kit = models.ForeignKey('datasets.Data', on_delete=models.SET_NULL, null=True, blank=True, related_name="phase_starting_kit")
 
+    accepts_only_result_submissions = models.BooleanField(default=False)
+
     class Meta:
         ordering = ('index',)
 

src/apps/competitions/tasks.py

@@ -84,6 +84,7 @@
     'hide_output',
     'hide_prediction_output',
     'hide_score_output',
+    'accepts_only_result_submissions',
 ]
 PHASE_FILES = [
     "input_data",
@@ -130,6 +131,7 @@ def _send_to_compute_worker(submission, is_scoring):
         "execution_time_limit": min(MAX_EXECUTION_TIME_LIMIT, submission.phase.execution_time_limit),
         "id": submission.pk,
         "is_scoring": is_scoring,
+        "submission_is_result_only": submission.phase.accepts_only_result_submissions
     }
 
     if not submission.detailed_result.name and submission.phase.competition.enable_detailed_results:

src/apps/competitions/tests/unpacker_test_data.py

@@ -216,6 +216,7 @@
         'hide_output': False,
         'hide_prediction_output': False,
         'hide_score_output': False,
+        'accepts_only_result_submissions': False
     },
     {
         'index': 1,
@@ -236,6 +237,7 @@
         'hide_output': False,
         'hide_prediction_output': False,
         'hide_score_output': False,
+        'accepts_only_result_submissions': False
     }
 ]
 

src/apps/competitions/unpackers/v1.py

@@ -91,6 +91,7 @@ def _unpack_phases(self):
                 'hide_output': phase.get('hide_output', False),
                 'hide_prediction_output': phase.get('hide_prediction_output', False),
                 'hide_score_output': phase.get('hide_score_output', False),
+                'accepts_only_result_submissions': phase.get('accepts_only_result_submissions', False)
             }
             execution_time_limit = phase.get('execution_time_limit')
             if execution_time_limit:

src/apps/competitions/unpackers/v2.py

@@ -200,6 +200,7 @@ def _unpack_phases(self):
                 'hide_output': phase_data.get('hide_output', False),
                 'hide_prediction_output': phase_data.get('hide_prediction_output', False),
                 'hide_score_output': phase_data.get('hide_score_output', False),
+                'accepts_only_result_submissions': phase_data.get('accepts_only_result_submissions', False)
             }
             try:
                 new_phase['tasks'] = phase_data['tasks']

src/static/riot/competitions/editor/_phases.tag

@@ -216,6 +216,16 @@
                             </div>
                         </div>
 
+                        <div class="field">
+                            <div class="ui checkbox">
+                                <label>Accept only result submissions 
+                                    <span data-tooltip="If checked, phase will accept only result submissions" data-inverted=""
+                              data-position="bottom center"><i class="help icon circle"></i></span>
+                                </label>
+                                <input type="checkbox" ref="accepts_only_result_submissions">
+                            </div>
+                        </div>
+
                         <div class="inline field" if="{phases.length > 0 && ![null, undefined, 0].includes(selected_phase_index)}">
                             <div class="ui checkbox">
                                 <input type="checkbox" name="auto_migrate_to_this_phase" ref="auto_migrate">
@@ -663,6 +673,7 @@
             self.refs.hide_output.checked = phase.hide_output
             self.refs.hide_prediction_output.checked = phase.hide_prediction_output
             self.refs.hide_score_output.checked = phase.hide_score_output
+            self.refs.accepts_only_result_submissions.checked = phase.accepts_only_result_submissions
 
             // Setting description in markdown editor
             self.simple_markdown_editor.value(self.phases[index].description || '')
@@ -829,6 +840,7 @@
             data.hide_output = self.refs.hide_output.checked
             data.hide_prediction_output = self.refs.hide_prediction_output.checked
             data.hide_score_output = self.refs.hide_score_output.checked
+            data.accepts_only_result_submissions = self.refs.accepts_only_result_submissions.checked
             _.forEach(number_fields, field => {
                 let str = _.get(data, field)
                 if (str) {