Skip to content

Add download many results and predictions #2019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: optimise_download_many
Choose a base branch
from
Open

Add download many results and predictions #2019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ee5a9ee
added option to bulk download scores files and predictions files
nicomy Oct 1, 2025
20ff485
cleaned code
nicomy Oct 2, 2025
0886f7c
exclude not finished submission inside download pred and scores
nicomy Oct 3, 2025
1e20fb4
merged
nicomy Oct 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 98 additions & 23 deletions src/apps/api/views/submissions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,28 @@ def destroy(self, request, *args, **kwargs):

self.perform_destroy(submission)
return Response(status=status.HTTP_204_NO_CONTENT)


def check_submission_permissions(self,request,submissions):
# Check permissions
if not request.user.is_authenticated:
raise PermissionDenied("You must be logged in to download submissions")
# Allow admins
if request.user.is_superuser or request.user.is_staff:
allowed = True
else:
# Build one Q object for "owner OR organizer"
organiser_q = (
Q(phase__competition__created_by=request.user) |
Q(phase__competition__collaborators=request.user)
)
# Submissions that violate the rule
disallowed = submissions.exclude(Q(owner=request.user) | organiser_q)
allowed = not disallowed.exists()
if not allowed:
raise PermissionDenied(
"You do not have permission to download one or more of the requested submissions"
)

@action(detail=True, methods=('DELETE',))
def soft_delete(self, request, pk):
Expand Down Expand Up @@ -348,6 +370,8 @@ def re_run_many_submissions(self, request):
submission.re_run()
return Response({})


#Todo : The 3 functions download many should be bundled inside a genereic with the function like "get_prediction_result" as a parameter instead of the same code 3 times.
@action(detail=False, methods=('POST',))
def download_many(self, request):
pks = request.data.get('pks')
Expand All @@ -357,50 +381,101 @@ def download_many(self, request):
# pks is already parsed as a list if JSON was sent properly
if not isinstance(pks, list):
return Response({"error": "`pks` must be a list"}, status=400)

# Get submissions
submissions = Submission.objects.filter(pk__in=pks).select_related(
"owner",
"phase",
"data"
)

if len(list(submissions)) != len(pks):
if len(list(submissions)) != len(pks):
return Response({"error": "One or more submission IDs are invalid"}, status=404)

# Nicolas Homberg : should create a function for this ?
# Check permissions
if not request.user.is_authenticated:
raise PermissionDenied("You must be logged in to download submissions")
# Allow admins
if request.user.is_superuser or request.user.is_staff:
allowed = True
else:
# Build one Q object for "owner OR organizer"
organiser_q = (
Q(phase__competition__created_by=request.user) |
Q(phase__competition__collaborators=request.user)
)
# Submissions that violate the rule
disallowed = submissions.exclude(Q(owner=request.user) | organiser_q)
allowed = not disallowed.exists()
if not allowed:
raise PermissionDenied(
"You do not have permission to download one or more of the requested submissions"
)
self.check_submission_permissions(request,submissions)

files = []

for sub in submissions:
file_path = sub.data.data_file.name.split('/')[-1]
short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}"
short_name = f"sub_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}"
# url = sub.data.data_file.url
url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file']
# url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file']
files.append({"name": short_name, "url": url})

return Response(files)

@action(detail=False, methods=('POST',))
def download_many_prediction(self, request):

pks = request.data.get('pks')
if not pks:
return Response({"error": "`pks` field is required"}, status=400)

if not isinstance(pks, list):
return Response({"error": "`pks` must be a list"}, status=400)

submissions = Submission.objects.filter(pk__in=pks).select_related(
"owner",
"phase",
"data"
)

if len(list(submissions)) != len(pks):
return Response({"error": "One or more submission IDs are invalid"}, status=404)

self.check_submission_permissions(request,submissions)

files = []
serializer = SubmissionFilesSerializer(context=self.get_serializer_context())

for sub in submissions:
if sub.status not in [ Submission.FINISHED]: #Submission.FAILED, Submission.CANCELLED
continue
file_path = sub.data.data_file.name.split('/')[-1]
detailed_name = f"pred_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}"
prediction_url = serializer.get_prediction_result(sub)
files.append({"name": detailed_name, "url": prediction_url})
return Response(files)

@action(detail=False, methods=('POST',))
def download_many_results(self, request):

pks = request.data.get('pks')
if not pks:
return Response({"error": "`pks` field is required"}, status=400)

if not isinstance(pks, list):
return Response({"error": "`pks` must be a list"}, status=400)

submissions = Submission.objects.filter(pk__in=pks).select_related(
"owner",
"phase",
"data"
)

if len(list(submissions)) != len(pks):
return Response({"error": "One or more submission IDs are invalid"}, status=404)

self.check_submission_permissions(request,submissions)

files = []
serializer = SubmissionFilesSerializer(context=self.get_serializer_context())


for sub in submissions:
if sub.status not in [ Submission.FINISHED]: #Submission.FAILED, Submission.CANCELLED
continue
file_path = sub.data.data_file.name.split('/')[-1]
complete_name = f"res_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}"
result_url = serializer.get_scoring_result(sub)
#detailed results is already in the results zip file but For very large detailed results it could be helpfull to remove it.
# detailed_result_url = serializer.get_scoring_result(sub)
files.append({"name": complete_name, "url": result_url})

return Response(files)

@action(detail=True, methods=('GET',))
def get_details(self, request, pk):
submission = super().get_object()
Expand Down
14 changes: 14 additions & 0 deletions src/static/js/ours/client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,20 @@ CODALAB.api = {
{ pks: pks } // body is JSON by convention
);
},
download_many_submissions_prediction: function (pks) {
return CODALAB.api.request(
'POST',
URLS.API + "submissions/download_many_prediction/",
{ pks: pks } // body is JSON by convention
);
},
download_many_submissions_results: function (pks) {
return CODALAB.api.request(
'POST',
URLS.API + "submissions/download_many_results/",
{ pks: pks } // body is JSON by convention
);
},

/*---------------------------------------------------------------------
Leaderboards
Expand Down
128 changes: 29 additions & 99 deletions src/static/riot/competitions/detail/submission_manager.tag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
</a>

<select class="ui dropdown" ref="submission_handling_operation">
<option value="download">Download selected submissions</option>
<option value="download">Download selected <b>submissions</b></option>
<option value="download_results">Download <b>results</b> of selected submissions</option>
<option value="download_prediction">Download <b>prediction</b> of selected submissions</option>
<!-- <option value="download">Download Score of selected submissions</option> -->
<option value="delete">Delete selected submissions</option>
<option value="rerun">Rerun selected submissions</option>
</select>
Expand Down Expand Up @@ -574,95 +577,9 @@
.modal('show')
CODALAB.events.trigger('submission_clicked')
}


// self.bulk_download = function () {
// const statusBox = document.getElementById('downloadStatus');
// const progressEl = document.getElementById('downloadProgress');
// const textEl = document.getElementById('progressText');

// statusBox.style.display = "flex";
// // statusBox.style.display = "inline";

self.bulk_download = function(files_type) {

// progressEl.style.display="flex";
// progressEl.value = 0;
// textEl.textContent = "Preparing download...";

// console.log("Files returned by server:", files);


// CODALAB.api.download_many_submissions(self.checked_submissions)
// .done( async function(files) {
// // .done( function(files) {
// console.log("Files returned by server:", files);

// const zip = new JSZip();
// const total = files.length;
// let completed = 0;
// const failed = [];

// const fetchFiles = files.map(async file => {
// try {
// const response = await fetch(file.url);

// if (!response.ok) {
// throw new Error(`HTTP ${response.status}`);
// }

// const blob = await response.blob();

// zip.file(file.name.replace(/[:/\\]/g, '_'), blob);
// } catch (err) {
// console.error(`Failed to fetch ${file.name}:`, err);
// failed.push(file.name);
// } finally {
// // Update progress regardless of success/failure
// completed++;
// const percent = Math.floor((completed / total) * 100);
// progressEl.value = percent;
// textEl.textContent = `${completed} / ${total} files (${percent}%)`;
// }
// });

// Promise.allSettled(fetchFiles).then(() => {
// // If some files failed, include them as failed.txt inside the zip
// if (failed.length > 0) {
// const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`;
// zip.file("failed.txt", failedContent);
// }


// textEl.textContent = "Generating bundle";
// progressEl.style.display = "none";

// zip.generateAsync({ type: "blob" }).then(blob => {
// const link = document.createElement("a");
// link.href = URL.createObjectURL(blob);
// link.download = "bulk_submissions.zip";
// document.body.appendChild(link);
// link.click();
// document.body.removeChild(link);

// if (failed.length > 0) {
// textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`;
// } else {
// textEl.textContent = "Download ready!";
// }

// setTimeout(() => {
// statusBox.style.display = "none";
// }, 5000);
// });
// });
// })
// .fail(function(err) {
// console.error("Error downloading submissions:", err);
// textEl.textContent = "Error downloading!";
// });
// };

self.bulk_download = function () {
const statusBox = document.getElementById('downloadStatus');
const progressEl = document.getElementById('downloadProgress');
const textEl = document.getElementById('progressText');
Expand All @@ -672,8 +589,21 @@
progressEl.value = 0;
textEl.textContent = "Preparing download...";

// Kick the API request
const req = CODALAB.api.download_many_submissions(self.checked_submissions);
let req ;
switch (files_type) {
case "submissions":
req = CODALAB.api.download_many_submissions(self.checked_submissions);
break;
case "results":
req = CODALAB.api.download_many_submissions_results(self.checked_submissions);
break;
case "predictions":
req = CODALAB.api.download_many_submissions_prediction(self.checked_submissions);
break;
default:
console.error("Default switch case in bulk function")
return;
}

// Common error handler
const handleError = (err) => {
Expand All @@ -682,18 +612,15 @@
setTimeout(() => { statusBox.style.display = "none"; }, 5000);
};

// Success handler (async because we await inside)
const handleSuccess = async (resp) => {
// Normalize response -> files array
let files = resp;
if (resp && typeof resp === 'object' && !Array.isArray(resp)) {
// common shapes: { files: [...] } or { data: [...] } or direct array
if (Array.isArray(resp.files)) files = resp.files;
else if (Array.isArray(resp.data)) files = resp.data;
else if (Array.isArray(resp.results)) files = resp.results;
else if (Array.isArray(resp)) files = resp;
else {
// if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg
if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0];
else {
console.warn("Unexpected response shape from download_many_submissions:", resp);
Expand All @@ -708,7 +635,7 @@
return;
}

console.log("Files returned by server:", files);
// console.log("Files returned by server:", files);

const zip = new JSZip();
const total = files.length;
Expand Down Expand Up @@ -767,7 +694,7 @@
const blob = await zip.generateAsync({ type: "blob" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "bulk_submissions.zip";
link.download = "bulk_" +files_type +".zip";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
Expand Down Expand Up @@ -807,15 +734,18 @@
var submission_operation = self.refs.submission_handling_operation.value
switch (submission_operation) {
case "delete":
// console.log("delete")
self.delete_selected_submissions()
break;
case "download":
// console.log("download")
self.bulk_download()
self.bulk_download("submissions")
break;
case "download_results":
self.bulk_download("results")
break;
case "download_prediction":
self.bulk_download("predictions")
break;
case "rerun":
// console.log("rerun")
self.rerun_selected_submissions()
break
default:
Expand Down