diff --git a/polyaxon_client/settings.py b/polyaxon_client/settings.py index a7f4b136d1..d818b4ec3d 100644 --- a/polyaxon_client/settings.py +++ b/polyaxon_client/settings.py @@ -135,3 +135,9 @@ VALIDATION_UNKNOWN_BEHAVIOUR = config.get_string('POLYAXON_VALIDATION_UNKNOWN_BEHAVIOUR', is_optional=True, default=RAISE) +WARN_UPLOAD_SIZE = config.get_int('POLYAXON_WARN_UPLOAD_SIZE', + is_optional=True, + default=1024 * 1024 * 10) +MAX_UPLOAD_SIZE = config.get_int('POLYAXON_MAX_UPLOAD_SIZE', + is_optional=True, + default=1024 * 1024 * 150) diff --git a/polyaxon_client/transport/http_transport.py b/polyaxon_client/transport/http_transport.py index 29b1c54588..e4ad6faed4 100644 --- a/polyaxon_client/transport/http_transport.py +++ b/polyaxon_client/transport/http_transport.py @@ -18,8 +18,6 @@ class HttpTransportMixin(object): """HTTP operations transport.""" - MAX_UPLOAD_SIZE = 1024 * 1024 * 150 - @property def session(self): if not hasattr(self, '_session'): @@ -119,13 +117,23 @@ def upload(self, headers=None, session=None): - if files_size > self.MAX_UPLOAD_SIZE: + if files_size > settings.WARN_UPLOAD_SIZE: + logger.warning( + "You are uploading %s, there's a hard limit of %s.\n" + "If you have data files in the current directory, " + "please make sure to add them to .polyaxonignore or " + "add them directly to your data volume, or upload them " + "separately using `polyaxon data` command and remove them from here.\n", + self.format_sizeof(settings.WARN_UPLOAD_SIZE), + self.format_sizeof(settings.MAX_UPLOAD_SIZE)) + + if files_size > settings.MAX_UPLOAD_SIZE: raise PolyaxonShouldExitError( "Files too large to sync, please keep it under {}.\n" "If you have data files in the current directory, " "please add them directly to your data volume, or upload them " "separately using `polyaxon data` command and remove them from here.\n".format( - self.format_sizeof(self.MAX_UPLOAD_SIZE))) + self.format_sizeof(settings.MAX_UPLOAD_SIZE))) files = to_list(files) if json_data: