Add warn size and make max size configurable

polyaxon · Apr 11, 2019 · 3878221 · 3878221
1 parent 9fa910a
commit 3878221
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 4 deletions.
diff --git a/polyaxon_client/settings.py b/polyaxon_client/settings.py
@@ -135,3 +135,9 @@
 VALIDATION_UNKNOWN_BEHAVIOUR = config.get_string('POLYAXON_VALIDATION_UNKNOWN_BEHAVIOUR',
                                                  is_optional=True,
                                                  default=RAISE)
+WARN_UPLOAD_SIZE = config.get_int('POLYAXON_WARN_UPLOAD_SIZE',
+                                  is_optional=True,
+                                  default=1024 * 1024 * 10)
+MAX_UPLOAD_SIZE = config.get_int('POLYAXON_MAX_UPLOAD_SIZE',
+                                 is_optional=True,
+                                 default=1024 * 1024 * 150)
diff --git a/polyaxon_client/transport/http_transport.py b/polyaxon_client/transport/http_transport.py
@@ -18,8 +18,6 @@
 
 class HttpTransportMixin(object):
     """HTTP operations transport."""
-    MAX_UPLOAD_SIZE = 1024 * 1024 * 150
-
     @property
     def session(self):
         if not hasattr(self, '_session'):
@@ -119,13 +117,23 @@ def upload(self,
                headers=None,
                session=None):
 
-        if files_size > self.MAX_UPLOAD_SIZE:
+        if files_size > settings.WARN_UPLOAD_SIZE:
+            logger.warning(
+                "You are uploading %s, there's a hard limit of %s.\n"
+                "If you have data files in the current directory, "
+                "please make sure to add them to .polyaxonignore or "
+                "add them directly to your data volume, or upload them "
+                "separately using `polyaxon data` command and remove them from here.\n",
+                self.format_sizeof(settings.WARN_UPLOAD_SIZE),
+                self.format_sizeof(settings.MAX_UPLOAD_SIZE))
+
+        if files_size > settings.MAX_UPLOAD_SIZE:
             raise PolyaxonShouldExitError(
                 "Files too large to sync, please keep it under {}.\n"
                 "If you have data files in the current directory, "
                 "please add them directly to your data volume, or upload them "
                 "separately using `polyaxon data` command and remove them from here.\n".format(
-                    self.format_sizeof(self.MAX_UPLOAD_SIZE)))
+                    self.format_sizeof(settings.MAX_UPLOAD_SIZE)))
 
         files = to_list(files)
         if json_data: