fix: use on-the-fly compression only for file objects (#177)

When we introduced the new `GzipStream` utility class
which handles the on-the-fly compression of the file
like object. We also started using it for in-memory
objects, like bytes, but it has introduced a minor
incompatibility with the existing SDKs.
This commit introduces a fix for that, by using the
new utility class only for file-like objects and handling
the other types as before (compressing in memory).

Signed-off-by: Norbert Biczo <[email protected]>

Author: pyrooka

ibm_cloud_sdk_core/base_service.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import gzip
+import io
 import json as json_import
 import logging
 import platform
@@ -421,7 +423,13 @@ def prepare_request(
         if self.get_enable_gzip_compression() and 'content-encoding' not in headers and request['data'] is not None:
             headers['content-encoding'] = 'gzip'
             request['headers'] = headers
-            request['data'] = GzipStream(request['data'])
+            # If the provided data is a file-like object, we create `GzipStream` which will handle
+            # the compression on-the-fly when the requests package starts reading its content.
+            # This helps avoid OOM errors when the opened file is too big.
+            # In any other cases, we use the in memory compression directly from
+            # the `gzip` package for backward compatibility.
+            raw_data = request['data']
+            request['data'] = GzipStream(raw_data) if isinstance(raw_data, io.IOBase) else gzip.compress(raw_data)
 
         # Next, we need to process the 'files' argument to try to fill in
         # any missing filenames where possible.

test/test_base_service.py

@@ -607,13 +607,13 @@ def test_gzip_compression():
     service.set_enable_gzip_compression(True)
     assert service.get_enable_gzip_compression()
     prepped = service.prepare_request('GET', url='', data=json.dumps({"foo": "bar"}))
-    assert prepped['data'].read() == gzip.compress(b'{"foo": "bar"}')
+    assert prepped['data'] == gzip.compress(b'{"foo": "bar"}')
     assert prepped['headers'].get('content-encoding') == 'gzip'
 
     # Should return compressed data when gzip is on for non-json data
     assert service.get_enable_gzip_compression()
     prepped = service.prepare_request('GET', url='', data=b'rawdata')
-    assert prepped['data'].read() == gzip.compress(b'rawdata')
+    assert prepped['data'] == gzip.compress(b'rawdata')
     assert prepped['headers'].get('content-encoding') == 'gzip'
 
     # Should return compressed data when gzip is on for gzip file data
@@ -624,7 +624,7 @@ def test_gzip_compression():
         with gzip.GzipFile(mode='rb', fileobj=t_f) as gz_f:
             gzip_data = gz_f.read()
         prepped = service.prepare_request('GET', url='', data=gzip_data)
-        assert prepped['data'].read() == gzip.compress(t_f.read())
+        assert prepped['data'] == gzip.compress(t_f.read())
         assert prepped['headers'].get('content-encoding') == 'gzip'
 
     # Should return compressed json data when gzip is on for gzip file json data
@@ -635,7 +635,7 @@ def test_gzip_compression():
         with gzip.GzipFile(mode='rb', fileobj=t_f) as gz_f:
             gzip_data = gz_f.read()
         prepped = service.prepare_request('GET', url='', data=gzip_data)
-        assert prepped['data'].read() == gzip.compress(t_f.read())
+        assert prepped['data'] == gzip.compress(t_f.read())
         assert prepped['headers'].get('content-encoding') == 'gzip'
 
     # Should return uncompressed data when content-encoding is set
@@ -715,7 +715,7 @@ def test_gzip_compression_external():
     assert service.service_url == 'https://mockurl'
     assert service.get_enable_gzip_compression() is True
     prepped = service.prepare_request('GET', url='', data=json.dumps({"foo": "bar"}))
-    assert prepped['data'].read() == gzip.compress(b'{"foo": "bar"}')
+    assert prepped['data'] == gzip.compress(b'{"foo": "bar"}')
     assert prepped['headers'].get('content-encoding') == 'gzip'