fix(io): BREAKING, wrap more out-stream usages

+ chrore(deps): depend on *contextlib2* for `ExitStack` in PY2.
+ refact(util): BREAKING API move consts out of utils.
+ style(pep8): fixe all sources.

Author: ankostis

.travis.yml

@@ -13,6 +13,7 @@ git:
 install:
   - pip install coveralls
   - pip install -I git+https://github.com/ankostis/[email protected]
+  - pip install -r requirements.txt
 script: 
   - ulimit -n 48
   - ulimit -n

gitdb/__init__.py

@@ -15,6 +15,6 @@
 
 
 # default imports
-from gitdb.base import *
-from gitdb.db import *
-from gitdb.stream import *
+from gitdb.base import *    # @IgnorePep8
+from gitdb.db import *      # @IgnorePep8
+from gitdb.stream import *  # @IgnorePep8

gitdb/db/loose.py

@@ -166,8 +166,8 @@ def info(self, sha):
 
     def stream(self, sha):
         m = self._map_loose_object(sha)
-        type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True)
-        return OStream(sha, type, size, stream)
+        typ, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True)
+        return OStream(sha, typ, size, stream)
 
     def has_object(self, sha):
         try:
@@ -247,7 +247,7 @@ def store(self, istream):
 
     def sha_iter(self):
         # find all files which look like an object, extract sha from there
-        for root, dirs, files in os.walk(self.root_path()):
+        for root, dirs, files in os.walk(self.root_path()):  # @UnusedVariable
             root_base = basename(root)
             if len(root_base) != 2:
                 continue

gitdb/pack.py

@@ -3,23 +3,30 @@
 # This module is part of GitDB and is released under
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
 """Contains PackIndexFile and PackFile implementations"""
+import array
+from binascii import crc32
+import os
+from struct import pack
+import sys
+import tempfile
 import zlib
 
+from gitdb.base import (
+    OInfo,
+    OStream,
+    OPackInfo,
+    OPackStream,
+    ODeltaStream,
+    ODeltaPackInfo,
+    ODeltaPackStream,
+)
+from gitdb.const import NULL_BYTE
 from gitdb.exc import (
     BadObject,
     AmbiguousObjectName,
     UnsupportedOperation,
     ParseError
 )
-
-from gitdb.util import (
-    mman,
-    LazyMixin,
-    unpack_from,
-    bin_to_hex,
-    byte_ord,
-)
-
 from gitdb.fun import (
     create_pack_object_header,
     pack_object_header_info,
@@ -33,46 +40,34 @@
     REF_DELTA,
     msb_size
 )
-
-try:
-    from gitdb_speedups._perf import PackIndexFile_sha_to_index
-except ImportError:
-    pass
-# END try c module
-
-from gitdb.base import (
-    OInfo,
-    OStream,
-    OPackInfo,
-    OPackStream,
-    ODeltaStream,
-    ODeltaPackInfo,
-    ODeltaPackStream,
-)
-
 from gitdb.stream import (
     DecompressMemMapReader,
     DeltaApplyReader,
     Sha1Writer,
     NullStream,
     FlexibleSha1Writer
 )
-
-from struct import pack
-from binascii import crc32
-
-from gitdb.const import NULL_BYTE
+from gitdb.util import (
+    mman,
+    LazyMixin,
+    bin_to_hex,
+    byte_ord,
+)
 from gitdb.utils.compat import (
     izip,
     buffer,
     xrange,
-    to_bytes
+    to_bytes,
+    unpack_from,
 )
 
-import tempfile
-import array
-import os
-import sys
+
+try:
+    from gitdb_speedups._perf import PackIndexFile_sha_to_index
+except ImportError:
+    pass
+# END try c module
+
 
 __all__ = ('PackIndexFile', 'PackFile', 'PackEntity')
 
@@ -290,8 +285,9 @@ def _make_cursor(self):
 
         # We will assume that the index will always fully fit into memory !
         if mman.window_size() > 0 and cursor.file_size() > mman.window_size():
-            raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (
-                self._indexpath, cursor.file_size(), mman.window_size()))
+            raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). "
+                                 "This is a limitation of the hardware." % (
+                                     self._indexpath, cursor.file_size(), mman.window_size()))
 
         return cursor
 
@@ -528,7 +524,7 @@ class PackFile(LazyMixin):
                  '_size',
                  '_version',
                  '_entered',
-    )
+                 )
     pack_signature = 0x5041434b     # 'PACK'
     pack_version_default = 2
 
@@ -603,7 +599,11 @@ def data(self):
         """
         :return: read-only data of this pack. It provides random access and usually
             is a memory map.
-        :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size"""
+
+        .. note::
+            This method is unsafe as it returns a window into a file which might be larger
+            than than the actual window size
+        """
         # can use map as we are starting at offset 0. Otherwise we would have to use buffer()
         return self._cursor.use_region().map()
 
@@ -761,7 +761,8 @@ def _object(self, sha, as_stream, index=-1):
             sha = self._index.sha(index)
         # END assure sha is present ( in output )
         offset = self._index.offset(index)
-        type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer())
+        type_id, uncomp_size, _ = pack_object_header_info(
+            self._pack._cursor.use_region(offset).buffer())
         if as_stream:
             if type_id not in delta_types:
                 packstream = self._pack.stream(offset)
@@ -784,7 +785,7 @@ def _object(self, sha, as_stream, index=-1):
             # the actual target size, as opposed to the size of the delta data
             streams = self.collect_streams_at_offset(offset)
             buf = streams[0].read(512)
-            offset, src_size = msb_size(buf)
+            offset, src_size = msb_size(buf)  # @UnusedVariable
             offset, target_size = msb_size(buf, offset)
 
             # collect the streams to obtain the actual object type
@@ -1019,8 +1020,9 @@ def write_pack(cls, object_iter, pack_write, index_write=None,
             # END for each object
 
             if actual_count != object_count:
-                raise ValueError(
-                    "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count))
+                raise ValueError("Expected to write %i objects into pack, "
+                                 "but received only %i from iterators" %
+                                 (object_count, actual_count))
             # END count assertion
 
             # write footer
@@ -1049,7 +1051,8 @@ def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.
         pack_write = lambda d: os.write(pack_fd, d)
         index_write = lambda d: os.write(index_fd, d)
 
-        pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression)
+        pack_binsha, _ = cls.write_pack(
+            object_iter, pack_write, index_write, object_count, zlib_compression)
         os.close(pack_fd)
         os.close(index_fd)
 

gitdb/stream.py

@@ -4,20 +4,19 @@
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
 
 from io import BytesIO
-
 import mmap
 import os
 import sys
 import zlib
 
+from gitdb.const import NULL_BYTE, BYTE_SPACE
 from gitdb.fun import (
     msb_size,
     stream_copy,
     apply_delta_data,
     connect_deltas,
     delta_types
 )
-
 from gitdb.util import (
     allocate_memory,
     LazyMixin,
@@ -27,10 +26,9 @@
     suppress,
     is_darwin,
 )
-
-from gitdb.const import NULL_BYTE, BYTE_SPACE
 from gitdb.utils.compat import buffer
 
+
 has_perf_mod = False
 PY26 = sys.version_info[:2] < (2, 7)
 try:
@@ -157,7 +155,8 @@ def data(self):
     def close(self):
         """Close our underlying stream of compressed bytes if this was allowed during initialization
         :return: True if we closed the underlying stream
-        :note: can be called safely
+
+        .. note::  can be called safely
         """
         if self._close:
             if hasattr(self._m, 'close'):
@@ -196,7 +195,7 @@ def compressed_bytes_read(self):
             # but keep the window at its current position
             self._br = 0
             if hasattr(self._zip, 'status'):
-                while self._zip.status == zlib.Z_OK:
+                while self._zip.status == zlib.Z_OK:  # @UndefinedVariable
                     self.read(mmap.PAGESIZE)
                 # END scrub-loop custom zlib
             else:
@@ -762,7 +761,7 @@ class FDStream(object):
     __slots__ = ('_fd',
                  '_pos',
                  '_entered',
-    )
+                 )
 
     def __init__(self, fd):
         self._fd = fd
@@ -794,9 +793,9 @@ def read(self, count=0):
             count = os.path.getsize(self._filepath)
         # END handle read everything
 
-        bytes = os.read(self._fd, count)
-        self._pos += len(bytes)
-        return bytes
+        bs = os.read(self._fd, count)
+        self._pos += len(bs)
+        return bs
 
     def fileno(self):
         return self._fd

gitdb/test/db/lib.py

@@ -3,32 +3,28 @@
 # This module is part of GitDB and is released under
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
 """Base classes for object db testing"""
-from gitdb.test.lib import (
-    with_rw_directory,
-    with_packs_rw,
-    fixture_path,
-    TestBase
-)
-
-from gitdb.stream import (
-    Sha1Writer,
-    ZippedStoreShaWriter
-)
+from io import BytesIO
+from struct import pack
 
 from gitdb.base import (
     IStream,
     OStream,
     OInfo
 )
-
 from gitdb.exc import BadObject
+from gitdb.stream import (
+    Sha1Writer,
+    ZippedStoreShaWriter
+)
+from gitdb.test.lib import (
+    with_rw_directory,  # @UnusedImport
+    with_packs_rw,      # @UnusedImport
+    fixture_path,       # @UnusedImport
+    TestBase
+)
 from gitdb.typ import str_blob_type
 from gitdb.utils.compat import xrange
 
-from io import BytesIO
-
-from struct import pack
-
 
 __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path')
 
@@ -98,10 +94,10 @@ def _assert_object_writing(self, db):
                     assert str_blob_type == info.type
                     assert info.size == len(data)
 
-                    ostream = db.stream(sha)
-                    assert ostream.read() == data
-                    assert ostream.type == str_blob_type
-                    assert ostream.size == len(data)
+                    with db.stream(sha) as ostream:
+                        assert ostream.read() == data
+                        assert ostream.type == str_blob_type
+                        assert ostream.size == len(data)
                 else:
                     self.failUnlessRaises(BadObject, db.info, sha)
                     self.failUnlessRaises(BadObject, db.stream, sha)
@@ -120,10 +116,9 @@ def _assert_object_writing(self, db):
                     db.set_ostream(ZippedStoreShaWriter())
                     db.store(istream)
                     assert istream.binsha == prev_sha
-                    new_ostream = db.ostream()
-
-                    # note: only works as long our store write uses the same compression
-                    # level, which is zip_best
-                    assert ostream.getvalue() == new_ostream.getvalue()
+                    with db.ostream() as new_ostream:
+                        # note: only works as long our store write uses the same compression
+                        # level, which is zip_best
+                        assert ostream.getvalue() == new_ostream.getvalue()
             # END for each data set
         # END for each dry_run mode

gitdb/test/db/test_git.py

@@ -3,13 +3,14 @@
 # This module is part of GitDB and is released under
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
 import os
+
+from gitdb.base import OStream, OInfo
+from gitdb.db import GitDB
+from gitdb.exc import BadObject
 from gitdb.test.db.lib import (
     TestDBBase,
     with_rw_directory
 )
-from gitdb.exc import BadObject
-from gitdb.db import GitDB
-from gitdb.base import OStream, OInfo
 from gitdb.util import bin_to_hex
 
 
@@ -25,7 +26,7 @@ def test_reading(self):
         gitdb_sha = next(gdb.sha_iter())
         assert isinstance(gdb.info(gitdb_sha), OInfo)
         with gdb.stream(gitdb_sha) as stream:
-            assert isinstance(gdb.stream(gitdb_sha), OStream)
+            assert isinstance(stream, OStream)
         ni = 50
         assert gdb.size() >= ni
         sha_list = list(gdb.sha_iter())