Made sure xrange is used instead of range in python 2

range in py2 will return a list, which can mean a lot of time and memory
is spent on generating it even though it's just used for iteration.

Simplified implementation of MAXSIZE

Author: Byron

gitdb/db/pack.py

@@ -18,6 +18,7 @@
 )
 
 from gitdb.pack import PackEntity
+from gitdb.utils.compat import xrange
 
 from functools import reduce
 
@@ -106,7 +107,7 @@ def sha_iter(self):
         for entity in self.entities():
             index = entity.index()
             sha_by_index = index.sha
-            for index in range(index.size()):
+            for index in xrange(index.size()):
                 yield sha_by_index(index)
             # END for each index
         # END for each entity

gitdb/ext/async

@@ -64,7 +64,7 @@
 from binascii import crc32
 
 from gitdb.const import NULL_BYTE
-from gitdb.utils.compat import izip, buffer
+from gitdb.utils.compat import izip, buffer, xrange
 
 import tempfile
 import array
@@ -208,7 +208,7 @@ def write(self, pack_sha, write):
         for t in self._objs:
             tmplist[ord(t[0][0])] += 1
         #END prepare fanout
-        for i in range(255):
+        for i in xrange(255):
             v = tmplist[i]
             sha_write(pack('>L', v))
             tmplist[i+1] += v
@@ -374,7 +374,7 @@ def _read_fanout(self, byte_offset):
         d = self._cursor.map()
         out = list()
         append = out.append
-        for i in range(256):
+        for i in xrange(256):
             append(unpack_from('>L', d, byte_offset + i*4)[0])
         # END for each entry
         return out
@@ -415,7 +415,7 @@ def offsets(self):
                 a.byteswap()
             return a
         else:
-            return tuple(self.offset(index) for index in range(self.size()))
+            return tuple(self.offset(index) for index in xrange(self.size()))
         # END handle version
 
     def sha_to_index(self, sha):
@@ -703,7 +703,7 @@ def _iter_objects(self, as_stream):
         """Iterate over all objects in our index and yield their OInfo or OStream instences"""
         _sha = self._index.sha
         _object = self._object
-        for index in range(self._index.size()):
+        for index in xrange(self._index.size()):
             yield _object(_sha(index), as_stream, index)
         # END for each index
 

gitdb/ext/smmap

@@ -22,6 +22,7 @@
 from gitdb.exc import BadObject
 from gitdb.typ import str_blob_type
 from gitdb.utils.encoding import force_bytes
+from gitdb.utils.compat import xrange
 
 from async import IteratorReader
 
@@ -43,7 +44,7 @@ def _assert_object_writing_simple(self, db):
         # write a bunch of objects and query their streams and info
         null_objs = db.size()
         ni = 250
-        for i in range(ni):
+        for i in xrange(ni):
             data = pack(">L", i)
             istream = IStream(str_blob_type, len(data), BytesIO(data))
             new_istream = db.store(istream)
@@ -131,7 +132,7 @@ def _assert_object_writing_async(self, db):
         """Test generic object writing using asynchronous access"""
         ni = 5000
         def istream_generator(offset=0, ni=ni):
-            for data_src in range(ni):
+            for data_src in xrange(ni):
                 data = bytes(data_src + offset)
                 yield IStream(str_blob_type, len(data), BytesIO(data))
             # END for each item

gitdb/pack.py

@@ -12,6 +12,7 @@
 )
 
 from gitdb.util import zlib
+from gitdb.utils.compat import xrange
 
 import sys
 import random
@@ -110,7 +111,7 @@ def make_bytes(size_in_bytes, randomize=False):
     """:return: string with given size in bytes
     :param randomize: try to produce a very random stream"""
     actual_size = size_in_bytes // 4
-    producer = range(actual_size)
+    producer = xrange(actual_size)
     if randomize:
         producer = list(producer)
         random.shuffle(producer)

gitdb/test/db/lib.py

@@ -26,6 +26,7 @@
 from gitdb.fun import delta_types
 from gitdb.exc import UnsupportedOperation
 from gitdb.util import to_bin_sha
+from gitdb.utils.compat import xrange
 from itertools import chain
 
 try:
@@ -64,7 +65,7 @@ def _assert_index_file(self, index, version, size):
         assert len(index.offsets()) == size
 
         # get all data of all objects
-        for oidx in range(index.size()):
+        for oidx in xrange(index.size()):
             sha = index.sha(oidx)
             assert oidx == index.sha_to_index(sha)
 

gitdb/test/lib.py

@@ -122,13 +122,10 @@ def byte_ord(b):
 
 #{ Routines
 
-def make_sha(source=None):
+def make_sha(source=''.encode("ascii")):
     """A python2.4 workaround for the sha/hashlib module fiasco
 
     **Note** From the dulwich project """
-    if source is None:
-        source = "".encode("ascii")
-
     try:
         return hashlib.sha1(source)
     except NameError:

gitdb/test/test_pack.py

@@ -4,8 +4,10 @@
 
 try:
     from itertools import izip
+    xrange = xrange
 except ImportError:
     izip = zip
+    xrange = range
 
 try:
     # Python 2
@@ -21,19 +23,7 @@ def buffer(obj, offset, size=None):
 
     memoryview = memoryview
 
-if PY3:
+try:
+    MAXSIZE = sys.maxint
+except AttributeError:
     MAXSIZE = sys.maxsize
-else:
-    # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
-    class X(object):
-        def __len__(self):
-            return 1 << 31
-    try:
-        len(X())
-    except OverflowError:
-        # 32-bit
-        MAXSIZE = int((1 << 31) - 1)
-    else:
-        # 64-bit
-        MAXSIZE = int((1 << 63) - 1)
-    del X