Update library and make setup.py work out of the box

lz4/LICENSE

@@ -0,0 +1,24 @@
+LZ4 Library
+Copyright (c) 2011-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

lz4/Makefile

@@ -1,14 +1,117 @@
-UNAME := $(shell uname)
+# ################################################################
+# LZ4 library - Makefile
+# Copyright (C) Yann Collet 2011-2015
+# All rights reserved.
+# 
+# BSD license
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice, this
+#   list of conditions and the following disclaimer in the documentation and/or
+#   other materials provided with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# 
+# You can contact the author at :
+#  - LZ4 source repository : http://code.google.com/p/lz4/
+#  - LZ4 source mirror : https://github.com/Cyan4973/lz4
+#  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
+# ################################################################
 
-LIBTYPE=so
+# Version numbers
+VERSION ?= 126
+LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
+LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
+LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
+LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 
-all: clean build
+DESTDIR?=
+PREFIX ?= /usr
+CFLAGS ?= -O3
+CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic
+
+LIBDIR?= $(PREFIX)/lib
+INCLUDEDIR=$(PREFIX)/include
+
+
+# OS X linker doesn't support -soname, and use different extension
+# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
+ifeq ($(shell uname), Darwin)
+	SHARED_EXT = dylib
+	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
+	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
+	SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+else
+	SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
+	SHARED_EXT = so
+	SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
+	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
+endif
+
+default: liblz4
+
+all: liblz4
+
+liblz4: lz4.c lz4hc.c lz4frame.c xxhash.c
+	@echo compiling static library
+	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
+	@$(AR) rcs liblz4.a lz4.o lz4hc.o lz4frame.o xxhash.o
+	@echo compiling dynamic library $(LIBVER)
+	@$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
+	@echo creating versioned links
+	@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
+	@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
 
 clean:
-	rm -f *.so *.dylib *.gch
+	@rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* liblz4.pc
+	@echo Cleaning library completed
+
+
+#------------------------------------------------------------------------
+#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
+
+liblz4.pc: liblz4.pc.in Makefile
+	@echo creating pkgconfig
+	@sed -e 's|@PREFIX@|$(PREFIX)|' \
+            -e 's|@LIBDIR@|$(LIBDIR)|' \
+            -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
+            -e 's|@VERSION@|$(VERSION)|' \
+             $< >$@
+
+install: liblz4 liblz4.pc
+	@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
+	@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
+	@cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
+	@cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
+	@cp -a liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
+	@install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	@install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	@install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	@echo lz4 static and shared library installed
 
-build:
-	gcc -O3 -shared -fPIC -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration -o liblz4.$(LIBTYPE) lz4.h lz4hc.h lz4.c lz4hc.c lz4_offset_hack.c
+uninstall:
+	@rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	@rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
+	@rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
+	@[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
+	@[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	@echo lz4 libraries successfully uninstalled
 
-debug:
-	gcc -O3 -shared -fPIC -g -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration -o liblz4.$(LIBTYPE) lz4.h lz4hc.h lz4.c lz4hc.c lz4_offset_hack.c
+endif

lz4/__init__.py

@@ -1,87 +1,95 @@
-import cffi, platform, os
+from cffi import FFI
+from os.path import dirname
+from os.path import join as pthjoin
+from os.path import relpath
 
-libtype = "so"
+ffi = FFI()
 
-ffi = cffi.FFI()
+ffi.cdef("int LZ4_compress   (const char*, char*, int);")
+ffi.cdef("int LZ4_decompress_safe (const char*, char*, int, int);")
+ffi.cdef("int LZ4_compressHC (const char*, char*, int);")
+ffi.cdef("int LZ4_compressBound (int);")
 
-ffi.cdef("int LZ4_compress   (const char* source, char* dest, int isize);")
-ffi.cdef("int LZ4_uncompress (const char* source, char* dest, int osize);")
-ffi.cdef("int LZ4_compressHC (const char* source, char* dest, int isize);")
-ffi.cdef("int LZ4_uncompress_offset (const char* source, char* dest, int osize, int offset);")
+path = dirname(__file__)
+lz4_files = ['lz4.c', 'lz4hc.c', 'xxhash.c']
+sources = [relpath(pthjoin(path, p)) for p in lz4_files]
+_lz4 = ffi.verify(sources=sources)
 
-candidate_names = [os.path.join(os.path.dirname(os.path.abspath(__file__)), "%s.%s" % (name, libtype)) for name in
-    ['liblz4', 'liblz4.pypy-20']
-]
-existing_names = [name for name in candidate_names if os.path.exists(name)]
-assert len(existing_names) >= 1, "Can't find compiled library."
 
-liblz4 = ffi.dlopen(existing_names[0])
+def _store_le32(c, x):
+    c[0] = chr(x & 0xff)
+    c[1] = chr((x >> 8) & 0xff)
+    c[2] = chr((x >> 16) & 0xff)
+    c[3] = chr((x >> 24) & 0xff)
 
-def LZ4_compressBound(isize):
-    return isize + (isize / 255) + 16
 
-def store_le32(c, x):
-    c[0] = chr(x & 0xff);
-    c[1] = chr((x >> 8) & 0xff);
-    c[2] = chr((x >> 16) & 0xff);
-    c[3] = chr((x >> 24) & 0xff);
-
-def load_le32(c):
-    return ord(c[0]) | (ord(c[1]) << 8) | (ord(c[2]) << 16) | (ord(c[3]) << 24)
+def _load_le32(c):
+    return ord(c[0])      | \
+        (ord(c[1]) << 8)  | \
+        (ord(c[2]) << 16) | \
+        (ord(c[3]) << 24)
 
 HDR_SIZE = 4
 INT_MAX = 2 ** 31
 
+
 def _compress_with(compressor, source):
     source_size = len(source)
-    
-    dest_size = HDR_SIZE + LZ4_compressBound(source_size)
+
+    dest_size = HDR_SIZE + _lz4.LZ4_compressBound(source_size)
     dest = ffi.new("char[]", dest_size)
     actual_size = 0
 
-    store_le32(dest, source_size)
+    _store_le32(dest, source_size)
     if source_size > 0:
         osize = compressor(source, dest + HDR_SIZE, source_size)
         actual_size = HDR_SIZE + osize
-        # ideally we would resize the buffer now, but I don't know how to do that with cffi
-
+
     return (dest, actual_size)
 
+
 def _compress(source):
-    return _compress_with(liblz4.LZ4_compress, source)
+    return _compress_with(_lz4.LZ4_compress, source)
 
-def _compressHC(source):
-    return _compress_with(liblz4.LZ4_compressHC, source)
 
-def compress(source):
-    data, size = _compress(source)
-    return ffi.buffer(data, size)[:]
+def _compressHC(source):
+    return _compress_with(_lz4.LZ4_compressHC, source)
 
-def compressHC(source):
-    data, size = _compressHC(source)
-    return ffi.buffer(data, size)[:]
 
 def _uncompress(source):
     source_size = len(source)
 
     if source_size < HDR_SIZE:
         raise ValueError("input too short")
-
-    dest_size = load_le32(source)
-    if dest_size > INT_MAX:
-        raise ValueError("invalid size in header: 0x%x" % dest_size)
-
-    if dest_size > 0:
-        dest = ffi.new("char[]", dest_size)
-        osize = liblz4.LZ4_uncompress_offset(source, dest, dest_size, HDR_SIZE)
 
-        if osize < 0:
-            raise ValueError("corrupt input at byte %d" % -osize)
-        elif osize < source_size - HDR_SIZE:
-            raise ValueError("decompression incomplete")
+    dest_size = _load_le32(source)
+    if dest_size > INT_MAX or dest_size < 0:
+        raise ValueError("invalid size in header: %d - 0x%x"
+                         % (dest_size, dest_size))
 
+    dest = ffi.new("char[]", dest_size)
+    osize = _lz4.LZ4_decompress_safe(source[4:], dest,
+                                     source_size - HDR_SIZE, dest_size)
+    if osize < 0:
+        raise ValueError("corrupt input at byte %d" % -osize)
     return (dest, dest_size)
 
+
+def compress(source):
+    data, size = _compress(source)
+    return ffi.buffer(data, size)[:]
+
+
+def compressHC(source):
+    data, size = _compressHC(source)
+    return ffi.buffer(data, size)[:]
+
+
 def uncompress(source):
     data, size = _uncompress(source)
-    return ffi.buffer(data, size)[:]
+    return ffi.buffer(data, size)[:]
+
+loads = uncompress
+dumps = compress
+
+__all__ = ['compress', 'compressHC', 'uncompress', 'loads', 'dumps']