Add hypothesis-powered tests

.github/workflows/linux.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Run test (3.8)
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
           pytest -v test
 
@@ -54,7 +54,7 @@ jobs:
 
       - name: Run test (3.9)
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
           pytest -v test
 
@@ -65,7 +65,7 @@ jobs:
 
       - name: Run test (3.7)
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
           pytest -v test
 
@@ -76,7 +76,7 @@ jobs:
 
       - name: Run test (3.6)
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
           pytest -v test
 

.github/workflows/mac.yml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Run test
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: -f dist/ --no-index
           pytest -v test
 
@@ -47,7 +47,7 @@ jobs:
 
       - name: Run test
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: -f dist/ --no-index
           pytest -v test
 
@@ -64,7 +64,7 @@ jobs:
 
       - name: Run test
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: -f dist/ --no-index
           pytest -v test
 
@@ -81,7 +81,7 @@ jobs:
 
       - name: Run test
         run: |
-          pip install pytest
+          pip install pytest hypothesis
           pip install -v msgpack --only-binary :all: -f dist/ --no-index
           pytest -v test
 

.gitignore

@@ -13,3 +13,4 @@ msgpack/*.cpp
 /tags
 /docs/_build
 .cache
+.hypothesis

.travis.yml

@@ -17,7 +17,7 @@ python:
 _pure: &pure
   install:
     - pip install -U pip
-    - pip install -U pytest pytest-cov codecov
+    - pip install -U pytest pytest-cov codecov hypothesis
     - pip install .
   script:
     - pytest --cov=msgpack -v test
@@ -33,6 +33,7 @@ matrix:
         - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
       install:
         - pip install -U pip
+        - pip install -U hypothesis
         - pip install -r requirements.txt
         - make cython
         - docker pull $DOCKER_IMAGE
@@ -48,6 +49,7 @@ matrix:
         - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64
       install:
         - pip install -U pip
+        - pip install -U hypothesis
         - pip install -r requirements.txt
         - make cython
         - docker pull $DOCKER_IMAGE
@@ -70,7 +72,7 @@ matrix:
 
 install:
   - pip install -U pip
-  - pip install -U pytest pytest-cov codecov
+  - pip install -U pytest pytest-cov codecov hypothesis
   - pip install -r requirements.txt  # Cython
   - make cython
   - pip install -e .

ci/runtests.bat

@@ -1,4 +1,4 @@
-%PYTHON%\python.exe -m pip install -U pip wheel pytest
+%PYTHON%\python.exe -m pip install -U pip wheel pytest hypothesis
 %PYTHON%\python.exe setup.py build_ext -i
 %PYTHON%\python.exe setup.py install
 %PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"

ci/runtests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 ${PYTHON} -VV
-${PYTHON} -m pip install setuptools wheel pytest
+${PYTHON} -m pip install setuptools wheel pytest hypothesis
 ${PYTHON} setup.py build_ext -if
 ${PYTHON} -c "from msgpack import _cmsgpack"
 ${PYTHON} setup.py bdist_wheel

docker/runtests.sh

@@ -8,7 +8,7 @@ for V in "${PYTHON_VERSIONS[@]}"; do
     PYBIN=/opt/python/$V/bin
     $PYBIN/python setup.py install
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
-    $PYBIN/pip install pytest
+    $PYBIN/pip install pytest hypothesis
     pushd test          # prevent importing msgpack package in current directory.
     $PYBIN/python -c 'import sys; print(hex(sys.maxsize))'
     $PYBIN/python -c 'from msgpack import _cmsgpack'  # Ensure extension is available

test/test_property_based.py

@@ -0,0 +1,84 @@
+import pytest
+from hypothesis import given, assume, strategies as st
+
+import msgpack
+
+try:
+    from msgpack import _cmsgpack
+except ImportError:
+    _cmsgpack = None
+from msgpack import fallback
+
+HYPOTHESIS_MAX = 3
+
+# https://github.com/msgpack/msgpack/blob/master/spec.md#type-system
+# TODO: test timestamps
+# TODO: test the extension type
+simple_types = (
+    st.integers(min_value=-(2 ** 63), max_value=2 ** 64 - 1)
+    | st.none()
+    | st.booleans()
+    | st.floats()
+    # TODO: The msgpack speck says that string objects may contain invalid byte sequence
+    | st.text(max_size=HYPOTHESIS_MAX)
+    | st.binary(max_size=HYPOTHESIS_MAX)
+)
+
+
+def composite_types(any_type):
+    return st.lists(any_type, max_size=HYPOTHESIS_MAX) | st.dictionaries(
+        simple_types, any_type, max_size=HYPOTHESIS_MAX
+    )
+
+
+any_type = st.recursive(simple_types, composite_types)
+
+
+@pytest.mark.skipif(_cmsgpack is None, reason="C extension is not available")
+@given(any_type)
+def test_extension_and_fallback_pack_identically(obj):
+    extension_packer = _cmsgpack.Packer()
+    fallback_packer = fallback.Packer()
+
+    assert extension_packer.pack(obj) == fallback_packer.pack(obj)
+
+
+# TODO: also test with strict_map_key=True
+@pytest.mark.parametrize("impl", [fallback, _cmsgpack])
+@given(obj=any_type)
+def test_roudtrip(obj, impl):
+    if impl is None:
+        pytest.skip("C extension is not available")
+    packer = impl.Packer()
+    buf = packer.pack(obj)
+    got = impl.unpackb(buf, strict_map_key=False)
+    # using obj == got fails because NaN != NaN
+    assert repr(obj) == repr(got)
+
+
+# TODO: also test with strict_map_key=True
+@pytest.mark.skipif(_cmsgpack is None, reason="C extension is not available")
+@given(st.binary(max_size=HYPOTHESIS_MAX))
+def test_extension_and_fallback_unpack_identically(buf):
+    try:
+        from_extension = _cmsgpack.unpackb(buf)
+    except (msgpack.ExtraData, ValueError) as e:
+        # Ignore the exception message. This avoids:
+        #
+        # Falsifying example: buf=b'\x00\x00'
+        # Error: ExtraData(0, bytearray(b'\x00')) != ExtraData(0, b'\x00')
+        #
+        # Falsifying example: buf=b'\xa2'
+        # Error: ValueError('2 exceeds max_str_len(1)') != ValueError('Unpack failed: incomplete input')
+        # See https://github.com/msgpack/msgpack-python/pull/464
+        from_extension = type(e)
+    except Exception as e:
+        from_extension = e
+    try:
+        from_fallback = fallback.unpackb(buf)
+    except (msgpack.ExtraData, ValueError) as e:
+        from_fallback = type(e)
+    except Exception as e:
+        from_fallback = e
+
+    assert repr(from_extension) == repr(from_fallback)

tox.ini

@@ -10,6 +10,7 @@ isolated_build = true
 [testenv]
 deps=
     pytest
+    hypothesis
 
 changedir=test
 commands=
@@ -23,6 +24,7 @@ setenv=
 basepython=python2.7-x86
 deps=
     pytest
+    hypothesis
 
 changedir=test
 commands=
@@ -34,6 +36,7 @@ commands=
 basepython=python3.4-x86
 deps=
     pytest
+    hypothesis
 
 changedir=test
 commands=