msgpack: support decimal extended type

Tarantool supports decimal type since version 2.2.1 [1]. This patch
introduced the support of Tarantool decimal type in msgpack decoders and
encoders. The Tarantool decimal type is mapped to the native Python
decimal.Decimal type.

Tarantool decimal numbers have 38 digits of precision, that is, the
total number of digits before and after the decimal point can be 38 [2].
If there are more digits arter the decimal point, the precision is lost.
If there are more digits before the decimal point, error is thrown. In
fact, there is also an exceptional case: if decimal starts with `0.`,
38 digits after the decimal point are supported without the loss of
precision. msgpack encoder checks if everything is alright. If number is
not a valid Tarantool decimal, the error is raised. If precision will be
lost on conversion, warning is issued.

Any Tarantool decimal could be converted to a Python decimal without the
loss of precision. Python decimals have its own user alterable precision
(defaulting to 28 places), but it's related only to arithmetic
operations: we can allocate 38-placed decimal disregarding of what
decimal module configuration is used [3].

1. tarantool/tarantool#692
2. https://www.tarantool.io/ru/doc/latest/reference/reference_lua/decimal/
3. https://docs.python.org/3/library/decimal.html

Closed #203

Author: DifferentialOrange

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 ### Added
+- Decimal type support (#203).
 
 ### Changed
 - Bump msgpack requirement to 1.0.4 (PR #223).

tarantool/error.py

@@ -109,10 +109,20 @@ class ConfigurationError(Error):
     Error of initialization with a user-provided configuration.
     '''
 
+class MsgpackError(Error):
+    '''
+    Error with encoding or decoding of MP_EXT types
+    '''
+
+class MsgpackWarning(UserWarning):
+    '''
+    Warning with encoding or decoding of MP_EXT types
+    '''
 
 __all__ = ("Warning", "Error", "InterfaceError", "DatabaseError", "DataError",
            "OperationalError", "IntegrityError", "InternalError",
-           "ProgrammingError", "NotSupportedError")
+           "ProgrammingError", "NotSupportedError", "MsgpackError",
+           "MsgpackWarning")
 
 # Monkey patch os.strerror for win32
 if sys.platform == "win32":

tarantool/ext_types/decimal.py

@@ -0,0 +1,226 @@
+from decimal import Decimal
+
+from tarantool.error import MsgpackError, MsgpackWarning, warn
+
+# https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
+#
+# The decimal MessagePack representation looks like this:
+# +--------+-------------------+------------+===============+
+# | MP_EXT | length (optional) | MP_DECIMAL | PackedDecimal |
+# +--------+-------------------+------------+===============+
+#
+# PackedDecimal has the following structure:
+#
+#  <--- length bytes -->
+# +-------+=============+
+# | scale |     BCD     |
+# +-------+=============+
+#
+# Here scale is either MP_INT or MP_UINT.
+# scale = number of digits after the decimal point
+#
+# BCD is a sequence of bytes representing decimal digits of the encoded number
+# (each byte has two decimal digits each encoded using 4-bit nibbles), so
+# byte >> 4 is the first digit  and byte & 0x0f is the second digit. The
+# leftmost digit in the array is the most significant. The rightmost digit in
+# the array is the least significant.
+#
+# The first byte of the BCD array contains the first digit of the number,
+# represented as follows:
+#
+# |  4 bits           |  4 bits           |
+#    = 0x                = the 1st digit
+#
+# (The first nibble contains 0 if the decimal number has an even number of
+# digits.) The last byte of the BCD array contains the last digit of the number
+# and the final nibble, represented as follows:
+#
+# |  4 bits           |  4 bits           |
+#    = the last digit    = nibble
+#
+# The final nibble represents the number’s sign:
+#
+#     0x0a, 0x0c, 0x0e, 0x0f stand for plus,
+#     0x0b and 0x0d stand for minus.
+
+EXT_ID = 1
+
+TARANTOOL_DECIMAL_MAX_DIGITS = 38
+
+def get_mp_sign(sign):
+    if sign == '+':
+        return 0x0c
+
+    if sign == '-':
+        return 0x0d
+
+    raise RuntimeError
+
+def add_mp_digit(digit, bytes_reverted, digit_count):
+    if digit_count % 2 == 0:
+        bytes_reverted[-1] = bytes_reverted[-1] | (digit << 4)
+    else:
+        bytes_reverted.append(digit)
+
+def check_valid_tarantool_decimal(str_repr, scale, first_digit_ind):
+# Decimal numbers have 38 digits of precision, that is, the total number of
+# digits before and after the decimal point can be 38. If there are more
+# digits arter the decimal point, the precision is lost. If there are more
+# digits before the decimal point, error is thrown.
+#
+# Tarantool 2.10.1-0-g482d91c66
+#
+# tarantool> decimal.new('10000000000000000000000000000000000000')
+# ---
+# - 10000000000000000000000000000000000000
+# ...
+#
+# tarantool> decimal.new('100000000000000000000000000000000000000')
+# ---
+# - error: '[string "return VERSION"]:1: variable ''VERSION'' is not declared'
+# ...
+#
+# tarantool> decimal.new('1.0000000000000000000000000000000000001')
+# ---
+# - 1.0000000000000000000000000000000000001
+# ...
+#
+# tarantool> decimal.new('1.00000000000000000000000000000000000001')
+# ---
+# - 1.0000000000000000000000000000000000000
+# ...
+#
+# In fact, there is also an exceptional case: if decimal starts with `0.`,
+# 38 digits after the decimal point are supported without the loss of precision. 
+#
+# tarantool> decimal.new('0.00000000000000000000000000000000000001')
+# ---
+# - 0.00000000000000000000000000000000000001
+# ...
+#
+# tarantool> decimal.new('0.000000000000000000000000000000000000001')
+# ---
+# - 0.00000000000000000000000000000000000000
+# ...
+    if scale > 0:
+        digit_count = len(str_repr) - 1 - first_digit_ind
+    else:
+        digit_count = len(str_repr) - first_digit_ind
+
+    if digit_count <= TARANTOOL_DECIMAL_MAX_DIGITS:
+        return True
+
+    if (digit_count - scale) > TARANTOOL_DECIMAL_MAX_DIGITS:
+        raise MsgpackError('Decimal cannot be encoded: Tarantool decimal ' + \
+                           'supports a maximum of 38 digits.')
+
+    starts_with_zero = str_repr[first_digit_ind] == '0'
+
+    if (    (digit_count > TARANTOOL_DECIMAL_MAX_DIGITS + 1) or \
+            (digit_count == TARANTOOL_DECIMAL_MAX_DIGITS + 1 \
+            and not starts_with_zero)):
+        warn('Decimal encoded with loss of precision: ' + \
+             'Tarantool decimal supports a maximum of 38 digits.',
+             MsgpackWarning)
+        return False
+
+    return True
+
+def strip_decimal_str(str_repr, scale, first_digit_ind):
+    assert scale > 0
+    # Strip extra bytes
+    str_repr = str_repr[:TARANTOOL_DECIMAL_MAX_DIGITS + 1 + first_digit_ind]
+
+    str_repr.rstrip('0')
+    str_repr.rstrip('.')
+    # Do not strips zeroes before the decimal point.
+    return str_repr
+
+def encode(obj):
+    # Non-scientific string with trailing zeroes removed
+    str_repr = format(obj, 'f')
+
+    bytes_reverted = bytearray()
+
+    scale = 0
+    for i in range(len(str_repr)):
+        str_digit = str_repr[i]
+        if str_digit == '.':
+            scale = len(str_repr) - i - 1
+            break
+
+    if str_repr[0] == '-':
+        sign = '-'
+        first_digit_ind = 1
+    else:
+        sign = '+'
+        first_digit_ind = 0
+
+    if not check_valid_tarantool_decimal(str_repr, scale, first_digit_ind):
+        str_repr = strip_decimal_str(str_repr, scale, first_digit_ind)
+
+    bytes_reverted.append(get_mp_sign(sign))
+
+    digit_count = 0
+
+    for i in range(len(str_repr) - 1, first_digit_ind - 1, -1):
+        str_digit = str_repr[i]
+        if str_digit == '.':
+            scale = len(str_repr) - i - 1
+            continue
+
+        add_mp_digit(int(str_digit), bytes_reverted, digit_count)
+        digit_count = digit_count + 1
+
+    # Remove leading zeroes since they already covered by scale
+    for i in range(len(bytes_reverted)):
+        if bytes_reverted[i] != 0:
+            break
+        bytes_reverted.pop()
+
+    bytes_reverted.append(scale)
+
+    return bytes(bytes_reverted[::-1])
+
+
+def get_str_sign(nibble):
+    if nibble == 0x0a or nibble == 0x0c or nibble == 0x0e or nibble == 0x0f:
+        return '+'
+
+    if nibble == 0x0b or nibble == 0x0d:
+        return '-'
+
+    raise MsgpackError('Unexpected MP_DECIMAL sign nibble')
+
+def add_str_digit(digit, digits_reverted, scale):
+    if not (0 <= digit <= 9):
+        raise MsgpackError('Unexpected MP_DECIMAL digit nibble')
+
+    if len(digits_reverted) == scale:
+        digits_reverted.append('.')
+
+    digits_reverted.append(str(digit))
+
+def decode(data):
+    scale = data[0] 
+
+    sign = get_str_sign(data[-1] & 0x0f)
+
+    # Parse from tail since scale is counted from the tail.
+    digits_reverted = []
+
+    add_str_digit((data[-1] & 0xf0) >> 4, digits_reverted, scale)
+
+    for i in range(len(data) - 2, 0, -1):
+        add_str_digit(data[i] & 0x0f, digits_reverted, scale)
+        add_str_digit((data[i] & 0xf0) >> 4, digits_reverted, scale)
+
+    # Add leading zeroes in case of 0.000... number.
+    for i in range(len(digits_reverted), scale + 1):
+        add_str_digit(0, digits_reverted, scale)
+
+    digits_reverted.append(sign)
+
+    str_repr = ''.join(digits_reverted[::-1])
+
+    return Decimal(str_repr)

tarantool/ext_types/packer.py

@@ -0,0 +1,9 @@
+from decimal import Decimal
+from msgpack import ExtType
+
+import tarantool.ext_types.decimal as ext_decimal
+
+def default(obj):
+    if isinstance(obj, Decimal):
+        return ExtType(ext_decimal.EXT_ID, ext_decimal.encode(obj))
+    raise TypeError("Unknown type: %r" % (obj,))

tarantool/ext_types/unpacker.py

@@ -0,0 +1,8 @@
+from decimal import Decimal
+
+import tarantool.ext_types.decimal as ext_decimal
+
+def ext_hook(code, data):
+    if code == ext_decimal.EXT_ID:
+        return ext_decimal.decode(data)
+    raise NotImplementedError("Unknown msgpack type: %d" % (code,))

tarantool/request.py

@@ -59,6 +59,8 @@
     binary_types
 )
 
+from tarantool.ext_types.packer import default as packer_default
+
 class Request(object):
     '''
     Represents a single request to the server in compliance with the
@@ -122,6 +124,8 @@ def __init__(self, conn):
         else:
             packer_kwargs['use_bin_type'] = True
 
+        packer_kwargs['default'] = packer_default
+
         self.packer = msgpack.Packer(**packer_kwargs)
 
     def _dumps(self, src):

tarantool/response.py

@@ -29,6 +29,7 @@
     tnt_strerror
 )
 
+from tarantool.ext_types.unpacker import ext_hook as unpacker_ext_hook
 
 class Response(Sequence):
     '''
@@ -86,6 +87,8 @@ def __init__(self, conn, response):
         if msgpack.version >= (1, 0, 0):
             unpacker_kwargs['strict_map_key'] = False
 
+        unpacker_kwargs['ext_hook'] = unpacker_ext_hook
+
         unpacker = msgpack.Unpacker(**unpacker_kwargs)
 
         unpacker.feed(response)

test/suites/__init__.py

@@ -15,12 +15,13 @@
 from .test_dbapi import TestSuite_DBAPI
 from .test_encoding import TestSuite_Encoding
 from .test_ssl import TestSuite_Ssl
+from .test_ext_types import TestSuite_ExtTypes
 
 test_cases = (TestSuite_Schema_UnicodeConnection,
               TestSuite_Schema_BinaryConnection,
               TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect,
               TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI,
-              TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl)
+              TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl, TestSuite_ExtTypes)
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

test/suites/lib/skip.py

@@ -43,6 +43,38 @@ def wrapper(self, *args, **kwargs):
 
     return wrapper
 
+def skip_or_run_test_pcall_require(func, REQUIRED_TNT_MODULE, msg):
+    """Decorator to skip or run tests depending on tarantool
+    module requre success or fail.
+
+    Also, it can be used with the 'setUp' method for skipping
+    the whole test suite.
+    """
+
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if func.__name__ == 'setUp':
+            func(self, *args, **kwargs)
+
+        srv = None
+
+        if hasattr(self, 'servers'):
+            srv = self.servers[0]
+
+        if hasattr(self, 'srv'):
+            srv = self.srv
+
+        assert srv is not None
+
+        resp = srv.admin("pcall(require, '%s')" % REQUIRED_TNT_MODULE)
+        if not resp[0]:
+            self.skipTest('Tarantool %s' % (msg, ))
+
+        if func.__name__ != 'setUp':
+            func(self, *args, **kwargs)
+
+    return wrapper
+
 
 def skip_or_run_test_python(func, REQUIRED_PYTHON_VERSION, msg):
     """Decorator to skip or run tests depending on the Python version.
@@ -101,3 +133,13 @@ def skip_or_run_conn_pool_test(func):
     return skip_or_run_test_python(func, '3.7',
                                    'does not support ConnectionPool')
 
+def skip_or_run_decimal_test(func):
+    """Decorator to skip or run decimal-related tests depending on
+    the tarantool version.
+
+    Tarantool supports decimal type only since 2.2.1 version.
+    See https://github.com/tarantool/tarantool/issues/692
+    """
+
+    return skip_or_run_test_pcall_require(func, 'decimal',
+                                      'does not support decimal type')